@lokascript/framework 2.0.0 → 2.1.0

package/src/core/pattern-matching/pattern-matcher.ts

@@ -210,7 +210,8 @@ export class PatternMatcher {
   private matchTokenSequence(
     tokens: TokenStream,
     patternTokens: PatternToken[],
-    captured: Map<SemanticRole, SemanticValue>
+    captured: Map<SemanticRole, SemanticValue>,
+    parentStopMarkers?: Set<string>
   ): boolean {
     // Skip leading conjunctions for Arabic (proclitics: و, ف, ول, وب, etc.)
     // BUT NOT if the pattern explicitly expects a conjunction (proclitic patterns)
@@ -251,6 +252,11 @@ export class PatternMatcher {
       // recognized marker keyword or end of input
       if (patternToken.type === 'role' && patternToken.greedy) {
         const stopMarkers = this.collectStopMarkers(patternTokens, i + 1);
+        // Merge parent-level stop markers so greedy roles inside groups
+        // know when to stop for tokens outside the group (e.g., SOV verb keywords)
+        if (parentStopMarkers) {
+          for (const m of parentStopMarkers) stopMarkers.add(m);
+        }
         const values: string[] = [];
         while (!tokens.isAtEnd()) {
           const nextToken = tokens.peek();
@@ -271,10 +277,78 @@ export class PatternMatcher {
         }
       }
 
+      // Group tokens: compute sibling stop markers and forward to inner matching
+      if (patternToken.type === 'group') {
+        const siblingStopMarkers = this.collectStopMarkers(patternTokens, i + 1);
+        if (parentStopMarkers) {
+          for (const m of parentStopMarkers) siblingStopMarkers.add(m);
+        }
+        const groupMatched = this.matchGroupToken(
+          tokens,
+          patternToken as PatternToken & { type: 'group' },
+          captured,
+          siblingStopMarkers
+        );
+        if (groupMatched) {
+          prevOptionalMark = null;
+          prevOptionalRole = null;
+          continue;
+        }
+        if (this.isOptional(patternToken)) {
+          continue;
+        }
+        return false;
+      }
+
       // Save stream position before attempting optional roles
       const isOptionalRole = patternToken.type === 'role' && patternToken.optional === true;
       const markBefore = isOptionalRole ? tokens.mark() : null;
 
+      // SOV marker-stealing guard: before matching a required role, check if
+      // the current input token would match the immediately-following pattern
+      // literal (the role's own postpositional marker). If so, skip consuming
+      // this token as a role value — it should be matched as the literal instead.
+      const isRoleToken = patternToken.type === 'role';
+      if (isRoleToken && !isOptionalRole) {
+        const nextPT = i + 1 < patternTokens.length ? patternTokens[i + 1] : null;
+        if (nextPT?.type === 'literal') {
+          const currentToken = tokens.peek();
+          if (currentToken && this.getMatchType(currentToken, nextPT.value) !== 'none') {
+            // Current token is the marker literal, not a role value — treat role as failed
+            this.logger.debug(
+              '  >> MARKER-STEAL GUARD: current token matches next literal, skipping role'
+            );
+            // Fall through to failure handling below
+            // (which will trigger backtracking of previous optional role)
+
+            // Match failed
+            this.logger.debug('  >> Token match FAILED');
+
+            if (this.isOptional(patternToken)) {
+              continue;
+            }
+
+            // Required token failed — try backtracking over the previous optional role
+            if (prevOptionalMark && prevOptionalRole) {
+              this.logger.debug('  >> BACKTRACKING: undoing optional role', prevOptionalRole);
+              tokens.reset(prevOptionalMark);
+              captured.delete(prevOptionalRole);
+              prevOptionalMark = null;
+              prevOptionalRole = null;
+
+              // Retry the current (failed) pattern token from the restored position
+              const retryMatched = this.matchPatternToken(tokens, patternToken, captured);
+              this.logger.debug('  >> Backtrack retry result:', retryMatched);
+              if (retryMatched) {
+                continue;
+              }
+            }
+
+            return false;
+          }
+        }
+      }
+
       const matched = this.matchPatternToken(tokens, patternToken, captured);
       this.logger.debug('  >> Match result:', matched);
 
@@ -436,10 +510,7 @@ export class PatternMatcher {
     if (possessiveValue) {
       // Validate expected types if specified
       if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
-        if (
-          !patternToken.expectedTypes.includes(possessiveValue.type) &&
-          !patternToken.expectedTypes.includes('expression')
-        ) {
+        if (!isTypeCompatible(possessiveValue.type, patternToken.expectedTypes)) {
           return patternToken.optional || false;
         }
       }
@@ -451,10 +522,7 @@ export class PatternMatcher {
     const methodCallValue = this.tryMatchMethodCallExpression(tokens);
     if (methodCallValue) {
       if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
-        if (
-          !patternToken.expectedTypes.includes(methodCallValue.type) &&
-          !patternToken.expectedTypes.includes('expression')
-        ) {
+        if (!isTypeCompatible(methodCallValue.type, patternToken.expectedTypes)) {
           return patternToken.optional || false;
         }
       }
@@ -479,10 +547,7 @@ export class PatternMatcher {
     const propertyAccessValue = this.tryMatchPropertyAccessExpression(tokens);
     if (propertyAccessValue) {
       if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
-        if (
-          !patternToken.expectedTypes.includes(propertyAccessValue.type) &&
-          !patternToken.expectedTypes.includes('expression')
-        ) {
+        if (!isTypeCompatible(propertyAccessValue.type, patternToken.expectedTypes)) {
           return patternToken.optional || false;
         }
       }
@@ -928,12 +993,18 @@ export class PatternMatcher {
   private matchGroupToken(
     tokens: TokenStream,
     patternToken: PatternToken & { type: 'group' },
-    captured: Map<SemanticRole, SemanticValue>
+    captured: Map<SemanticRole, SemanticValue>,
+    parentStopMarkers?: Set<string>
   ): boolean {
     const mark = tokens.mark();
     const capturedBefore = new Set(captured.keys());
 
-    const success = this.matchTokenSequence(tokens, patternToken.tokens, captured);
+    const success = this.matchTokenSequence(
+      tokens,
+      patternToken.tokens,
+      captured,
+      parentStopMarkers
+    );
     if (success) return true;
 
     // Reset from failed attempt
@@ -965,7 +1036,12 @@ export class PatternMatcher {
           for (let s = 0; s < offset; s++) tokens.advance();
 
           // Retry group match from marker position
-          const retrySuccess = this.matchTokenSequence(tokens, patternToken.tokens, captured);
+          const retrySuccess = this.matchTokenSequence(
+            tokens,
+            patternToken.tokens,
+            captured,
+            parentStopMarkers
+          );
           if (retrySuccess) return true;
 
           // Retry failed — full reset
@@ -1050,7 +1126,10 @@ export class PatternMatcher {
             break;
           }
         }
-        break;
+        // For optional groups, continue scanning past them so the greedy
+        // capture also sees subsequent required literals (e.g., the SOV verb
+        // keyword after an optional WHERE group).
+        if (!pt.optional) break;
       }
     }
     return markers;
@@ -1061,8 +1140,12 @@ export class PatternMatcher {
    */
   private isStopMarker(token: LanguageToken, stopMarkers: Set<string>): boolean {
     if (stopMarkers.size === 0) return false;
-    const value = (token.normalized || token.value).toLowerCase();
-    return stopMarkers.has(value);
+    // Check both raw value and normalized form against stop markers.
+    // Pattern literals use native keywords (e.g., '選択') while tokenizers may
+    // set normalized to the English equivalent (e.g., 'select'). Both must match.
+    if (stopMarkers.has(token.value.toLowerCase())) return true;
+    if (token.normalized && stopMarkers.has(token.normalized.toLowerCase())) return true;
+    return false;
   }
 
   /**

package/src/core/types.ts

@@ -56,7 +56,8 @@ export type SemanticValue =
   | SelectorValue
   | ReferenceValue
   | PropertyPathValue
-  | ExpressionValue;
+  | ExpressionValue
+  | FlagValue;
 
 /**
  * Expected value types for role tokens.
@@ -93,6 +94,49 @@ export interface ExpressionValue {
   readonly raw: string;
 }
 
+/**
+ * A boolean flag value — present (+flag) or negated (~flag).
+ * Used in declarative domains for no-value attributes like primary-key, not-null.
+ */
+export interface FlagValue {
+  readonly type: 'flag';
+  readonly name: string;
+  readonly enabled: boolean;
+}
+
+// =============================================================================
+// Annotations & Diagnostics (v1.2)
+// =============================================================================
+
+/** A metadata annotation on a node (v1.2). */
+export interface Annotation {
+  readonly name: string;
+  readonly value?: string;
+}
+
+/**
+ * A type constraint diagnostic attached to a node (v1.2).
+ *
+ * Named `ProtocolDiagnostic` to avoid collision with the framework's own
+ * `Diagnostic` type (which has `severity: 'error' | 'warning' | 'info'`
+ * and `suggestion?: string`).
+ */
+export interface ProtocolDiagnostic {
+  readonly level: 'error' | 'warning';
+  readonly role: string;
+  readonly message: string;
+  readonly code: string;
+}
+
+/** Async coordination variant (v1.2). */
+export type AsyncVariant = 'all' | 'race';
+
+/** A single arm in a match command (v1.2). */
+export interface MatchArm {
+  readonly pattern: SemanticValue;
+  readonly body: SemanticNode[];
+}
+
 // =============================================================================
 // Semantic Nodes
 // =============================================================================
@@ -106,6 +150,10 @@ export interface SemanticNode {
   readonly action: ActionType;
   readonly roles: ReadonlyMap<SemanticRole, SemanticValue>;
   readonly metadata?: SemanticMetadata;
+  /** Metadata annotations (v1.2). */
+  readonly annotations?: readonly Annotation[];
+  /** Type constraint diagnostics (v1.2). */
+  readonly diagnostics?: readonly ProtocolDiagnostic[];
 }
 
 /**
@@ -137,9 +185,27 @@ export interface SourcePosition {
 
 /**
  * A command semantic node - represents a single DSL command.
+ *
+ * v1.2 adds optional fields for try/catch/finally, async coordination (all/race),
+ * and pattern matching (match/arms). These are encoded as fields on command nodes
+ * (matching the protocol wire format) rather than new node kinds.
  */
 export interface CommandSemanticNode extends SemanticNode {
   readonly kind: 'command';
+  /** try body — commands to execute in the try block (v1.2). */
+  readonly body?: readonly SemanticNode[];
+  /** catch branch — commands to execute on error (v1.2). */
+  readonly catchBranch?: readonly SemanticNode[];
+  /** finally branch — cleanup commands that always execute (v1.2). */
+  readonly finallyBranch?: readonly SemanticNode[];
+  /** Async coordination variant: all (wait for all) or race (first wins) (v1.2). */
+  readonly asyncVariant?: AsyncVariant;
+  /** Async body — concurrent commands for all/race (v1.2). */
+  readonly asyncBody?: readonly SemanticNode[];
+  /** Match arms — pattern/body pairs for match command (v1.2). */
+  readonly arms?: readonly MatchArm[];
+  /** Default arm — executed when no match arm matches (v1.2). */
+  readonly defaultArm?: readonly SemanticNode[];
 }
 
 /**
@@ -177,7 +243,7 @@ export interface ConditionalSemanticNode extends SemanticNode {
 export interface CompoundSemanticNode extends SemanticNode {
   readonly kind: 'compound';
   readonly statements: SemanticNode[];
-  readonly chainType: 'then' | 'and' | 'async' | 'sequential';
+  readonly chainType: 'then' | 'and' | 'async' | 'sequential' | 'pipe';
 }
 
 /**
@@ -444,6 +510,13 @@ export function createExpression(raw: string): ExpressionValue {
   return { type: 'expression', raw };
 }
 
+/**
+ * Create a boolean flag value (+flag or ~flag)
+ */
+export function createFlag(name: string, enabled: boolean = true): FlagValue {
+  return { type: 'flag', name, enabled };
+}
+
 /**
  * Create a command semantic node
  */
@@ -542,6 +615,7 @@ export function createCompoundNode(
  * eliminating the need for `as any` casts in domain code generators.
  */
 export function extractValue(value: SemanticValue): string {
+  if (value.type === 'flag') return value.name;
   if ('raw' in value && value.raw !== undefined) return String(value.raw);
   if ('value' in value && value.value !== undefined) return String(value.value);
   if (value.type === 'property-path') return `${extractValue(value.object)}.${value.property}`;
@@ -587,3 +661,72 @@ export function createLoopNode(
     ...(metadata && { metadata }),
   };
 }
+
+// =============================================================================
+// v1.2 Factory Helpers
+// =============================================================================
+
+/**
+ * Create a try/catch/finally command node (v1.2).
+ */
+export function createTryNode(
+  body: SemanticNode[],
+  catchBranch?: SemanticNode[],
+  finallyBranch?: SemanticNode[],
+  metadata?: SemanticMetadata
+): CommandSemanticNode {
+  return {
+    kind: 'command',
+    action: 'try',
+    roles: new Map(),
+    body,
+    ...(catchBranch && catchBranch.length > 0 ? { catchBranch } : {}),
+    ...(finallyBranch && finallyBranch.length > 0 ? { finallyBranch } : {}),
+    ...(metadata ? { metadata } : {}),
+  };
+}
+
+/**
+ * Create an async coordination (all/race) command node (v1.2).
+ */
+export function createAsyncNode(
+  variant: AsyncVariant,
+  asyncBody: SemanticNode[],
+  metadata?: SemanticMetadata
+): CommandSemanticNode {
+  return {
+    kind: 'command',
+    action: variant,
+    roles: new Map(),
+    asyncVariant: variant,
+    asyncBody,
+    ...(metadata ? { metadata } : {}),
+  };
+}
+
+/**
+ * Create a match command node with pattern arms (v1.2).
+ */
+export function createMatchNode(
+  roles: Record<SemanticRole, SemanticValue> | Map<SemanticRole, SemanticValue>,
+  arms: MatchArm[],
+  defaultArm?: SemanticNode[],
+  metadata?: SemanticMetadata
+): CommandSemanticNode {
+  const rolesMap = roles instanceof Map ? roles : new Map(Object.entries(roles));
+  return {
+    kind: 'command',
+    action: 'match',
+    roles: rolesMap,
+    arms,
+    ...(defaultArm && defaultArm.length > 0 ? { defaultArm } : {}),
+    ...(metadata ? { metadata } : {}),
+  };
+}
+
+/** Wire format envelope with version metadata (v1.2). */
+export interface LSEEnvelope {
+  readonly lseVersion: string;
+  readonly features?: readonly string[];
+  readonly nodes: readonly SemanticNode[];
+}

package/src/feedback/confidence-gate.test.ts

@@ -0,0 +1,111 @@
+/**
+ * Tests for Confidence-Aware Disambiguation
+ */
+
+import { describe, it, expect } from 'vitest';
+import { needsDisambiguation, buildDisambiguation } from './confidence-gate';
+import { createCommandNode, createSelector, createLiteral } from '../core/types';
+
+// =============================================================================
+// needsDisambiguation()
+// =============================================================================
+
+describe('needsDisambiguation', () => {
+  it('returns false for high confidence', () => {
+    expect(needsDisambiguation(0.9)).toBe(false);
+  });
+
+  it('returns false for very low confidence', () => {
+    expect(needsDisambiguation(0.3)).toBe(false);
+  });
+
+  it('returns true for borderline confidence', () => {
+    expect(needsDisambiguation(0.6)).toBe(true);
+  });
+
+  it('returns true at exact low threshold', () => {
+    expect(needsDisambiguation(0.5)).toBe(true);
+  });
+
+  it('returns false at exact high threshold', () => {
+    expect(needsDisambiguation(0.7)).toBe(false);
+  });
+
+  it('supports custom thresholds', () => {
+    expect(needsDisambiguation(0.45, 0.4, 0.6)).toBe(true);
+    expect(needsDisambiguation(0.3, 0.4, 0.6)).toBe(false);
+    expect(needsDisambiguation(0.7, 0.4, 0.6)).toBe(false);
+  });
+});
+
+// =============================================================================
+// buildDisambiguation()
+// =============================================================================
+
+describe('buildDisambiguation', () => {
+  const nodeA = createCommandNode('toggle', { patient: createSelector('#button') });
+  const nodeB = createCommandNode('toggle', { patient: createSelector('.button') });
+
+  it('builds a disambiguation request', () => {
+    const result = buildDisambiguation('toggle button', [
+      { action: 'toggle', confidence: 0.62, node: nodeA, description: 'toggle class on #button' },
+      { action: 'toggle', confidence: 0.58, node: nodeB, description: 'toggle .button class' },
+    ]);
+
+    expect(result.input).toBe('toggle button');
+    expect(result.candidates.length).toBe(2);
+    expect(result.question).toContain('0.62');
+  });
+
+  it('sorts candidates by confidence (highest first)', () => {
+    const result = buildDisambiguation('test', [
+      { action: 'add', confidence: 0.5, node: nodeB },
+      { action: 'toggle', confidence: 0.65, node: nodeA },
+    ]);
+
+    expect(result.candidates[0].action).toBe('toggle');
+    expect(result.candidates[1].action).toBe('add');
+  });
+
+  it('includes bracket syntax in candidates', () => {
+    const result = buildDisambiguation('test', [
+      { action: 'toggle', confidence: 0.6, node: nodeA },
+    ]);
+
+    expect(result.candidates[0].explicit).toContain('[toggle');
+  });
+
+  it('generates option labels (a, b, c)', () => {
+    const result = buildDisambiguation('test', [
+      { action: 'a', confidence: 0.6, node: nodeA },
+      { action: 'b', confidence: 0.55, node: nodeB },
+    ]);
+
+    expect(result.question).toContain('(a)');
+    expect(result.question).toContain('(b)');
+  });
+
+  it('includes Which did you mean? prompt', () => {
+    const result = buildDisambiguation('test', [
+      { action: 'toggle', confidence: 0.6, node: nodeA },
+    ]);
+
+    expect(result.question).toContain('Which did you mean?');
+  });
+
+  it('uses custom description when provided', () => {
+    const result = buildDisambiguation('test', [
+      { action: 'toggle', confidence: 0.6, node: nodeA, description: 'Custom description' },
+    ]);
+
+    expect(result.candidates[0].description).toBe('Custom description');
+  });
+
+  it('generates default description when not provided', () => {
+    const result = buildDisambiguation('test', [
+      { action: 'toggle', confidence: 0.6, node: nodeA },
+    ]);
+
+    expect(result.candidates[0].description).toContain('toggle');
+  });
+});

package/src/feedback/confidence-gate.ts

@@ -0,0 +1,70 @@
+/**
+ * Confidence-Aware Disambiguation
+ *
+ * When parse confidence is borderline (between low and high thresholds),
+ * generates a disambiguation request with alternative interpretations
+ * instead of blindly accepting or rejecting.
+ */
+
+import type { SemanticNode } from '../core/types';
+import { renderExplicit } from '../ir/explicit-renderer';
+import type { DisambiguationRequest, DisambiguationCandidate } from './types';
+
+// =============================================================================
+// Public API
+// =============================================================================
+
+/**
+ * Check whether a confidence score falls in the disambiguation range.
+ *
+ * @returns true if the confidence is ambiguous (between low and high thresholds)
+ */
+export function needsDisambiguation(
+  confidence: number,
+  low: number = 0.5,
+  high: number = 0.7
+): boolean {
+  return confidence >= low && confidence < high;
+}
+
+/**
+ * Build a disambiguation request from candidate interpretations.
+ *
+ * @param input - The original ambiguous input
+ * @param candidates - Alternative interpretations with confidence scores
+ */
+export function buildDisambiguation(
+  input: string,
+  candidates: ReadonlyArray<{
+    action: string;
+    confidence: number;
+    node: SemanticNode;
+    description?: string;
+  }>
+): DisambiguationRequest {
+  const sortedCandidates = [...candidates].sort((a, b) => b.confidence - a.confidence);
+
+  const disambiguationCandidates: DisambiguationCandidate[] = sortedCandidates.map((c, i) => ({
+    action: c.action,
+    confidence: c.confidence,
+    explicit: renderExplicit(c.node),
+    description: c.description || `${c.action} (option ${String.fromCharCode(97 + i)})`,
+  }));
+
+  const optionLines = disambiguationCandidates.map(
+    (c, i) =>
+      `  (${String.fromCharCode(97 + i)}) ${c.explicit} — ${c.description} (conf: ${c.confidence.toFixed(2)})`
+  );
+
+  const question = [
+    `Your input parsed with confidence ${sortedCandidates[0]?.confidence.toFixed(2) ?? '?'}.`,
+    ...optionLines,
+    'Which did you mean?',
+  ].join('\n');
+
+  return {
+    input,
+    candidates: disambiguationCandidates,
+    question,
+  };
+}