@lokascript/framework 2.0.0 → 2.2.0

package/src/core/pattern-matching/pattern-matcher.ts

@@ -210,7 +210,8 @@ export class PatternMatcher {
   private matchTokenSequence(
     tokens: TokenStream,
     patternTokens: PatternToken[],
-    captured: Map<SemanticRole, SemanticValue>
+    captured: Map<SemanticRole, SemanticValue>,
+    parentStopMarkers?: Set<string>
   ): boolean {
     // Skip leading conjunctions for Arabic (proclitics: و, ف, ول, وب, etc.)
     // BUT NOT if the pattern explicitly expects a conjunction (proclitic patterns)
@@ -251,6 +252,11 @@ export class PatternMatcher {
       // recognized marker keyword or end of input
       if (patternToken.type === 'role' && patternToken.greedy) {
         const stopMarkers = this.collectStopMarkers(patternTokens, i + 1);
+        // Merge parent-level stop markers so greedy roles inside groups
+        // know when to stop for tokens outside the group (e.g., SOV verb keywords)
+        if (parentStopMarkers) {
+          for (const m of parentStopMarkers) stopMarkers.add(m);
+        }
         const values: string[] = [];
         while (!tokens.isAtEnd()) {
           const nextToken = tokens.peek();
@@ -271,10 +277,78 @@ export class PatternMatcher {
         }
       }
 
+      // Group tokens: compute sibling stop markers and forward to inner matching
+      if (patternToken.type === 'group') {
+        const siblingStopMarkers = this.collectStopMarkers(patternTokens, i + 1);
+        if (parentStopMarkers) {
+          for (const m of parentStopMarkers) siblingStopMarkers.add(m);
+        }
+        const groupMatched = this.matchGroupToken(
+          tokens,
+          patternToken as PatternToken & { type: 'group' },
+          captured,
+          siblingStopMarkers
+        );
+        if (groupMatched) {
+          prevOptionalMark = null;
+          prevOptionalRole = null;
+          continue;
+        }
+        if (this.isOptional(patternToken)) {
+          continue;
+        }
+        return false;
+      }
+
       // Save stream position before attempting optional roles
       const isOptionalRole = patternToken.type === 'role' && patternToken.optional === true;
       const markBefore = isOptionalRole ? tokens.mark() : null;
 
+      // SOV marker-stealing guard: before matching a required role, check if
+      // the current input token would match the immediately-following pattern
+      // literal (the role's own postpositional marker). If so, skip consuming
+      // this token as a role value — it should be matched as the literal instead.
+      const isRoleToken = patternToken.type === 'role';
+      if (isRoleToken && !isOptionalRole) {
+        const nextPT = i + 1 < patternTokens.length ? patternTokens[i + 1] : null;
+        if (nextPT?.type === 'literal') {
+          const currentToken = tokens.peek();
+          if (currentToken && this.getMatchType(currentToken, nextPT.value) !== 'none') {
+            // Current token is the marker literal, not a role value — treat role as failed
+            this.logger.debug(
+              '  >> MARKER-STEAL GUARD: current token matches next literal, skipping role'
+            );
+            // Fall through to failure handling below
+            // (which will trigger backtracking of previous optional role)
+
+            // Match failed
+            this.logger.debug('  >> Token match FAILED');
+
+            if (this.isOptional(patternToken)) {
+              continue;
+            }
+
+            // Required token failed — try backtracking over the previous optional role
+            if (prevOptionalMark && prevOptionalRole) {
+              this.logger.debug('  >> BACKTRACKING: undoing optional role', prevOptionalRole);
+              tokens.reset(prevOptionalMark);
+              captured.delete(prevOptionalRole);
+              prevOptionalMark = null;
+              prevOptionalRole = null;
+
+              // Retry the current (failed) pattern token from the restored position
+              const retryMatched = this.matchPatternToken(tokens, patternToken, captured);
+              this.logger.debug('  >> Backtrack retry result:', retryMatched);
+              if (retryMatched) {
+                continue;
+              }
+            }
+
+            return false;
+          }
+        }
+      }
+
       const matched = this.matchPatternToken(tokens, patternToken, captured);
       this.logger.debug('  >> Match result:', matched);
 
@@ -436,10 +510,7 @@ export class PatternMatcher {
     if (possessiveValue) {
       // Validate expected types if specified
       if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
-        if (
-          !patternToken.expectedTypes.includes(possessiveValue.type) &&
-          !patternToken.expectedTypes.includes('expression')
-        ) {
+        if (!isTypeCompatible(possessiveValue.type, patternToken.expectedTypes)) {
           return patternToken.optional || false;
         }
       }
@@ -451,10 +522,7 @@ export class PatternMatcher {
     const methodCallValue = this.tryMatchMethodCallExpression(tokens);
     if (methodCallValue) {
       if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
-        if (
-          !patternToken.expectedTypes.includes(methodCallValue.type) &&
-          !patternToken.expectedTypes.includes('expression')
-        ) {
+        if (!isTypeCompatible(methodCallValue.type, patternToken.expectedTypes)) {
           return patternToken.optional || false;
         }
       }
@@ -479,10 +547,7 @@ export class PatternMatcher {
     const propertyAccessValue = this.tryMatchPropertyAccessExpression(tokens);
     if (propertyAccessValue) {
       if (patternToken.expectedTypes && patternToken.expectedTypes.length > 0) {
-        if (
-          !patternToken.expectedTypes.includes(propertyAccessValue.type) &&
-          !patternToken.expectedTypes.includes('expression')
-        ) {
+        if (!isTypeCompatible(propertyAccessValue.type, patternToken.expectedTypes)) {
           return patternToken.optional || false;
         }
       }
@@ -928,12 +993,18 @@ export class PatternMatcher {
   private matchGroupToken(
     tokens: TokenStream,
     patternToken: PatternToken & { type: 'group' },
-    captured: Map<SemanticRole, SemanticValue>
+    captured: Map<SemanticRole, SemanticValue>,
+    parentStopMarkers?: Set<string>
   ): boolean {
     const mark = tokens.mark();
     const capturedBefore = new Set(captured.keys());
 
-    const success = this.matchTokenSequence(tokens, patternToken.tokens, captured);
+    const success = this.matchTokenSequence(
+      tokens,
+      patternToken.tokens,
+      captured,
+      parentStopMarkers
+    );
     if (success) return true;
 
     // Reset from failed attempt
@@ -965,7 +1036,12 @@ export class PatternMatcher {
           for (let s = 0; s < offset; s++) tokens.advance();
 
           // Retry group match from marker position
-          const retrySuccess = this.matchTokenSequence(tokens, patternToken.tokens, captured);
+          const retrySuccess = this.matchTokenSequence(
+            tokens,
+            patternToken.tokens,
+            captured,
+            parentStopMarkers
+          );
           if (retrySuccess) return true;
 
           // Retry failed — full reset
@@ -1050,7 +1126,10 @@ export class PatternMatcher {
             break;
           }
         }
-        break;
+        // For optional groups, continue scanning past them so the greedy
+        // capture also sees subsequent required literals (e.g., the SOV verb
+        // keyword after an optional WHERE group).
+        if (!pt.optional) break;
       }
     }
     return markers;
@@ -1061,8 +1140,12 @@ export class PatternMatcher {
    */
   private isStopMarker(token: LanguageToken, stopMarkers: Set<string>): boolean {
     if (stopMarkers.size === 0) return false;
-    const value = (token.normalized || token.value).toLowerCase();
-    return stopMarkers.has(value);
+    // Check both raw value and normalized form against stop markers.
+    // Pattern literals use native keywords (e.g., '選択') while tokenizers may
+    // set normalized to the English equivalent (e.g., 'select'). Both must match.
+    if (stopMarkers.has(token.value.toLowerCase())) return true;
+    if (token.normalized && stopMarkers.has(token.normalized.toLowerCase())) return true;
+    return false;
   }
 
   /**

package/src/core/tokenization/morphology/base-normalizer.ts

@@ -0,0 +1,249 @@
+/**
+ * BaseMorphologicalNormalizer — Shared base class for language normalizers.
+ *
+ * Provides the common suffix/prefix stripping loop, reflexive verb handling,
+ * and normalize() pipeline. Language-specific normalizers extend this class
+ * and provide their conjugation rules.
+ *
+ * Phase 3.1 of parser-ecosystem-plan-v3.
+ */
+
+import type {
+  MorphologicalNormalizer,
+  NormalizationResult,
+  ConjugationType,
+  SuffixRule,
+  PrefixRule,
+} from './types';
+import { noChange, normalized } from './types';
+
+/**
+ * Conjugation ending rule for verb classes (Romance languages etc.)
+ * Broader than SuffixRule — includes the replacement stem (e.g., strip -ando, add -ar).
+ */
+export interface ConjugationEnding {
+  readonly ending: string;
+  readonly stem: string;
+  readonly confidence: number;
+  readonly type: ConjugationType;
+}
+
+/**
+ * Configuration for BaseMorphologicalNormalizer.
+ * Subclasses provide this in their constructor.
+ */
+export interface NormalizerConfig {
+  /** Language code */
+  readonly language: string;
+
+  /** Minimum word length to attempt normalization */
+  readonly minWordLength?: number;
+
+  /** Minimum stem length after stripping (default: 2) */
+  readonly minStemLength?: number;
+
+  /** Conjugation endings sorted longest-first */
+  readonly endings?: readonly ConjugationEnding[];
+
+  /** Suffix rules (for SuffixRule-style normalizers) */
+  readonly suffixRules?: readonly SuffixRule[];
+
+  /** Prefix rules */
+  readonly prefixRules?: readonly PrefixRule[];
+
+  /** Reflexive suffixes (for Romance languages) */
+  readonly reflexiveSuffixes?: readonly string[];
+
+  /** Infinitive endings (for checking if already normalized) */
+  readonly infinitiveEndings?: readonly string[];
+}
+
+/**
+ * Abstract base class for morphological normalizers.
+ *
+ * Subclasses must implement `isNormalizable()` and can override any
+ * normalization step. The default `normalize()` pipeline is:
+ *
+ * 1. Check if already in dictionary form → noChange
+ * 2. Try reflexive normalization (if reflexiveSuffixes configured)
+ * 3. Try conjugation endings (if endings configured)
+ * 4. Try suffix rules (if suffixRules configured)
+ * 5. Try prefix rules (if prefixRules configured)
+ * 6. Return noChange
+ */
+export abstract class BaseMorphologicalNormalizer implements MorphologicalNormalizer {
+  readonly language: string;
+  protected readonly config: NormalizerConfig;
+
+  constructor(config: NormalizerConfig) {
+    this.language = config.language;
+    this.config = {
+      minWordLength: 3,
+      minStemLength: 2,
+      ...config,
+    };
+  }
+
+  /**
+   * Check if a word can be normalized. Subclasses must implement this
+   * with language-specific script/character detection.
+   */
+  abstract isNormalizable(word: string): boolean;
+
+  /**
+   * Standard normalization pipeline. Override for custom behavior.
+   */
+  normalize(word: string): NormalizationResult {
+    const lower = word.toLowerCase();
+
+    // Check if already in dictionary form
+    if (this.isAlreadyNormalized(lower)) {
+      return noChange(word);
+    }
+
+    // Try reflexive normalization (Romance languages)
+    if (this.config.reflexiveSuffixes) {
+      const reflexive = this.tryReflexiveNormalization(lower);
+      if (reflexive) return reflexive;
+    }
+
+    // Try conjugation endings
+    if (this.config.endings) {
+      const conjugation = this.tryConjugationEndings(lower);
+      if (conjugation) return conjugation;
+    }
+
+    // Try suffix rules
+    if (this.config.suffixRules) {
+      const suffix = this.trySuffixRules(lower);
+      if (suffix) return suffix;
+    }
+
+    // Try prefix rules
+    if (this.config.prefixRules) {
+      const prefix = this.tryPrefixRules(lower);
+      if (prefix) return prefix;
+    }
+
+    return noChange(word);
+  }
+
+  /**
+   * Check if word is already in dictionary form (e.g., ends in -ar/-er/-ir).
+   * Override for language-specific checks.
+   */
+  protected isAlreadyNormalized(word: string): boolean {
+    if (this.config.infinitiveEndings) {
+      return this.config.infinitiveEndings.some(e => word.endsWith(e));
+    }
+    return false;
+  }
+
+  /**
+   * Try to strip reflexive suffixes and normalize the remainder.
+   * Common in Romance languages (Spanish, Portuguese, French).
+   */
+  protected tryReflexiveNormalization(word: string): NormalizationResult | null {
+    const suffixes = this.config.reflexiveSuffixes;
+    if (!suffixes) return null;
+
+    for (const suffix of suffixes) {
+      if (!word.endsWith(suffix)) continue;
+      const remainder = word.slice(0, -suffix.length);
+
+      // Check if remainder is already an infinitive
+      if (this.isAlreadyNormalized(remainder)) {
+        return normalized(remainder, 0.88, {
+          removedSuffixes: [suffix],
+          conjugationType: 'reflexive',
+        });
+      }
+
+      // Try to normalize the remainder
+      const inner = this.tryConjugationEndings(remainder) || this.trySuffixRules(remainder);
+      if (inner && inner.stem !== remainder) {
+        return normalized(inner.stem, inner.confidence * 0.95, {
+          removedSuffixes: [suffix, ...(inner.metadata?.removedSuffixes || [])],
+          conjugationType: 'reflexive',
+        });
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Try conjugation endings (verb class endings like -ar/-er/-ir patterns).
+   * Endings must be pre-sorted longest-first.
+   */
+  protected tryConjugationEndings(word: string): NormalizationResult | null {
+    const endings = this.config.endings;
+    if (!endings) return null;
+
+    const minStem = this.config.minStemLength ?? 2;
+
+    for (const rule of endings) {
+      if (!word.endsWith(rule.ending)) continue;
+
+      const stemBase = word.slice(0, -rule.ending.length);
+      if (stemBase.length < minStem) continue;
+
+      const infinitive = stemBase + rule.stem;
+      return normalized(infinitive, rule.confidence, {
+        removedSuffixes: [rule.ending],
+        conjugationType: rule.type,
+      });
+    }
+
+    return null;
+  }
+
+  /**
+   * Try SuffixRule-style normalization.
+   * Rules must be pre-sorted longest-first.
+   */
+  protected trySuffixRules(word: string): NormalizationResult | null {
+    const rules = this.config.suffixRules;
+    if (!rules) return null;
+
+    const defaultMinStem = this.config.minStemLength ?? 2;
+
+    for (const rule of rules) {
+      if (!word.endsWith(rule.pattern)) continue;
+
+      const stem = word.slice(0, -rule.pattern.length);
+      const minStem = rule.minStemLength ?? defaultMinStem;
+      if (stem.length < minStem) continue;
+
+      const result = stem + (rule.replacement || '');
+      return normalized(result, rule.confidence, {
+        removedSuffixes: [rule.pattern],
+        ...(rule.conjugationType && { conjugationType: rule.conjugationType }),
+      });
+    }
+
+    return null;
+  }
+
+  /**
+   * Try PrefixRule-style normalization.
+   */
+  protected tryPrefixRules(word: string): NormalizationResult | null {
+    const rules = this.config.prefixRules;
+    if (!rules) return null;
+
+    for (const rule of rules) {
+      if (!word.startsWith(rule.pattern)) continue;
+
+      const remainder = word.slice(rule.pattern.length);
+      const minRemaining = rule.minRemaining ?? this.config.minStemLength ?? 2;
+      if (remainder.length < minRemaining) continue;
+
+      return normalized(remainder, 1.0 - rule.confidencePenalty, {
+        removedPrefixes: [rule.pattern],
+      });
+    }
+
+    return null;
+  }
+}

package/src/core/tokenization/morphology/index.ts

@@ -1,5 +1,7 @@
 /**
- * Morphological normalization types and interfaces
+ * Morphological normalization types, interfaces, and base class
  */
 
 export * from './types';
+export { BaseMorphologicalNormalizer } from './base-normalizer';
+export type { ConjugationEnding, NormalizerConfig } from './base-normalizer';

package/src/core/types.ts

@@ -8,6 +8,8 @@
  * These types are domain-agnostic - they work for any DSL (SQL, animations, etc.)
  */
 
+import type { Diagnostic } from '../generation/diagnostics';
+
 // =============================================================================
 // Action Types (Generic)
 // =============================================================================
@@ -56,7 +58,8 @@ export type SemanticValue =
   | SelectorValue
   | ReferenceValue
   | PropertyPathValue
-  | ExpressionValue;
+  | ExpressionValue
+  | FlagValue;
 
 /**
  * Expected value types for role tokens.
@@ -93,6 +96,49 @@ export interface ExpressionValue {
   readonly raw: string;
 }
 
+/**
+ * A boolean flag value — present (+flag) or negated (~flag).
+ * Used in declarative domains for no-value attributes like primary-key, not-null.
+ */
+export interface FlagValue {
+  readonly type: 'flag';
+  readonly name: string;
+  readonly enabled: boolean;
+}
+
+// =============================================================================
+// Annotations & Diagnostics (v1.2)
+// =============================================================================
+
+/** A metadata annotation on a node (v1.2). */
+export interface Annotation {
+  readonly name: string;
+  readonly value?: string;
+}
+
+/**
+ * A type constraint diagnostic attached to a node (v1.2).
+ *
+ * Named `ProtocolDiagnostic` to avoid collision with the framework's own
+ * `Diagnostic` type (which has `severity: 'error' | 'warning' | 'info'`
+ * and `suggestion?: string`).
+ */
+export interface ProtocolDiagnostic {
+  readonly level: 'error' | 'warning';
+  readonly role: string;
+  readonly message: string;
+  readonly code: string;
+}
+
+/** Async coordination variant (v1.2). */
+export type AsyncVariant = 'all' | 'race';
+
+/** A single arm in a match command (v1.2). */
+export interface MatchArm {
+  readonly pattern: SemanticValue;
+  readonly body: SemanticNode[];
+}
+
 // =============================================================================
 // Semantic Nodes
 // =============================================================================
@@ -106,6 +152,10 @@ export interface SemanticNode {
   readonly action: ActionType;
   readonly roles: ReadonlyMap<SemanticRole, SemanticValue>;
   readonly metadata?: SemanticMetadata;
+  /** Metadata annotations (v1.2). */
+  readonly annotations?: readonly Annotation[];
+  /** Diagnostics from parsing, validation, and schema checks (v1.2, extended v1.2.1). */
+  readonly diagnostics?: readonly Diagnostic[];
 }
 
 /**
@@ -137,9 +187,27 @@ export interface SourcePosition {
 
 /**
  * A command semantic node - represents a single DSL command.
+ *
+ * v1.2 adds optional fields for try/catch/finally, async coordination (all/race),
+ * and pattern matching (match/arms). These are encoded as fields on command nodes
+ * (matching the protocol wire format) rather than new node kinds.
  */
 export interface CommandSemanticNode extends SemanticNode {
   readonly kind: 'command';
+  /** try body — commands to execute in the try block (v1.2). */
+  readonly body?: readonly SemanticNode[];
+  /** catch branch — commands to execute on error (v1.2). */
+  readonly catchBranch?: readonly SemanticNode[];
+  /** finally branch — cleanup commands that always execute (v1.2). */
+  readonly finallyBranch?: readonly SemanticNode[];
+  /** Async coordination variant: all (wait for all) or race (first wins) (v1.2). */
+  readonly asyncVariant?: AsyncVariant;
+  /** Async body — concurrent commands for all/race (v1.2). */
+  readonly asyncBody?: readonly SemanticNode[];
+  /** Match arms — pattern/body pairs for match command (v1.2). */
+  readonly arms?: readonly MatchArm[];
+  /** Default arm — executed when no match arm matches (v1.2). */
+  readonly defaultArm?: readonly SemanticNode[];
 }
 
 /**
@@ -177,7 +245,7 @@ export interface ConditionalSemanticNode extends SemanticNode {
 export interface CompoundSemanticNode extends SemanticNode {
   readonly kind: 'compound';
   readonly statements: SemanticNode[];
-  readonly chainType: 'then' | 'and' | 'async' | 'sequential';
+  readonly chainType: 'then' | 'and' | 'async' | 'sequential' | 'pipe';
 }
 
 /**
@@ -444,6 +512,13 @@ export function createExpression(raw: string): ExpressionValue {
   return { type: 'expression', raw };
 }
 
+/**
+ * Create a boolean flag value (+flag or ~flag)
+ */
+export function createFlag(name: string, enabled: boolean = true): FlagValue {
+  return { type: 'flag', name, enabled };
+}
+
 /**
  * Create a command semantic node
  */
@@ -542,6 +617,7 @@ export function createCompoundNode(
  * eliminating the need for `as any` casts in domain code generators.
  */
 export function extractValue(value: SemanticValue): string {
+  if (value.type === 'flag') return value.name;
   if ('raw' in value && value.raw !== undefined) return String(value.raw);
   if ('value' in value && value.value !== undefined) return String(value.value);
   if (value.type === 'property-path') return `${extractValue(value.object)}.${value.property}`;
@@ -587,3 +663,72 @@ export function createLoopNode(
     ...(metadata && { metadata }),
   };
 }
+
+// =============================================================================
+// v1.2 Factory Helpers
+// =============================================================================
+
+/**
+ * Create a try/catch/finally command node (v1.2).
+ */
+export function createTryNode(
+  body: SemanticNode[],
+  catchBranch?: SemanticNode[],
+  finallyBranch?: SemanticNode[],
+  metadata?: SemanticMetadata
+): CommandSemanticNode {
+  return {
+    kind: 'command',
+    action: 'try',
+    roles: new Map(),
+    body,
+    ...(catchBranch && catchBranch.length > 0 ? { catchBranch } : {}),
+    ...(finallyBranch && finallyBranch.length > 0 ? { finallyBranch } : {}),
+    ...(metadata ? { metadata } : {}),
+  };
+}
+
+/**
+ * Create an async coordination (all/race) command node (v1.2).
+ */
+export function createAsyncNode(
+  variant: AsyncVariant,
+  asyncBody: SemanticNode[],
+  metadata?: SemanticMetadata
+): CommandSemanticNode {
+  return {
+    kind: 'command',
+    action: variant,
+    roles: new Map(),
+    asyncVariant: variant,
+    asyncBody,
+    ...(metadata ? { metadata } : {}),
+  };
+}
+
+/**
+ * Create a match command node with pattern arms (v1.2).
+ */
+export function createMatchNode(
+  roles: Record<SemanticRole, SemanticValue> | Map<SemanticRole, SemanticValue>,
+  arms: MatchArm[],
+  defaultArm?: SemanticNode[],
+  metadata?: SemanticMetadata
+): CommandSemanticNode {
+  const rolesMap = roles instanceof Map ? roles : new Map(Object.entries(roles));
+  return {
+    kind: 'command',
+    action: 'match',
+    roles: rolesMap,
+    arms,
+    ...(defaultArm && defaultArm.length > 0 ? { defaultArm } : {}),
+    ...(metadata ? { metadata } : {}),
+  };
+}
+
+/** Wire format envelope with version metadata (v1.2). */
+export interface LSEEnvelope {
+  readonly lseVersion: string;
+  readonly features?: readonly string[];
+  readonly nodes: readonly SemanticNode[];
+}