@lokascript/framework 2.5.1 → 2.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lokascript/framework",
-  "version": "2.5.1",
+  "version": "2.6.0",
   "description": "Generic framework for building multilingual DSLs with semantic parsing and grammar transformation",
   "type": "module",
   "main": "dist/index.cjs",
@@ -69,7 +69,7 @@
     "test": "vitest",
     "test:run": "vitest run",
     "test:watch": "vitest watch",
-    "test:check": "vitest run --reporter=dot 2>&1 | tail -5",
+    "test:check": "VITEST_QUIET=1 bash ../../scripts/vitest-run.sh --reporter=dot",
     "test:coverage": "vitest run --coverage",
     "test:unit": "vitest run src/core/**/*.test.ts src/generation/**/*.test.ts",
     "test:integration": "vitest run src/__test__/**/*.test.ts src/api/**/*.test.ts",
@@ -96,7 +96,7 @@
   "author": "LokaScript Contributors",
   "license": "MIT",
   "dependencies": {
-    "@lokascript/intent": "^2.5.1"
+    "@lokascript/intent": "^2.6.0"
   },
   "devDependencies": {
     "@types/node": "^20.0.0",

package/src/core/tokenization/base-tokenizer.ts

@@ -32,6 +32,61 @@ import { getDefaultExtractors } from './default-extractors';
 // Uses the canonical list from OperatorExtractor to avoid duplication.
 const SIMPLE_TOKENIZER_OPERATOR_SET = new Set(DEFAULT_OPERATORS);
 
+/**
+ * Normalized concepts that are matched via the pattern matcher's ROLE-MARKER
+ * MECHANISM (the source/destination/event clause matchers in
+ * `packages/semantic/src/parser/pattern-matcher.ts`), which peeks/advances a
+ * SINGLE token and checks `.value`/`.normalized`. A multi-word phrase carrying
+ * one of these must NOT be pre-matched as a single keyword token — doing so
+ * shadows the single-word marker those clause matchers expect (e.g. id
+ * `ke dalam`=into hides the `ke` destination marker; ko `할 때`=eventMarker
+ * pre-empts SOV event extraction).
+ *
+ * NOTE — what is *not* here. Prepositional modifiers that the generated patterns
+ * expose as ordinary pattern LITERALS (`before`/`after` in put-before/after,
+ * `until` in repeat-until) are deliberately absent: those are read by
+ * `matchLiteralToken`, which compares the whole token by exact value OR
+ * normalized form (`getMatchType`), so a multi-word marker token (`से पहले`,
+ * `cho đến khi`) matches the literal's value/alternatives directly with no
+ * special handling. Keeping them out lets `tryMultiWordKeyword` emit them as one
+ * token — the profile-driven replacement for the per-language hardcoded compound
+ * lists (Task #10). `into` stays excluded because it IS consumed by the role
+ * mechanism in some languages (id destination `ke`), and `from`/`to`/`with`/
+ * `on`/`at`/`of`/`as`/`by`/`in` are genuine role markers. Command verbs /
+ * control-flow / event names were always absent. See `multiWordKeywords` /
+ * `tryMultiWordKeyword`.
+ */
+const MARKER_CONCEPT_NORMALIZEDS: ReadonlySet<string> = new Set([
+  // Role-marker role names (profile.roleMarkers normalizeds)
+  'patient',
+  'destination',
+  'source',
+  'style',
+  'event',
+  'eventMarker',
+  'agent',
+  'goal',
+  'manner',
+  // Prepositional / positional modifier concepts matched via the role mechanism
+  // (profile.keywords "Modifiers"). `before`/`after`/`until` are intentionally
+  // NOT here — they are pattern literals (see the note above).
+  'into',
+  'from',
+  'to',
+  'with',
+  'at',
+  'of',
+  'as',
+  'by',
+  'in',
+  'on',
+  'over',
+  'under',
+  'between',
+  'through',
+  'without',
+]);
+
 // =============================================================================
 // Types
 // =============================================================================
@@ -40,6 +95,39 @@ const SIMPLE_TOKENIZER_OPERATOR_SET = new Set(DEFAULT_OPERATORS);
 // for backward compatibility with code importing from this module.
 export type { KeywordEntry };
 
+/**
+ * Standard DOM event names recognized in every language as universal fallbacks.
+ * The i18n grammar transformer emits these verbatim (no native dictionary form),
+ * so each tokenizer must accept them or English-named event handlers won't parse.
+ * Kept to genuine DOM event names (not command verbs) to minimize collisions; the
+ * registration is `!has`-guarded so any native keyword of the same spelling wins.
+ */
+const ENGLISH_DOM_EVENT_NAMES: readonly string[] = [
+  'click',
+  'dblclick',
+  'input',
+  'change',
+  'submit',
+  'keydown',
+  'keyup',
+  'keypress',
+  'mousedown',
+  'mouseup',
+  'mouseover',
+  'mouseout',
+  'mouseenter',
+  'mouseleave',
+  'mousemove',
+  'pointerdown',
+  'pointerup',
+  'pointermove',
+  'focus',
+  'blur',
+  'load',
+  'resize',
+  'scroll',
+];
+
 /**
  * Profile interface for keyword derivation.
  * Matches the structure of LanguageProfile but only includes fields needed for tokenization.
@@ -81,9 +169,33 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
   /** Keywords derived from profile, sorted longest-first for greedy matching */
   protected profileKeywords: KeywordEntry[] = [];
 
+  /**
+   * Space-containing profile keywords (multi-word phrases), longest-first.
+   * Used by `tryMultiWordKeyword` so natural spaced forms (hi `मेल खाता`,
+   * vi `chuyển đổi`, es `tecla abajo`, …) tokenize as ONE keyword — the
+   * profile-driven replacement for the per-language hardcoded compound lists.
+   * Empty for no-space (CJK) languages, so they are unaffected.
+   */
+  protected multiWordKeywords: KeywordEntry[] = [];
+
   /** Map for O(1) keyword lookups by lowercase native word */
   protected profileKeywordMap: Map<string, KeywordEntry> = new Map();
 
+  /**
+   * The raw EXTRAS list passed to initializeKeywordsFromProfile, kept pre-dedup.
+   * The keyword map is keyed by native word with last-wins insertion, so a
+   * duplicate native word inside the extras silently shadows the earlier entry
+   * (e.g. a `nächste→closest` entry shadowing `nächste→next` broke German
+   * positional expressions). Exposed so consistency tests can detect such
+   * intra-extras collisions, which are invisible in the deduplicated map.
+   */
+  private rawExtraEntries: KeywordEntry[] = [];
+
+  /** Raw extras as passed in, pre-dedup — for consistency tests. */
+  getExtraKeywordEntries(): readonly KeywordEntry[] {
+    return this.rawExtraEntries;
+  }
+
   /**
    * Pluggable value extractors for domain-specific syntax.
    * When registered, BaseTokenizer will use extractor-based tokenization instead of legacy methods.
@@ -171,6 +283,19 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
       }
       if (pos >= input.length) break;
 
+      // Multi-word keyword pre-match: a profile keyword containing a space
+      // (e.g. hi `मेल खाता`, vi `chuyển đổi`, es `tecla abajo`) is matched as ONE
+      // keyword token at a word boundary, longest-first. Runs before the
+      // per-language extractors so natural spaced multi-word keywords tokenize
+      // without each tokenizer hardcoding a compound list. No-op for single-word
+      // and no-space (CJK) languages (multiWordKeywords is empty).
+      const multiWord = this.tryMultiWordKeyword(input, pos);
+      if (multiWord) {
+        tokens.push(multiWord);
+        pos = multiWord.position.end;
+        continue;
+      }
+
       // Try registered extractors in order
       let extracted = false;
       for (const extractor of this.extractors) {
@@ -297,6 +422,7 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
   ): void {
     // Use a Map to deduplicate, with extras taking precedence
     const keywordMap = new Map<string, KeywordEntry>();
+    this.rawExtraEntries = extras;
 
     // Extract from keywords (command translations)
     if (profile.keywords) {
@@ -356,6 +482,20 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
       }
     }
 
+    // Register English DOM event names as universal fallbacks. The i18n grammar
+    // transformer has no native form for most DOM events, so it passes them
+    // through verbatim (`on keyup …` → `<on-marker> keyup …`). Without these,
+    // non-English token streams treat `keyup`/`keydown`/`resize`/… as bare
+    // identifiers, and event handlers using them (often with `[key==…]` guards)
+    // fail to parse. Guarded by `!has` so any native mapping wins (same policy
+    // as the English-reference fallbacks above). Generalizes the per-language
+    // registration introduced for Hebrew in #272.
+    for (const evt of ENGLISH_DOM_EVENT_NAMES) {
+      if (!keywordMap.has(evt)) {
+        keywordMap.set(evt, { native: evt, normalized: evt });
+      }
+    }
+
     // Add extra entries (literals, positional, events) - these OVERRIDE profile entries
     for (const extra of extras) {
       keywordMap.set(extra.native, extra);
@@ -366,6 +506,20 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
       (a, b) => b.native.length - a.native.length
     );
 
+    // Multi-word (space-containing) keywords, for longest-phrase matching at a
+    // token boundary. Already longest-first (profileKeywords is sorted above).
+    // Marker/modifier concepts are EXCLUDED: those are matched positionally by
+    // the pattern matcher (role markers), and greedily consuming a multi-word
+    // marker phrase shadows the single-word marker patterns rely on — e.g. id
+    // `ke dalam` (into) would swallow the `ke` destination marker, and ko `할 때`
+    // (eventMarker) would pre-empt the SOV event extraction. Command verbs,
+    // control-flow, and event-name keywords (vi `với mỗi`=for, es `tecla abajo`=
+    // keydown, bn `তৈরি করুন`=make) are kept — the pattern matcher treats those
+    // as keyword literals, so one-token matching is strictly better.
+    this.multiWordKeywords = this.profileKeywords.filter(
+      k => k.native.includes(' ') && !MARKER_CONCEPT_NORMALIZEDS.has(k.normalized)
+    );
+
     // Build Map for O(1) lookups (case-insensitive + diacritic-insensitive)
     // This allows matching both 'بدّل' (with shadda) and 'بدل' (without) to the same entry
     this.profileKeywordMap = new Map();
@@ -417,6 +571,40 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
     return null;
   }
 
+  /**
+   * Match the longest multi-word (space-containing) profile keyword at `pos`,
+   * requiring the match to end at a word boundary. The profile-driven
+   * counterpart of the per-language hardcoded compound lists (the hindi and
+   * vietnamese keyword extractors). Returns a keyword token (with the normalized
+   * form) or null. Case-sensitive against the stored native form, mirroring
+   * `tryProfileKeyword`/`isKeywordStart` (the i18n dicts emit a fixed surface
+   * case). No-op when `multiWordKeywords` is empty (no-space/CJK languages).
+   *
+   * @param input - Input string
+   * @param pos - Current position (must be a token-start boundary)
+   * @param isWordChar - End-boundary predicate (defaults to Unicode letter/digit/_)
+   */
+  protected tryMultiWordKeyword(
+    input: string,
+    pos: number,
+    isWordChar: (char: string) => boolean = ch => /[\p{L}\p{N}_]/u.test(ch)
+  ): LanguageToken | null {
+    if (this.multiWordKeywords.length === 0) return null;
+    const rest = input.slice(pos);
+    for (const entry of this.multiWordKeywords) {
+      if (!rest.startsWith(entry.native)) continue;
+      const after = input[pos + entry.native.length];
+      if (after !== undefined && isWordChar(after)) continue; // not a word boundary
+      return createToken(
+        entry.native,
+        'keyword',
+        createPosition(pos, pos + entry.native.length),
+        entry.normalized
+      );
+    }
+    return null;
+  }
+
   /**
    * Check if the remaining input starts with any known keyword.
    * Useful for non-space languages to detect word boundaries.
@@ -430,6 +618,37 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
     return this.profileKeywords.some(entry => remaining.startsWith(entry.native));
   }
 
+  /**
+   * Check if a known keyword starts at the given position AND ends at a word
+   * boundary (end of input or a non-word character).
+   *
+   * Space-delimited languages must use this (not `isKeywordStart`) for
+   * word-walk break checks: the keyword table includes English canonical
+   * fallbacks (me, it, you, …), so a raw `startsWith` check splits any native
+   * word with an embedded fallback mid-word (e.g. Quechua ñit'iy contains
+   * "it"). CJK/no-space tokenizers rely on mid-text keyword starts and must
+   * keep using `isKeywordStart`.
+   *
+   * @param input - Input string
+   * @param pos - Current position
+   * @param isWordChar - Language-specific word-character predicate; pass the
+   *   tokenizer's letter classifier so e.g. the Quechua glottal apostrophe
+   *   counts as part of a word. Defaults to Unicode letters/digits/underscore.
+   * @returns true if a keyword starts here and is not followed by a word char
+   */
+  protected isKeywordStartAtBoundary(
+    input: string,
+    pos: number,
+    isWordChar: (char: string) => boolean = ch => /[\p{L}\p{N}_]/u.test(ch)
+  ): boolean {
+    const remaining = input.slice(pos);
+    return this.profileKeywords.some(entry => {
+      if (!remaining.startsWith(entry.native)) return false;
+      const after = input[pos + entry.native.length];
+      return after === undefined || !isWordChar(after);
+    });
+  }
+
   /**
    * Look up a keyword by native word (case-insensitive).
    * O(1) lookup using the keyword map.

package/src/core/tokenization/keyword-boundary.test.ts

@@ -0,0 +1,73 @@
+/**
+ * Contract tests for BaseTokenizer.isKeywordStartAtBoundary().
+ *
+ * The keyword table injects English canonical reference words (me, it, you, …)
+ * for every language. Space-delimited tokenizers that break their word-walk on
+ * a raw isKeywordStart() therefore split native words around embedded English
+ * fallbacks (e.g. Quechua ñit'iy contains "it"). The boundary-aware variant
+ * only fires when the keyword match ends at a word boundary.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { BaseTokenizer } from './base-tokenizer';
+import type { TokenKind } from '../types';
+
+class ProbeTokenizer extends BaseTokenizer {
+  readonly language = 'xx';
+  readonly direction = 'ltr' as const;
+
+  constructor() {
+    super();
+    this.initializeKeywordsFromProfile({
+      keywords: { toggle: { primary: 'tikray' } },
+      // references inject the English canonical fallbacks "it" and "me" too
+      references: { it: 'pay', me: 'nuqa' },
+    });
+  }
+
+  classifyToken(): TokenKind {
+    return 'identifier';
+  }
+
+  probeRaw(input: string, pos: number): boolean {
+    return this.isKeywordStart(input, pos);
+  }
+
+  probeBoundary(input: string, pos: number, isWordChar?: (ch: string) => boolean): boolean {
+    return this.isKeywordStartAtBoundary(input, pos, isWordChar);
+  }
+}
+
+describe('isKeywordStartAtBoundary', () => {
+  const t = new ProbeTokenizer();
+
+  it('does not fire on an English fallback embedded mid-word', () => {
+    // "it" inside "umitaq" — raw check fires, boundary check must not
+    expect(t.probeRaw('umitaq', 2)).toBe(true);
+    expect(t.probeBoundary('umitaq', 2)).toBe(false);
+    // "me" inside "umema"
+    expect(t.probeRaw('umema', 1)).toBe(true);
+    expect(t.probeBoundary('umema', 1)).toBe(false);
+  });
+
+  it('fires when the keyword match ends at end of input', () => {
+    expect(t.probeBoundary('umit', 2)).toBe(true);
+  });
+
+  it('fires when the keyword match is followed by a non-word char', () => {
+    expect(t.probeBoundary('um it goes', 3)).toBe(true);
+    expect(t.probeBoundary('xit.', 1)).toBe(true);
+  });
+
+  it('respects a language-specific word-char predicate', () => {
+    // Default predicate: apostrophe is not a word char → boundary fires
+    expect(t.probeBoundary("xit'y", 1)).toBe(true);
+    // Quechua-style predicate counting the glottal apostrophe → no boundary
+    const quechuaLike = (ch: string) => /[a-zñ'’]/i.test(ch);
+    expect(t.probeBoundary("xit'y", 1, quechuaLike)).toBe(false);
+  });
+
+  it('returns false when no keyword starts at the position at all', () => {
+    expect(t.probeBoundary('umitaq', 0)).toBe(false);
+  });
+});

package/src/interfaces/value-extractor.ts

@@ -406,6 +406,21 @@ export interface TokenizerContext {
    */
   isKeywordStart(input: string, position: number): boolean;
 
+  /**
+   * Like `isKeywordStart`, but only true when the keyword match ends at a
+   * word boundary (end of input or a char rejected by `isWordChar`).
+   * Space-delimited languages must use this for word-walk break checks —
+   * the keyword table includes English canonical fallbacks (me, it, you, …),
+   * so the raw check splits native words mid-word (e.g. Quechua ñit'iy
+   * contains "it"). Optional for backward compatibility with hand-rolled
+   * contexts; callers should treat absence as "no boundary keyword here".
+   */
+  isKeywordStartAtBoundary?(
+    input: string,
+    position: number,
+    isWordChar?: (char: string) => boolean
+  ): boolean;
+
   /**
    * Optional morphological normalizer for this language.
    */
@@ -449,6 +464,11 @@ export function createTokenizerContext(tokenizer: {
   lookupKeyword(native: string): KeywordEntry | undefined;
   isKeyword(native: string): boolean;
   isKeywordStart(input: string, position: number): boolean;
+  isKeywordStartAtBoundary?(
+    input: string,
+    position: number,
+    isWordChar?: (char: string) => boolean
+  ): boolean;
   normalizer?: MorphologicalNormalizer;
 }): TokenizerContext {
   const ctx: TokenizerContext = {
@@ -457,6 +477,9 @@ export function createTokenizerContext(tokenizer: {
     lookupKeyword: tokenizer.lookupKeyword.bind(tokenizer),
     isKeyword: tokenizer.isKeyword.bind(tokenizer),
     isKeywordStart: tokenizer.isKeywordStart.bind(tokenizer),
+    ...(tokenizer.isKeywordStartAtBoundary
+      ? { isKeywordStartAtBoundary: tokenizer.isKeywordStartAtBoundary.bind(tokenizer) }
+      : {}),
   };
 
   if (tokenizer.normalizer) {