@lokascript/i18n 2.3.0 → 2.4.0

package/src/grammar/transformer.ts

@@ -62,6 +62,101 @@ function getCommandKeywordsForLocale(locale: string): Set<string> {
  * Example: "wait 2s toggle .highlight"
  * Returns: ["wait 2s", "toggle .highlight"]
  */
+/**
+ * A reactive-block structure pulled apart by `extractBlockStructure`.
+ * Block-syntactic tokens (head, optional condition expression, optional
+ * connector, optional `end` tail) live outside the inner body so the
+ * body can be transformed by the regular pipeline (SOV reordering,
+ * possessives, etc.) without having block delimiters dragged into role
+ * values or moved by the canonical-order reorder.
+ */
+interface BlockStructure {
+  headKeyword: string;
+  /** Condition expression for `when X changes Y` / `unless X Y`. */
+  prefixExpr?: string;
+  /** `changes` for `when`; undefined for `live`/`unless`. */
+  connector?: string;
+  body: string;
+  /** `end` if the source had one; undefined otherwise. */
+  tailKeyword?: string;
+}
+
+/**
+ * Detect a reactive block (`live ... end`, `when X changes Y [end]`,
+ * `unless X Y [end]`) and decompose it. Returns `null` when the input
+ * is not a reactive block, when there's content after the matched
+ * `end`, or when the heuristic can't locate a body — all of which fall
+ * through to the standard `parseStatement` path.
+ */
+function extractBlockStructure(input: string, sourceLocale: string): BlockStructure | null {
+  const tokens = input.split(/\s+/);
+  const head = tokens[0]?.toLowerCase();
+  if (!head || !BLOCK_HEAD_KEYWORDS.has(head)) return null;
+
+  // Depth-aware match for the closing `end` so nested blocks
+  // (`live when X changes Y end end`) slice correctly.
+  let depth = 1;
+  let endIdx = -1;
+  for (let i = 1; i < tokens.length; i++) {
+    const t = tokens[i].toLowerCase();
+    if (BLOCK_HEAD_KEYWORDS.has(t)) depth++;
+    else if (t === 'end') {
+      depth--;
+      if (depth === 0) {
+        endIdx = i;
+        break;
+      }
+    }
+  }
+
+  // If there's trailing content after the matched `end`, bail out and
+  // let the existing splitter handle it. (`splitOnThen` normally
+  // separates trailing code before we get here.)
+  if (endIdx !== -1 && endIdx !== tokens.length - 1) return null;
+
+  const inner = endIdx !== -1 ? tokens.slice(1, endIdx) : tokens.slice(1);
+  const base: BlockStructure = { headKeyword: tokens[0], body: '' };
+  if (endIdx !== -1) base.tailKeyword = tokens[endIdx];
+
+  if (head === 'live') {
+    return { ...base, body: inner.join(' ') };
+  }
+
+  if (head === 'when') {
+    // Reactive: `when <expr> changes <body>`. Without `changes`, fall
+    // through to the standard event-wait path (parseConditional).
+    const idx = inner.findIndex(t => t.toLowerCase() === 'changes');
+    if (idx >= 0) {
+      return {
+        ...base,
+        prefixExpr: inner.slice(0, idx).join(' '),
+        connector: inner[idx],
+        body: inner.slice(idx + 1).join(' '),
+      };
+    }
+    return null;
+  }
+
+  // `unless <cond> <body>`: condition runs up to the first command
+  // keyword in `inner`. Heuristic — works because hyperscript bodies
+  // always start with a command verb, and `unless` conditions rarely
+  // contain bare command keywords as values.
+  const commands = getCommandKeywordsForLocale(sourceLocale);
+  let bodyStart = -1;
+  for (let i = 0; i < inner.length; i++) {
+    if (commands.has(inner[i].toLowerCase())) {
+      bodyStart = i;
+      break;
+    }
+  }
+  if (bodyStart <= 0) return null;
+  return {
+    ...base,
+    prefixExpr: inner.slice(0, bodyStart).join(' '),
+    body: inner.slice(bodyStart).join(' '),
+  };
+}
+
 function splitCompoundStatement(input: string, sourceLocale: string): string[] {
   // First, split on newlines (preserving non-empty lines)
   const lines = input
@@ -265,6 +360,19 @@ const BOUNDARY_MODIFIERS = new Set([
   'over',
 ]);
 
+/**
+ * Block-introducing keywords whose body should not be split at command
+ * boundaries by `splitOnCommandBoundaries`. Inputs starting with one of
+ * these are also routed around `parseStatement` entirely by
+ * `extractBlockStructure` + `transformBlock` so block-syntactic tokens
+ * never reach `parseCommand`/`parseConditional` (where they'd be
+ * misinterpreted as command verbs or swept into role values).
+ *
+ * `if` deliberately stays out: `if X then Y end` already works via the
+ * `splitOnThen` + `parseConditional` path.
+ */
+const BLOCK_HEAD_KEYWORDS = new Set(['live', 'when', 'unless']);
+
 function splitOnCommandBoundaries(input: string, sourceLocale: string): string[] {
   const commandKeywords = getCommandKeywordsForLocale(sourceLocale);
   const tokens = input.split(/\s+/);
@@ -282,10 +390,23 @@ function splitOnCommandBoundaries(input: string, sourceLocale: string): string[]
   // So we need to track whether we've seen the first command yet
   let seenFirstCommand = !isEventHandler; // If not event handler, we're already past the "first command" phase
 
+  // Track block-scope depth (live/when/bind/if/unless/for/while/...). While
+  // inside a block, do not split on command boundaries — the body belongs
+  // to the block head and must transform as one unit. See comments on
+  // BLOCK_HEAD_KEYWORDS for the failure mode this prevents.
+  let blockDepth = 0;
+
   for (let i = 0; i < tokens.length; i++) {
     const token = tokens[i];
     const lowerToken = token.toLowerCase();
 
+    // Update block-scope depth before any split decision.
+    if (BLOCK_HEAD_KEYWORDS.has(lowerToken)) {
+      blockDepth++;
+    } else if (lowerToken === 'end' && blockDepth > 0) {
+      blockDepth--;
+    }
+
     // If this is a command keyword and we already have tokens in current part
     if (commandKeywords.has(lowerToken) && currentPart.length > 0) {
       // Check if the previous token looks like it could end a command
@@ -301,6 +422,13 @@ function splitOnCommandBoundaries(input: string, sourceLocale: string): string[]
         continue;
       }
 
+      // Don't split inside a block (live/when/bind/unless body, etc.).
+      // The block head and its body must transform as one statement.
+      if (blockDepth > 0) {
+        currentPart.push(token);
+        continue;
+      }
+
       if (!BOUNDARY_MODIFIERS.has(prevLower) && !commandKeywords.has(prevLower)) {
         // This looks like a command boundary - save current part and start new one
         parts.push(currentPart.join(' '));
@@ -1165,6 +1293,15 @@ export class GrammarTransformer {
    * Transform a single hyperscript statement (no compound "then" chains).
    */
   private transformSingle(input: string): string {
+    // 0. Reactive block? Route around parseStatement entirely so
+    //    block-syntactic tokens (live/when/unless/end) aren't treated
+    //    as command verbs or swept into role values, and so SOV/VSO
+    //    reorder applies only inside the body.
+    const block = extractBlockStructure(input, this.sourceProfile.code);
+    if (block) {
+      return this.transformBlock(block);
+    }
+
     // 1. Parse into semantic roles
     const parsed = parseStatement(input, this.sourceProfile.code);
     if (!parsed) {
@@ -1202,6 +1339,30 @@ export class GrammarTransformer {
     return joinTokens(reordered.map(e => e.translated || e.value));
   }
 
+  /**
+   * Translate a reactive block by translating the head/tail/connector
+   * via the dictionary, recursively transforming the body through the
+   * regular pipeline, and rejoining in source-language position order.
+   * Block-syntactic tokens are never reordered: they're delimiters, not
+   * arguments, and authors expect them at start/end positions
+   * regardless of target word order.
+   */
+  private transformBlock(block: BlockStructure): string {
+    const src = this.sourceProfile.code;
+    const dst = this.targetProfile.code;
+
+    const head = translateWord(block.headKeyword, src, dst);
+    const tail = block.tailKeyword ? translateWord(block.tailKeyword, src, dst) : '';
+    const connector = block.connector ? translateWord(block.connector, src, dst) : '';
+    const prefix = block.prefixExpr ? translateMultiWordValue(block.prefixExpr, src, dst) : '';
+
+    // Recurse through `transform()` (not `transformSingle`) so the body
+    // gets `then`-splitting and nested-block handling for free.
+    const body = this.transform(block.body);
+
+    return [head, prefix, connector, body, tail].filter(s => s.length > 0).join(' ');
+  }
+
   /**
    * Find the best matching rule for this statement
    */

package/src/parser/he.ts

@@ -0,0 +1,29 @@
+// packages/i18n/src/parser/he.ts
+
+import { he } from '../dictionaries/he';
+import { createKeywordProvider } from './create-provider';
+import type { KeywordProvider } from './types';
+
+/**
+ * Hebrew keyword provider for the hyperscript parser.
+ *
+ * Enables parsing hyperscript written in Hebrew (RTL, SVO):
+ * - `ב לחיצה מתג .active` → parses as `on click toggle .active`
+ *
+ * English keywords are also accepted (mixed mode).
+ *
+ * @example
+ * ```typescript
+ * import { heKeywords } from '@lokascript/i18n/parser/he';
+ * import { Parser } from '@hyperfixi/core';
+ *
+ * const parser = new Parser({ keywords: heKeywords });
+ * parser.parse('ב לחיצה מתג .active');
+ * ```
+ */
+export const heKeywords: KeywordProvider = createKeywordProvider(he, 'he', {
+  allowEnglishFallback: true,
+});
+
+// Re-export for convenience
+export { he as heDictionary } from '../dictionaries/he';

package/src/parser/index.ts

@@ -55,6 +55,7 @@ export { bnKeywords, bnDictionary } from './bn';
 export { thKeywords, thDictionary } from './th';
 export { msKeywords, msDictionary } from './ms';
 export { tlKeywords, tlDictionary } from './tl';
+export { heKeywords, heDictionary } from './he';
 
 // Locale management
 export { LocaleManager, detectBrowserLocale } from './locale-manager';

package/src/schema-alignment.test.ts

@@ -0,0 +1,154 @@
+/**
+ * Schema ↔ i18n Alignment Tests
+ *
+ * Verifies that every CommandSchema exported by @lokascript/semantic has a
+ * matching English dictionary entry in en.ts's `commands` category, and that
+ * every COMPLETE language dictionary covers the same set of command keys as
+ * English.
+ *
+ * Catches drift going forward: adding a new schema without updating en.ts,
+ * or adding commands to en.ts without propagating via the semantic profile
+ * generator.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { commandSchemas } from '@lokascript/semantic';
+import { dictionaries } from './dictionaries';
+import type { Dictionary } from './types';
+import { DICTIONARY_CATEGORIES } from './types';
+import { COMPLETE_LANGUAGES } from './test-utils/complete-languages';
+
+/**
+ * Dictionaries store each English keyword once in whichever category it
+ * primarily belongs to — `focus`/`blur` end up in `events`, `empty` in
+ * `expressions`. This helper looks up a keyword across all categories.
+ */
+function findInDictionary(dict: Dictionary, keyword: string): string | undefined {
+  for (const cat of DICTIONARY_CATEGORIES) {
+    const v = dict[cat]?.[keyword];
+    if (typeof v === 'string' && v.length > 0) return v;
+  }
+  return undefined;
+}
+
+// Actions that don't need a user-facing keyword translation at command
+// position. They're either AST meta-nodes (`compound`) or definition-time
+// feature keywords (`init`, `behavior`) handled elsewhere in the parser.
+const NON_USER_FACING_ACTIONS = new Set(['compound', 'init', 'behavior']);
+
+// Event handler 'on' is a top-level feature, not a command the user writes at
+// command position in the same way. It's handled specially by the parser.
+// Still, it has an i18n entry under `commands` in en.ts.
+
+describe('Schema ↔ i18n alignment', () => {
+  const schemaActions = Object.keys(commandSchemas).filter(
+    action => !NON_USER_FACING_ACTIONS.has(action)
+  );
+
+  describe('every command schema has an English dictionary entry', () => {
+    const en = dictionaries.en as Dictionary;
+    schemaActions.forEach(action => {
+      it(`en dictionary contains '${action}'`, () => {
+        const value = findInDictionary(en, action);
+        expect(
+          value,
+          `CommandSchema '${action}' is exported from @lokascript/semantic but has no en.ts dictionary entry in any category. Add it to packages/i18n/src/dictionaries/en.ts and run \`npm run generate:language-assets\`.`
+        ).toBeDefined();
+      });
+    });
+  });
+
+  describe('Phase 1 commands (v0.9.90) are present in all complete languages', () => {
+    const PHASE_1_COMMANDS = [
+      'focus',
+      'blur',
+      'empty',
+      'open',
+      'close',
+      'select',
+      'clear',
+      'reset',
+      'breakpoint',
+    ] as const;
+
+    COMPLETE_LANGUAGES.forEach(code => {
+      const dict = dictionaries[code] as Dictionary | undefined;
+      if (!dict) return;
+
+      it(`${code}: has all 9 Phase 1 commands (any category)`, () => {
+        const missing: string[] = [];
+        for (const cmd of PHASE_1_COMMANDS) {
+          if (!findInDictionary(dict, cmd)) {
+            missing.push(cmd);
+          }
+        }
+        expect(
+          missing,
+          `Missing Phase 1 commands in ${code}: ${missing.join(', ')}. ` +
+            `Fix by adding to packages/semantic/src/generators/profiles/${code}.ts ` +
+            `and running \`npm run generate:language-assets\`.`
+        ).toEqual([]);
+      });
+    });
+  });
+
+  describe('Phase 2 comparators (v0.9.90) are present in all complete languages', () => {
+    // Expression operators wired into the core Pratt parser. Unlike Phase 1
+    // commands, these live in the hand-written `expressions` category (or
+    // `modifiers` for `between`) rather than deriving from semantic profiles.
+    const PHASE_2_COMPARATORS = ['starts with', 'ends with', 'between', 'ignoring case'] as const;
+
+    COMPLETE_LANGUAGES.forEach(code => {
+      const dict = dictionaries[code] as Dictionary | undefined;
+      if (!dict) return;
+
+      it(`${code}: has all Phase 2 comparators (any category)`, () => {
+        const missing: string[] = [];
+        for (const op of PHASE_2_COMPARATORS) {
+          if (!findInDictionary(dict, op)) {
+            missing.push(op);
+          }
+        }
+        expect(
+          missing,
+          `Missing Phase 2 comparators in ${code}: ${missing.join(', ')}. ` +
+            `Add to packages/i18n/src/dictionaries/${code}.ts's \`expressions\` ` +
+            `category (or \`modifiers\` for \`between\`).`
+        ).toEqual([]);
+      });
+    });
+  });
+
+  describe('Phase 3 collection ops (v0.9.90) are present in all complete languages', () => {
+    // Infix operators on arrays and strings — parsed as custom Pratt AST nodes
+    // by the core parser. `where` predates v0.9.90 and lives in `logical`; the
+    // four `<verb> by/to` forms live alongside `starts with` in `expressions`.
+    const PHASE_3_COLLECTIONS = [
+      'where',
+      'sorted by',
+      'mapped to',
+      'split by',
+      'joined by',
+    ] as const;
+
+    COMPLETE_LANGUAGES.forEach(code => {
+      const dict = dictionaries[code] as Dictionary | undefined;
+      if (!dict) return;
+
+      it(`${code}: has all Phase 3 collection ops (any category)`, () => {
+        const missing: string[] = [];
+        for (const op of PHASE_3_COLLECTIONS) {
+          if (!findInDictionary(dict, op)) {
+            missing.push(op);
+          }
+        }
+        expect(
+          missing,
+          `Missing Phase 3 collection ops in ${code}: ${missing.join(', ')}. ` +
+            `Add to packages/i18n/src/dictionaries/${code}.ts's \`expressions\` ` +
+            `category (or \`logical\` for \`where\`).`
+        ).toEqual([]);
+      });
+    });
+  });
+});

package/src/test-utils/complete-languages.ts

@@ -0,0 +1,40 @@
+/**
+ * Languages with complete (non-placeholder) dictionaries.
+ *
+ * Shared by `translation-validation.test.ts` and `schema-alignment.test.ts`
+ * so a new dictionary only needs to be added in one place to enter the test
+ * matrix.
+ *
+ * This is intentionally distinct from `@lokascript/semantic`'s
+ * `SUPPORTED_LANGUAGES`: a language can be loadable by the semantic parser
+ * (e.g. `tr` / Turkish) yet not be "complete" by i18n's definition if its
+ * dictionary is missing newer Phase command/comparator entries. When a
+ * dictionary catches up to en.ts, add its code here to enable the full
+ * test sweep.
+ */
+export const COMPLETE_LANGUAGES = [
+  'en',
+  'es',
+  'ja',
+  'ko',
+  'ar',
+  'id',
+  'pt',
+  'it',
+  'vi',
+  'qu',
+  'sw',
+  'pl',
+  'ru',
+  'zh',
+  'hi',
+  'bn',
+  'de',
+  'th',
+  'fr',
+  'uk',
+  'tl',
+  'ms',
+] as const;
+
+export type CompleteLanguage = (typeof COMPLETE_LANGUAGES)[number];

package/src/translation-validation.test.ts

@@ -8,36 +8,11 @@
 import { describe, it, expect } from 'vitest';
 import { dictionaries } from './dictionaries';
 import type { Dictionary } from './types';
+import { COMPLETE_LANGUAGES } from './test-utils/complete-languages';
 
 // Focus on when/where which were the main TODO items we fixed
 const REQUIRED_LOGICAL_KEYWORDS = ['when', 'where', 'and', 'or', 'not'] as const;
 
-// Languages that are fully implemented (not placeholder stubs)
-const COMPLETE_LANGUAGES = [
-  'en',
-  'es',
-  'ja',
-  'ko',
-  'ar',
-  'id',
-  'pt',
-  'it',
-  'vi',
-  'qu',
-  'sw',
-  'pl',
-  'ru',
-  'zh',
-  'hi',
-  'bn',
-  'de',
-  'th',
-  'fr',
-  'uk',
-  'tl',
-  'ms',
-] as const;
-
 // Languages with placeholder translations (none currently)
 const INCOMPLETE_LANGUAGES = [] as const;