@lokascript/framework 4.4.0 → 4.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lokascript/framework",
-  "version": "4.4.0",
+  "version": "4.6.0",
   "description": "Generic framework for building multilingual DSLs with semantic parsing and grammar transformation",
   "type": "module",
   "main": "dist/index.cjs",
@@ -96,10 +96,10 @@
   "author": "LokaScript Contributors",
   "license": "MIT",
   "dependencies": {
-    "@lokascript/intent": "^4.4.0"
+    "@lokascript/intent": "^4.6.0"
   },
   "devDependencies": {
-    "@lokascript/domains": "^3.0.1",
+    "@lokascript/domains": "^4.0.0",
     "@types/node": "^26.6.3",
     "@vitest/coverage-v8": "^5.0.0",
     "tsup": "^8.0.0",

package/src/__test__/fixtures/sql-dsl.ts

@@ -47,6 +47,9 @@ export const selectCommand = defineCommand({
       description: 'WHERE clause condition',
       required: false,
       expectedTypes: ['expression'],
+      // As in domain-sql: the condition is the rest of the input (`age > 18`). Without it the
+      // condition read `age` and dropped `> 18`, which parse accepted until it refused unread input.
+      greedy: true,
       svoPosition: 0,
       sovPosition: 0,
       markerOverride: { en: 'where', es: 'donde' },

package/src/__test__/sql-integration.test.ts

@@ -46,8 +46,8 @@ describe('SQL DSL Integration', () => {
       const condition = result.roles.get('condition');
       expect(condition).toBeDefined();
       if (condition && 'raw' in condition) {
-        // The condition role captures the expression after WHERE
-        expect(condition.raw).toBeTruthy();
+        // The condition role captures the whole expression after WHERE
+        expect(condition.raw).toBe('age > 18');
       }
     });
   });

package/src/api/create-dsl.test.ts

@@ -13,6 +13,8 @@ import {
   defineRole,
   BaseTokenizer,
   getDefaultExtractors,
+  UnreadInputError,
+  TranslationReadBackError,
   type SemanticNode,
 } from '../index';
 
@@ -156,6 +158,35 @@ function createBilingualTestDSL() {
   });
 }
 
+/** The bilingual DSL with a custom renderer (and, optionally, a code generator). */
+function createBilingualTestDSLWith(
+  renderer: (node: SemanticNode, language: string) => string | null,
+  codeGenerator?: { generate(node: SemanticNode): string }
+) {
+  return createMultilingualDSL({
+    name: 'TestSQL-rendered',
+    schemas: [selectSchema, insertSchema],
+    languages: [
+      {
+        code: 'en',
+        name: 'English',
+        nativeName: 'English',
+        tokenizer: new EnglishTestTokenizer(),
+        patternProfile: englishProfile,
+      },
+      {
+        code: 'es',
+        name: 'Spanish',
+        nativeName: 'Español',
+        tokenizer: new SpanishTestTokenizer(),
+        patternProfile: spanishProfile,
+      },
+    ],
+    renderer,
+    ...(codeGenerator ? { codeGenerator } : {}),
+  });
+}
+
 function createTestDSL() {
   return createMultilingualDSL({
     name: 'TestSQL',
@@ -396,4 +427,93 @@ describe('MultilingualDSL: Explicit Syntax Support', () => {
       expect(result.code).toContain('SELECT');
     });
   });
+
+  // ---------------------------------------------------------------------------
+  // Input left unread (DOMAINS_RETHINK.md, Phase 3)
+  // ---------------------------------------------------------------------------
+
+  describe('input left unread', () => {
+    it('refuses a parse that reads the command only by dropping what follows it', () => {
+      const dsl = createTestDSL();
+      let error: unknown;
+      try {
+        dsl.parse('select name from users banana', 'en');
+      } catch (e) {
+        error = e;
+      }
+      expect(error).toBeInstanceOf(UnreadInputError);
+      const refusal = error as UnreadInputError;
+      expect(refusal.code).toBe('unread-input');
+      expect(refusal.unread).toBe('banana');
+      expect(refusal.reading.action).toBe('select');
+      expect(refusal.message).toMatch(/leaving "banana" unread/);
+    });
+
+    it('compile and validate report the refusal instead of compiling the shorter reading', () => {
+      const dsl = createTestDSL();
+      const compiled = dsl.compile('select name from users banana banana', 'en');
+      expect(compiled.ok).toBe(false);
+      expect(compiled.errors?.[0]).toMatch(/leaving "banana banana" unread/);
+      expect(dsl.validate('select name from users banana', 'en').valid).toBe(false);
+    });
+
+    it('translate refuses too, rather than translating the shorter reading', () => {
+      const dsl = createBilingualTestDSL();
+      expect(() => dsl.translate('select name from users banana', 'en', 'es')).toThrow(
+        UnreadInputError
+      );
+    });
+
+    it('a translation that does not read back is refused, unless { lossy: "allow" }', () => {
+      // A renderer that drops the required source: the Spanish does not parse at all.
+      const dropsSource = createBilingualTestDSLWith(node =>
+        node.action === 'select' ? 'seleccionar nombre' : null
+      );
+      let error: unknown;
+      try {
+        dropsSource.translate('select nombre from users', 'en', 'es');
+      } catch (e) {
+        error = e;
+      }
+      expect(error).toBeInstanceOf(TranslationReadBackError);
+      const refusal = error as TranslationReadBackError;
+      expect(refusal.code).toBe('translation-does-not-read-back');
+      expect(refusal.partial).toBe('seleccionar nombre');
+      expect(refusal.reading).toBeNull();
+      expect(
+        dropsSource.translate('select nombre from users', 'en', 'es', { lossy: 'allow' })
+      ).toBe('seleccionar nombre');
+    });
+
+    it('a translation that reads with a value changed is refused, naming the role', () => {
+      const changesSource = createBilingualTestDSLWith(node =>
+        node.action === 'select' ? 'seleccionar nombre de clientes' : null
+      );
+      expect(() => changesSource.translate('select nombre from users', 'en', 'es')).toThrow(
+        /source: users → clientes/
+      );
+    });
+
+    it('a difference the code generator normalizes still reads back', () => {
+      // Upper case in the render, lower case in the generator: the nodes differ, the code does not.
+      const shouts = createBilingualTestDSLWith(
+        node => (node.action === 'select' ? 'seleccionar NOMBRE de USERS' : null),
+        {
+          generate: node =>
+            `${node.action} ${[...node.roles.values()]
+              .map(v => String((v as { raw?: unknown }).raw ?? '').toLowerCase())
+              .join(' ')}`,
+        }
+      );
+      expect(shouts.translate('select nombre from users', 'en', 'es')).toBe(
+        'seleccionar NOMBRE de USERS'
+      );
+    });
+
+    it('input read whole still parses, and nothing matching at all is still "No pattern matched"', () => {
+      const dsl = createTestDSL();
+      expect(dsl.parse('select name from users', 'en').action).toBe('select');
+      expect(() => dsl.parse('banana', 'en')).toThrow(/No pattern matched/);
+    });
+  });
 });

package/src/api/create-dsl.ts

@@ -2,7 +2,12 @@
  * Main API for creating multilingual DSLs
  */
 
-import type { SemanticNode, LanguageTokenizer, LanguagePattern } from '../core/types';
+import type {
+  SemanticNode,
+  LanguageTokenizer,
+  LanguagePattern,
+  PatternMatchResult,
+} from '../core/types';
 import type { CommandSchema } from '../schema';
 import { PatternMatcher, type PatternMatcherProfile } from '../core/pattern-matching';
 import { generatePattern, type PatternGenLanguageProfile } from '../generation/pattern-generator';
@@ -16,6 +21,7 @@ import {
   InMemoryProfileProvider,
 } from '../interfaces';
 import { isExplicitSyntax, parseExplicit } from '../ir';
+import { readingDifference } from './reading';
 import type { SchemaLookup } from '../ir';
 
 /**
@@ -177,6 +183,8 @@ export interface ValidationResult {
   readonly valid: boolean;
   readonly node?: SemanticNode;
   readonly errors?: string[];
+  /** Set when the input was refused for a known reason: `'unread-input'` (`UnreadInputError`). */
+  readonly errorCode?: string;
 }
 
 /**
@@ -186,6 +194,8 @@ export interface CompileResult {
   readonly ok: boolean;
   readonly code?: string;
   readonly errors?: string[];
+  /** Set when the input was refused for a known reason: `'unread-input'` (`UnreadInputError`). */
+  readonly errorCode?: string;
   readonly node?: SemanticNode;
   readonly metadata?: {
     readonly parser: string;
@@ -193,6 +203,55 @@ export interface CompileResult {
   };
 }
 
+/**
+ * Thrown by `parse` (and so `compile`, `validate`, `translate`) when the input reads as a command
+ * only by leaving some of it unread: `select name from users banana` is not `select name from users`.
+ * `reading` is what the longest such match read, `unread` the text after it.
+ */
+export class UnreadInputError extends Error {
+  readonly code = 'unread-input';
+
+  constructor(
+    readonly input: string,
+    readonly language: string,
+    readonly reading: SemanticNode,
+    readonly unread: string
+  ) {
+    super(`"${input}" (${language}) reads as ${reading.action} only by leaving "${unread}" unread`);
+    this.name = 'UnreadInputError';
+  }
+}
+
+/** Options for `translate`. */
+export interface TranslateOptions {
+  /**
+   * `'allow'` returns a translation that does not read back as the source instead of throwing
+   * `TranslationReadBackError`, as semantic's `translate` does with the same option.
+   */
+  readonly lossy?: 'allow' | 'refuse';
+}
+
+/**
+ * Thrown by `translate` when its output, read in the target language, is not the source command:
+ * it does not read at all, or reads as another command, or with a role lost or changed. `partial` is
+ * what `translate` would have returned; `reading` what it reads as (null when it does not read).
+ */
+export class TranslationReadBackError extends Error {
+  readonly code = 'translation-does-not-read-back';
+
+  constructor(
+    readonly partial: string,
+    readonly from: string,
+    readonly to: string,
+    readonly source: SemanticNode,
+    readonly reading: SemanticNode | null,
+    readonly why: string
+  ) {
+    super(`translation ${from} → ${to} does not read back as the source (${why}): "${partial}"`);
+    this.name = 'TranslationReadBackError';
+  }
+}
+
 /**
  * Main multilingual DSL interface.
  */
@@ -208,7 +267,17 @@ export interface MultilingualDSL {
   compile(input: string, language: string): CompileResult;
 
   // Translation
-  translate(input: string, fromLanguage: string, toLanguage: string): string;
+  /**
+   * Parse in one language, render in another. The output must read back as the source command
+   * (the same action and role values, or the same compiled code); when it does not,
+   * `TranslationReadBackError`, unless `{ lossy: 'allow' }`.
+   */
+  translate(
+    input: string,
+    fromLanguage: string,
+    toLanguage: string,
+    options?: TranslateOptions
+  ): string;
 
   /**
    * Render a parsed node back to natural language.
@@ -366,30 +435,66 @@ class MultilingualDSLImpl implements MultilingualDSL {
       code: language,
     };
 
-    // Try to match each pattern
+    // Every pattern is tried, and only a match that reads the WHOLE input counts; of those, the
+    // highest priority wins, then the highest confidence, then the earliest pattern. The first match
+    // used to win however much it read, so a trailing word was dropped without a word (the domain
+    // gate measured 1,744 of 1,744 such inputs accepted) and an earlier, shorter pattern could
+    // shadow the one that reads everything.
+    const total = tokens.tokens.length;
+    let best: PatternMatchResult | null = null;
+    let longestPartial: PatternMatchResult | null = null;
     for (const pattern of patterns) {
+      const mark = tokens.mark();
       const match = this.matcher.matchPattern(tokens, pattern, profile);
-      if (match) {
-        // Build semantic node
-        const node: SemanticNode = {
-          kind: 'command',
-          action: pattern.command,
-          roles: match.captured,
-          metadata: {
-            sourceLanguage: language,
-            sourceText: input,
-            patternId: pattern.id,
-            confidence: match.confidence,
-          },
-        };
-
-        return { node, confidence: match.confidence };
+      tokens.reset(mark);
+      if (!match) continue;
+      if (match.consumedTokens < total) {
+        if (!longestPartial || match.consumedTokens > longestPartial.consumedTokens)
+          longestPartial = match;
+        continue;
       }
+      if (
+        !best ||
+        match.pattern.priority > best.pattern.priority ||
+        (match.pattern.priority === best.pattern.priority &&
+          match.confidence > best.confidence + 0.001)
+      )
+        best = match;
     }
 
+    if (best) {
+      const node = this.nodeOf(best, input, language);
+      return { node, confidence: best.confidence };
+    }
+    if (longestPartial) {
+      const unread = tokens.tokens
+        .slice(longestPartial.consumedTokens)
+        .map(t => t.value)
+        .join(' ');
+      throw new UnreadInputError(
+        input,
+        language,
+        this.nodeOf(longestPartial, input, language),
+        unread
+      );
+    }
     throw new Error(`No pattern matched for input: ${input}`);
   }
 
+  private nodeOf(match: PatternMatchResult, input: string, language: string): SemanticNode {
+    return {
+      kind: 'command',
+      action: match.pattern.command,
+      roles: match.captured,
+      metadata: {
+        sourceLanguage: language,
+        sourceText: input,
+        patternId: match.pattern.id,
+        confidence: match.confidence,
+      },
+    };
+  }
+
   validate(input: string, language: string): ValidationResult {
     try {
       const result = this.parseWithConfidence(input, language);
@@ -401,11 +506,17 @@ class MultilingualDSLImpl implements MultilingualDSL {
       return {
         valid: false,
         errors: [error instanceof Error ? error.message : String(error)],
+        ...(error instanceof UnreadInputError ? { errorCode: error.code } : {}),
       };
     }
   }
 
-  translate(input: string, fromLanguage: string, toLanguage: string): string {
+  translate(
+    input: string,
+    fromLanguage: string,
+    toLanguage: string,
+    options?: TranslateOptions
+  ): string {
     // Explicit syntax is language-agnostic — return unchanged
     if (isExplicitSyntax(input)) return input;
 
@@ -432,12 +543,18 @@ class MultilingualDSLImpl implements MultilingualDSL {
     const rendered = this.registry.getSupportedLanguages().includes(toLanguage)
       ? this.render(node, toLanguage)
       : null;
-    if (rendered != null) return rendered;
+    if (rendered != null) return this.readsBack(rendered, node, fromLanguage, toLanguage, options);
 
     // render() covered nothing for this action/language pair; fall back to the
     // grammar transformer when profiles exist, otherwise name both remedies.
     if (this.hasGrammarProfile(fromLanguage) && this.hasGrammarProfile(toLanguage)) {
-      return this.transformer.transform(input, fromLanguage, toLanguage);
+      return this.readsBack(
+        this.transformer.transform(input, fromLanguage, toLanguage),
+        node,
+        fromLanguage,
+        toLanguage,
+        options
+      );
     }
     throw new Error(
       `translate() could not render action "${node.action}" in language "${toLanguage}": no ` +
@@ -448,6 +565,40 @@ class MultilingualDSLImpl implements MultilingualDSL {
     );
   }
 
+  /**
+   * `rendered`, read in `to`, must be `source`: the same action and role values, or, when the DSL
+   * compiles, the same code (a parser may keep a translated word its generator normalizes: bdd ja
+   * `表示` is `visible`). The rule is the domain gate's (lokascript-domains). A translation used to be
+   * returned whatever it read as: ja/ko bdd `select` dropped its value, and nothing said so.
+   */
+  private readsBack(
+    rendered: string,
+    source: SemanticNode,
+    from: string,
+    to: string,
+    options?: TranslateOptions
+  ): string {
+    if (options?.lossy === 'allow') return rendered;
+    let back: SemanticNode;
+    try {
+      back = this.parseWithConfidence(rendered, to).node;
+    } catch (error) {
+      const why = error instanceof Error ? error.message : String(error);
+      throw new TranslationReadBackError(rendered, from, to, source, null, why);
+    }
+    const difference = readingDifference(source, back);
+    if (!difference) return rendered;
+    if (this.codeGenerator) {
+      try {
+        if (this.codeGenerator.generate(back) === this.codeGenerator.generate(source))
+          return rendered;
+      } catch {
+        // a generator that cannot compile one of them does not vouch for either
+      }
+    }
+    throw new TranslationReadBackError(rendered, from, to, source, back, difference);
+  }
+
   private hasGrammarProfile(language: string): boolean {
     return this.profileProvider.getProfile(language) != null;
   }
@@ -477,6 +628,7 @@ class MultilingualDSLImpl implements MultilingualDSL {
       return {
         ok: false,
         errors: [error instanceof Error ? error.message : String(error)],
+        ...(error instanceof UnreadInputError ? { errorCode: error.code } : {}),
       };
     }
   }

package/src/api/domain-registry.test.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect, beforeEach } from 'vitest';
 import { DomainRegistry } from './domain-registry';
 import type { DomainDescriptor, MCPToolDefinition } from './domain-registry';
-import type { MultilingualDSL } from './create-dsl';
+import { UnreadInputError, type MultilingualDSL } from './create-dsl';
 import type { SemanticNode } from '../core/types';
 
 // =============================================================================
@@ -11,10 +11,19 @@ import type { SemanticNode } from '../core/types';
 function createMockDSL(): MultilingualDSL {
   return {
     parse(input: string, language: string): SemanticNode {
+      // A rendering is `[<lang>] <source>`: read the source back out of it.
+      const source = input.replace(/^\[\w+\] /, '');
+      if (source.includes(' zzqx'))
+        throw new UnreadInputError(
+          input,
+          language,
+          this.parse(source.replace(' zzqx', ''), language),
+          'zzqx'
+        );
       return {
         kind: 'command',
-        action: input.split(/\s+/)[0].toLowerCase(),
-        roles: new Map([['raw', { type: 'expression' as const, raw: input }]]),
+        action: source.split(/\s+/)[0].toLowerCase(),
+        roles: new Map([['raw', { type: 'expression' as const, raw: source }]]),
         metadata: { sourceLanguage: language },
       };
     },
@@ -33,7 +42,9 @@ function createMockDSL(): MultilingualDSL {
       try {
         const node = this.parse(input, language);
         return { ok: true, code: `compiled(${input})`, node };
-      } catch {
+      } catch (error) {
+        if (error instanceof UnreadInputError)
+          return { ok: false, errors: [error.message], errorCode: error.code };
         return { ok: false, errors: ['Compile failed'] };
       }
     },
@@ -46,7 +57,14 @@ function createMockDSL(): MultilingualDSL {
   };
 }
 
+/** Writes the whole source, so a rendering reads back. */
 function mockRenderer(node: SemanticNode, language: string): string {
+  const raw = (node.roles.get('raw') as { raw?: string } | undefined)?.raw ?? node.action;
+  return `[${language}] ${raw}`;
+}
+
+/** Writes the action alone: `select name` comes out as `select`, which does not read back. */
+function lossyRenderer(node: SemanticNode, language: string): string {
   return `[${language}] ${node.action}`;
 }
 
@@ -236,6 +254,19 @@ describe('DomainRegistry', () => {
         expect(data.ok).toBe(true);
         expect(data.code).toBe('compiled(select name)');
       });
+
+      it('reports input left unread as an UNREAD_INPUT diagnostic', async () => {
+        registry.register(createTestDescriptor());
+        const result = await registry.handleToolCall('compile_test', {
+          query: 'select name zzqx',
+          language: 'en',
+        });
+
+        const data = JSON.parse(result!.content[0].text);
+        expect(data.ok).toBe(false);
+        expect(data.diagnostics[0].code).toBe('UNREAD_INPUT');
+        expect(data.diagnostics[0].message).toMatch(/zzqx/);
+      });
     });
 
     describe('validate', () => {
@@ -263,10 +294,44 @@ describe('DomainRegistry', () => {
         const data = JSON.parse(result!.content[0].text);
         expect(data.input.query).toBe('select name');
         expect(data.input.language).toBe('en');
-        expect(data.rendered.text).toBe('[ja] select');
+        expect(data.ok).toBe(true);
+        expect(data.rendered.text).toBe('[ja] select name');
         expect(data.rendered.language).toBe('ja');
       });
 
+      it('refuses a rendering that does not read back, with its text under partial', async () => {
+        registry.register(createTestDescriptor({ getRenderer: () => lossyRenderer }));
+        const result = await registry.handleToolCall('translate_test', {
+          query: 'select name',
+          from: 'en',
+          to: 'ja',
+        });
+
+        expect(result!.isError).toBe(true);
+        const data = JSON.parse(result!.content[0].text);
+        expect(data.ok).toBe(false);
+        expect(data.partial).toBe('[ja] select');
+        expect(data.rendered).toBeUndefined();
+        expect(data.diagnostics[0].code).toBe('TRANSLATION_DOES_NOT_READ_BACK');
+        expect(data.diagnostics[0].message).toMatch(/raw: select name → select/);
+      });
+
+      it('refuses input left unread, naming it', async () => {
+        registry.register(createTestDescriptor());
+        const result = await registry.handleToolCall('translate_test', {
+          query: 'select name zzqx',
+          from: 'en',
+          to: 'ja',
+        });
+
+        expect(result!.isError).toBe(true);
+        const data = JSON.parse(result!.content[0].text);
+        expect(data.ok).toBe(false);
+        expect(data.unread).toBe('zzqx');
+        expect(data.reading.action).toBe('select');
+        expect(data.diagnostics[0].code).toBe('UNREAD_INPUT');
+      });
+
       it('returns error when from/to missing', async () => {
         registry.register(createTestDescriptor());