@lokascript/framework 4.5.0 → 4.7.0

package/src/api/create-dsl.ts

@@ -2,11 +2,25 @@
  * Main API for creating multilingual DSLs
  */
 
-import type { SemanticNode, LanguageTokenizer, LanguagePattern } from '../core/types';
+import type {
+  SemanticNode,
+  LanguageTokenizer,
+  LanguagePattern,
+  PatternMatchResult,
+  RolePatternToken,
+  TokenStream,
+} from '../core/types';
+import { extractRoleValue } from '../core/types';
 import type { CommandSchema } from '../schema';
-import { PatternMatcher, type PatternMatcherProfile } from '../core/pattern-matching';
+import { PatternMatcher } from '../core/pattern-matching';
+import { MatchSkeleton, createStockReader, stockConfidenceModel } from '../core/matching';
 import { generatePattern, type PatternGenLanguageProfile } from '../generation/pattern-generator';
 import { createDomainRenderer, type DomainRenderFn } from '../generation/renderer';
+import {
+  createSchemaPatternGenerator,
+  type SchemaPatternProfile,
+} from '../generation/schema-patterns';
+import { renderPattern, selectRenderPattern } from '../generation/pattern-render';
 import { GrammarTransformer, type LanguageProfile as GrammarProfile } from '../grammar';
 import {
   type Dictionary,
@@ -16,6 +30,7 @@ import {
   InMemoryProfileProvider,
 } from '../interfaces';
 import { isExplicitSyntax, parseExplicit } from '../ir';
+import { readingDifference } from './reading';
 import type { SchemaLookup } from '../ir';
 
 /**
@@ -168,6 +183,20 @@ export interface DSLConfig {
 
   /** Custom patterns to supplement generated ones */
   readonly customPatterns?: LanguagePattern[];
+
+  /**
+   * What reads and writes the DSL (domains rethink Phase 4.7). `'classic'`, the
+   * default: framework's first pattern generator, matcher and schema renderer.
+   * `'core'`: the core semantic's hyperscript parser runs — patterns from
+   * `createSchemaPatternGenerator` (positions descending, as a domain's schemas
+   * read), matched by `MatchSkeleton` with the stock reader and confidence
+   * model, and rendered through those patterns where the domain's own renderer
+   * declines. Parsing's whole-input rule, the refusals, translation's
+   * read-back, compile, explicit syntax and extensions are the same on both.
+   * The domains move to `'core'` one at a time against their gate (Phase 5);
+   * `'classic'` is removed in a later major, once no domain uses it.
+   */
+  readonly engine?: 'classic' | 'core';
 }
 
 /**
@@ -177,6 +206,8 @@ export interface ValidationResult {
   readonly valid: boolean;
   readonly node?: SemanticNode;
   readonly errors?: string[];
+  /** Set when the input was refused for a known reason: `'unread-input'` (`UnreadInputError`). */
+  readonly errorCode?: string;
 }
 
 /**
@@ -186,6 +217,8 @@ export interface CompileResult {
   readonly ok: boolean;
   readonly code?: string;
   readonly errors?: string[];
+  /** Set when the input was refused for a known reason: `'unread-input'` (`UnreadInputError`). */
+  readonly errorCode?: string;
   readonly node?: SemanticNode;
   readonly metadata?: {
     readonly parser: string;
@@ -193,6 +226,55 @@ export interface CompileResult {
   };
 }
 
+/**
+ * Thrown by `parse` (and so `compile`, `validate`, `translate`) when the input reads as a command
+ * only by leaving some of it unread: `select name from users banana` is not `select name from users`.
+ * `reading` is what the longest such match read, `unread` the text after it.
+ */
+export class UnreadInputError extends Error {
+  readonly code = 'unread-input';
+
+  constructor(
+    readonly input: string,
+    readonly language: string,
+    readonly reading: SemanticNode,
+    readonly unread: string
+  ) {
+    super(`"${input}" (${language}) reads as ${reading.action} only by leaving "${unread}" unread`);
+    this.name = 'UnreadInputError';
+  }
+}
+
+/** Options for `translate`. */
+export interface TranslateOptions {
+  /**
+   * `'allow'` returns a translation that does not read back as the source instead of throwing
+   * `TranslationReadBackError`, as semantic's `translate` does with the same option.
+   */
+  readonly lossy?: 'allow' | 'refuse';
+}
+
+/**
+ * Thrown by `translate` when its output, read in the target language, is not the source command:
+ * it does not read at all, or reads as another command, or with a role lost or changed. `partial` is
+ * what `translate` would have returned; `reading` what it reads as (null when it does not read).
+ */
+export class TranslationReadBackError extends Error {
+  readonly code = 'translation-does-not-read-back';
+
+  constructor(
+    readonly partial: string,
+    readonly from: string,
+    readonly to: string,
+    readonly source: SemanticNode,
+    readonly reading: SemanticNode | null,
+    readonly why: string
+  ) {
+    super(`translation ${from} → ${to} does not read back as the source (${why}): "${partial}"`);
+    this.name = 'TranslationReadBackError';
+  }
+}
+
 /**
  * Main multilingual DSL interface.
  */
@@ -208,7 +290,17 @@ export interface MultilingualDSL {
   compile(input: string, language: string): CompileResult;
 
   // Translation
-  translate(input: string, fromLanguage: string, toLanguage: string): string;
+  /**
+   * Parse in one language, render in another. The output must read back as the source command
+   * (the same action and role values, or the same compiled code); when it does not,
+   * `TranslationReadBackError`, unless `{ lossy: 'allow' }`.
+   */
+  translate(
+    input: string,
+    fromLanguage: string,
+    toLanguage: string,
+    options?: TranslateOptions
+  ): string;
 
   /**
    * Render a parsed node back to natural language.
@@ -239,7 +331,11 @@ class DSLRegistry {
    *   subset of the DSL's languages. A built-in command missing a keyword stays
    *   an error, since that is a domain-authoring mistake.
    */
-  constructor(config: DSLConfig, partiallyTranslatedActions: ReadonlySet<string> = new Set()) {
+  constructor(
+    config: DSLConfig,
+    partiallyTranslatedActions: ReadonlySet<string>,
+    private readonly generate: DSLEngine['generate']
+  ) {
     this.schemas = config.schemas;
     this.partiallyTranslatedActions = partiallyTranslatedActions;
 
@@ -262,7 +358,7 @@ class DSLRegistry {
       ) {
         continue; // this extension command was not translated into this language
       }
-      const pattern = generatePattern(schema, lang.patternProfile);
+      const pattern = this.generate(schema, lang.patternProfile);
       patterns.push(pattern);
     }
 
@@ -287,7 +383,7 @@ class DSLRegistry {
  */
 class MultilingualDSLImpl implements MultilingualDSL {
   private registry: DSLRegistry;
-  private matcher: PatternMatcher;
+  private match: DSLEngine['match'];
   private transformer: GrammarTransformer;
   private profileProvider: ProfileProvider;
   private codeGenerator?: CodeGenerator;
@@ -300,10 +396,11 @@ class MultilingualDSLImpl implements MultilingualDSL {
     config: DSLConfig,
     registry: DSLRegistry,
     transformer: GrammarTransformer,
-    profileProvider: ProfileProvider
+    profileProvider: ProfileProvider,
+    engine: DSLEngine
   ) {
     this.registry = registry;
-    this.matcher = new PatternMatcher();
+    this.match = engine.match;
     this.transformer = transformer;
     this.profileProvider = profileProvider;
     if (config.codeGenerator) {
@@ -321,10 +418,7 @@ class MultilingualDSLImpl implements MultilingualDSL {
     // Last resort for render(): covers extension commands with no custom
     // renderer, and any built-in action the domain renderer declines. Lazy, so
     // a DSL that never renders pays nothing.
-    this.schemaFallbackRenderer = createDomainRenderer({
-      schemas: config.schemas,
-      profiles: config.languages.map(l => l.patternProfile),
-    });
+    this.schemaFallbackRenderer = engine.fallbackRenderer(config, registry);
 
     // Build SchemaLookup from config schemas for explicit syntax validation
     const schemaMap = new Map(config.schemas.map(s => [s.action, s]));
@@ -361,35 +455,66 @@ class MultilingualDSLImpl implements MultilingualDSL {
     // Get patterns for language
     const patterns = this.registry.getPatterns(language);
 
-    // Create profile for matcher
-    const profile: PatternMatcherProfile = {
-      code: language,
-    };
-
-    // Try to match each pattern
+    // Every pattern is tried, and only a match that reads the WHOLE input counts; of those, the
+    // highest priority wins, then the highest confidence, then the earliest pattern. The first match
+    // used to win however much it read, so a trailing word was dropped without a word (the domain
+    // gate measured 1,744 of 1,744 such inputs accepted) and an earlier, shorter pattern could
+    // shadow the one that reads everything.
+    const total = tokens.tokens.length;
+    let best: PatternMatchResult | null = null;
+    let longestPartial: PatternMatchResult | null = null;
     for (const pattern of patterns) {
-      const match = this.matcher.matchPattern(tokens, pattern, profile);
-      if (match) {
-        // Build semantic node
-        const node: SemanticNode = {
-          kind: 'command',
-          action: pattern.command,
-          roles: match.captured,
-          metadata: {
-            sourceLanguage: language,
-            sourceText: input,
-            patternId: pattern.id,
-            confidence: match.confidence,
-          },
-        };
-
-        return { node, confidence: match.confidence };
+      const mark = tokens.mark();
+      const match = this.match(tokens, pattern, language);
+      tokens.reset(mark);
+      if (!match) continue;
+      if (match.consumedTokens < total) {
+        if (!longestPartial || match.consumedTokens > longestPartial.consumedTokens)
+          longestPartial = match;
+        continue;
       }
+      if (
+        !best ||
+        match.pattern.priority > best.pattern.priority ||
+        (match.pattern.priority === best.pattern.priority &&
+          match.confidence > best.confidence + 0.001)
+      )
+        best = match;
     }
 
+    if (best) {
+      const node = this.nodeOf(best, input, language);
+      return { node, confidence: best.confidence };
+    }
+    if (longestPartial) {
+      const unread = tokens.tokens
+        .slice(longestPartial.consumedTokens)
+        .map(t => t.value)
+        .join(' ');
+      throw new UnreadInputError(
+        input,
+        language,
+        this.nodeOf(longestPartial, input, language),
+        unread
+      );
+    }
     throw new Error(`No pattern matched for input: ${input}`);
   }
 
+  private nodeOf(match: PatternMatchResult, input: string, language: string): SemanticNode {
+    return {
+      kind: 'command',
+      action: match.pattern.command,
+      roles: match.captured,
+      metadata: {
+        sourceLanguage: language,
+        sourceText: input,
+        patternId: match.pattern.id,
+        confidence: match.confidence,
+      },
+    };
+  }
+
   validate(input: string, language: string): ValidationResult {
     try {
       const result = this.parseWithConfidence(input, language);
@@ -401,11 +526,17 @@ class MultilingualDSLImpl implements MultilingualDSL {
       return {
         valid: false,
         errors: [error instanceof Error ? error.message : String(error)],
+        ...(error instanceof UnreadInputError ? { errorCode: error.code } : {}),
       };
     }
   }
 
-  translate(input: string, fromLanguage: string, toLanguage: string): string {
+  translate(
+    input: string,
+    fromLanguage: string,
+    toLanguage: string,
+    options?: TranslateOptions
+  ): string {
     // Explicit syntax is language-agnostic — return unchanged
     if (isExplicitSyntax(input)) return input;
 
@@ -432,12 +563,18 @@ class MultilingualDSLImpl implements MultilingualDSL {
     const rendered = this.registry.getSupportedLanguages().includes(toLanguage)
       ? this.render(node, toLanguage)
       : null;
-    if (rendered != null) return rendered;
+    if (rendered != null) return this.readsBack(rendered, node, fromLanguage, toLanguage, options);
 
     // render() covered nothing for this action/language pair; fall back to the
     // grammar transformer when profiles exist, otherwise name both remedies.
     if (this.hasGrammarProfile(fromLanguage) && this.hasGrammarProfile(toLanguage)) {
-      return this.transformer.transform(input, fromLanguage, toLanguage);
+      return this.readsBack(
+        this.transformer.transform(input, fromLanguage, toLanguage),
+        node,
+        fromLanguage,
+        toLanguage,
+        options
+      );
     }
     throw new Error(
       `translate() could not render action "${node.action}" in language "${toLanguage}": no ` +
@@ -448,6 +585,40 @@ class MultilingualDSLImpl implements MultilingualDSL {
     );
   }
 
+  /**
+   * `rendered`, read in `to`, must be `source`: the same action and role values, or, when the DSL
+   * compiles, the same code (a parser may keep a translated word its generator normalizes: bdd ja
+   * `表示` is `visible`). The rule is the domain gate's (lokascript-domains). A translation used to be
+   * returned whatever it read as: ja/ko bdd `select` dropped its value, and nothing said so.
+   */
+  private readsBack(
+    rendered: string,
+    source: SemanticNode,
+    from: string,
+    to: string,
+    options?: TranslateOptions
+  ): string {
+    if (options?.lossy === 'allow') return rendered;
+    let back: SemanticNode;
+    try {
+      back = this.parseWithConfidence(rendered, to).node;
+    } catch (error) {
+      const why = error instanceof Error ? error.message : String(error);
+      throw new TranslationReadBackError(rendered, from, to, source, null, why);
+    }
+    const difference = readingDifference(source, back);
+    if (!difference) return rendered;
+    if (this.codeGenerator) {
+      try {
+        if (this.codeGenerator.generate(back) === this.codeGenerator.generate(source))
+          return rendered;
+      } catch {
+        // a generator that cannot compile one of them does not vouch for either
+      }
+    }
+    throw new TranslationReadBackError(rendered, from, to, source, back, difference);
+  }
+
   private hasGrammarProfile(language: string): boolean {
     return this.profileProvider.getProfile(language) != null;
   }
@@ -477,6 +648,7 @@ class MultilingualDSLImpl implements MultilingualDSL {
       return {
         ok: false,
         errors: [error instanceof Error ? error.message : String(error)],
+        ...(error instanceof UnreadInputError ? { errorCode: error.code } : {}),
       };
     }
   }
@@ -678,6 +850,109 @@ export function createMultilingualDSL(config: DSLConfig): MultilingualDSL {
   });
 
   // Create registry and implementation
-  const registry = new DSLRegistry(effectiveConfig, extensionActions);
-  return new MultilingualDSLImpl(effectiveConfig, registry, transformer, profileProvider);
+  const engine = engineFor(config);
+  const registry = new DSLRegistry(effectiveConfig, extensionActions, engine.generate);
+  return new MultilingualDSLImpl(effectiveConfig, registry, transformer, profileProvider, engine);
+}
+
+// =============================================================================
+// The engines: how a DSL turns its schemas into patterns and reads with them
+// =============================================================================
+
+/**
+ * What differs between the engines {@link DSLConfig.engine} names: how each
+ * command's pattern is generated, how input is matched against it, and the
+ * renderer `render()` falls back to. Parsing's whole-input rule, refusals,
+ * translation's read-back, compile, explicit syntax and extensions are shared.
+ */
+interface DSLEngine {
+  generate(schema: CommandSchema, profile: PatternGenLanguageProfile): LanguagePattern;
+  match(tokens: TokenStream, pattern: LanguagePattern, language: string): PatternMatchResult | null;
+  fallbackRenderer(config: DSLConfig, registry: DSLRegistry): DomainRenderFn;
+}
+
+/**
+ * The engine `config.engine` names. An unknown name throws rather than reading
+ * with the default: a domain that misspells `'core'` would otherwise measure
+ * the classic engine against its gate and take the result for the core's.
+ */
+function engineFor(config: DSLConfig): DSLEngine {
+  const name = config.engine ?? 'classic';
+  if (name === 'classic') return classicEngine();
+  if (name === 'core') return coreEngine();
+  throw new Error(`Unknown DSL engine "${String(name)}" (expected "classic" or "core")`);
+}
+
+/** framework's first generator and matcher, and its schema renderer (`engine: 'classic'`). */
+function classicEngine(): DSLEngine {
+  const matcher = new PatternMatcher();
+  return {
+    generate: (schema, profile) => generatePattern(schema, profile),
+    match: (tokens, pattern, language) => matcher.matchPattern(tokens, pattern, { code: language }),
+    fallbackRenderer: config =>
+      createDomainRenderer({
+        schemas: config.schemas,
+        profiles: config.languages.map(l => l.patternProfile),
+      }),
+  };
+}
+
+/**
+ * A domain profile as the schema-pattern generator reads it: every role marker
+ * with a side (the word order's default where the profile names none).
+ */
+function schemaProfileOf(profile: PatternGenLanguageProfile): SchemaPatternProfile {
+  const side = profile.wordOrder === 'SOV' ? 'after' : 'before';
+  const roleMarkers: Record<string, NonNullable<SchemaPatternProfile['roleMarkers'][string]>> = {};
+  for (const [role, marker] of Object.entries(profile.roleMarkers ?? {})) {
+    roleMarkers[role] = {
+      primary: marker.primary,
+      ...(marker.alternatives ? { alternatives: marker.alternatives } : {}),
+      position: marker.position ?? side,
+    };
+  }
+  return {
+    code: profile.code,
+    wordOrder: profile.wordOrder,
+    keywords: profile.keywords,
+    roleMarkers,
+  };
+}
+
+/**
+ * The core (domains rethink Phase 4, `engine: 'core'`): the schema-pattern
+ * generator, the skeleton, the stock reader.
+ */
+function coreEngine(): DSLEngine {
+  const generator = createSchemaPatternGenerator({ positionOrder: 'descending' });
+  const matcher = new MatchSkeleton(createStockReader(), stockConfidenceModel);
+  return {
+    generate: (schema, profile) => generator.generatePattern(schema, schemaProfileOf(profile)),
+    match: (tokens, pattern) => matcher.matchPattern(tokens, pattern),
+    fallbackRenderer: (config, registry) => {
+      const schemas = new Map(config.schemas.map(s => [s.action, s]));
+      const hooks = {
+        hasRealRole: (node: SemanticNode, role: string) => {
+          const value = node.roles.get(role) as { implicit?: unknown } | undefined;
+          return value !== undefined && value.implicit !== true;
+        },
+        writeRole: (token: RolePatternToken, node: SemanticNode) => {
+          if (!node.roles.has(token.role)) return null;
+          let value = extractRoleValue(node, token.role);
+          const spec = schemas.get(node.action)?.roles.find(r => r.role === token.role);
+          if (spec?.quoteMultiword && /\s/.test(value) && !/^(["']).*\1$/.test(value)) {
+            value = `"${value}"`;
+          }
+          return value || null;
+        },
+      };
+      return (node, language) => {
+        const patterns = registry.getPatterns(language).filter(p => p.command === node.action);
+        const pattern = selectRenderPattern(node, patterns, hooks);
+        if (!pattern) return null;
+        const text = renderPattern(node, pattern, hooks).join(' ');
+        return text || null;
+      };
+    },
+  };
 }

package/src/api/domain-registry.test.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect, beforeEach } from 'vitest';
 import { DomainRegistry } from './domain-registry';
 import type { DomainDescriptor, MCPToolDefinition } from './domain-registry';
-import type { MultilingualDSL } from './create-dsl';
+import { UnreadInputError, type MultilingualDSL } from './create-dsl';
 import type { SemanticNode } from '../core/types';
 
 // =============================================================================
@@ -11,10 +11,19 @@ import type { SemanticNode } from '../core/types';
 function createMockDSL(): MultilingualDSL {
   return {
     parse(input: string, language: string): SemanticNode {
+      // A rendering is `[<lang>] <source>`: read the source back out of it.
+      const source = input.replace(/^\[\w+\] /, '');
+      if (source.includes(' zzqx'))
+        throw new UnreadInputError(
+          input,
+          language,
+          this.parse(source.replace(' zzqx', ''), language),
+          'zzqx'
+        );
       return {
         kind: 'command',
-        action: input.split(/\s+/)[0].toLowerCase(),
-        roles: new Map([['raw', { type: 'expression' as const, raw: input }]]),
+        action: source.split(/\s+/)[0].toLowerCase(),
+        roles: new Map([['raw', { type: 'expression' as const, raw: source }]]),
         metadata: { sourceLanguage: language },
       };
     },
@@ -33,7 +42,9 @@ function createMockDSL(): MultilingualDSL {
       try {
         const node = this.parse(input, language);
         return { ok: true, code: `compiled(${input})`, node };
-      } catch {
+      } catch (error) {
+        if (error instanceof UnreadInputError)
+          return { ok: false, errors: [error.message], errorCode: error.code };
         return { ok: false, errors: ['Compile failed'] };
       }
     },
@@ -46,7 +57,14 @@ function createMockDSL(): MultilingualDSL {
   };
 }
 
+/** Writes the whole source, so a rendering reads back. */
 function mockRenderer(node: SemanticNode, language: string): string {
+  const raw = (node.roles.get('raw') as { raw?: string } | undefined)?.raw ?? node.action;
+  return `[${language}] ${raw}`;
+}
+
+/** Writes the action alone: `select name` comes out as `select`, which does not read back. */
+function lossyRenderer(node: SemanticNode, language: string): string {
   return `[${language}] ${node.action}`;
 }
 
@@ -236,6 +254,19 @@ describe('DomainRegistry', () => {
         expect(data.ok).toBe(true);
         expect(data.code).toBe('compiled(select name)');
       });
+
+      it('reports input left unread as an UNREAD_INPUT diagnostic', async () => {
+        registry.register(createTestDescriptor());
+        const result = await registry.handleToolCall('compile_test', {
+          query: 'select name zzqx',
+          language: 'en',
+        });
+
+        const data = JSON.parse(result!.content[0].text);
+        expect(data.ok).toBe(false);
+        expect(data.diagnostics[0].code).toBe('UNREAD_INPUT');
+        expect(data.diagnostics[0].message).toMatch(/zzqx/);
+      });
     });
 
     describe('validate', () => {
@@ -263,10 +294,44 @@ describe('DomainRegistry', () => {
         const data = JSON.parse(result!.content[0].text);
         expect(data.input.query).toBe('select name');
         expect(data.input.language).toBe('en');
-        expect(data.rendered.text).toBe('[ja] select');
+        expect(data.ok).toBe(true);
+        expect(data.rendered.text).toBe('[ja] select name');
         expect(data.rendered.language).toBe('ja');
       });
 
+      it('refuses a rendering that does not read back, with its text under partial', async () => {
+        registry.register(createTestDescriptor({ getRenderer: () => lossyRenderer }));
+        const result = await registry.handleToolCall('translate_test', {
+          query: 'select name',
+          from: 'en',
+          to: 'ja',
+        });
+
+        expect(result!.isError).toBe(true);
+        const data = JSON.parse(result!.content[0].text);
+        expect(data.ok).toBe(false);
+        expect(data.partial).toBe('[ja] select');
+        expect(data.rendered).toBeUndefined();
+        expect(data.diagnostics[0].code).toBe('TRANSLATION_DOES_NOT_READ_BACK');
+        expect(data.diagnostics[0].message).toMatch(/raw: select name → select/);
+      });
+
+      it('refuses input left unread, naming it', async () => {
+        registry.register(createTestDescriptor());
+        const result = await registry.handleToolCall('translate_test', {
+          query: 'select name zzqx',
+          from: 'en',
+          to: 'ja',
+        });
+
+        expect(result!.isError).toBe(true);
+        const data = JSON.parse(result!.content[0].text);
+        expect(data.ok).toBe(false);
+        expect(data.unread).toBe('zzqx');
+        expect(data.reading.action).toBe('select');
+        expect(data.diagnostics[0].code).toBe('UNREAD_INPUT');
+      });
+
       it('returns error when from/to missing', async () => {
         registry.register(createTestDescriptor());