@lokascript/framework 2.0.0 → 2.2.0

package/src/api/create-dsl.ts

@@ -14,6 +14,8 @@ import {
   InMemoryDictionary,
   InMemoryProfileProvider,
 } from '../interfaces';
+import { isExplicitSyntax, parseExplicit } from '../ir';
+import type { SchemaLookup } from '../ir';
 
 /**
  * Language configuration for a DSL.
@@ -172,6 +174,7 @@ class MultilingualDSLImpl implements MultilingualDSL {
   private matcher: PatternMatcher;
   private transformer: GrammarTransformer;
   private codeGenerator?: CodeGenerator;
+  private schemaLookup: SchemaLookup;
 
   constructor(config: DSLConfig, registry: DSLRegistry, transformer: GrammarTransformer) {
     this.registry = registry;
@@ -180,6 +183,12 @@ class MultilingualDSLImpl implements MultilingualDSL {
     if (config.codeGenerator) {
       this.codeGenerator = config.codeGenerator;
     }
+
+    // Build SchemaLookup from config schemas for explicit syntax validation
+    const schemaMap = new Map(config.schemas.map(s => [s.action, s]));
+    this.schemaLookup = {
+      getSchema: (action: string) => schemaMap.get(action),
+    };
   }
 
   parse(input: string, language: string): SemanticNode {
@@ -188,6 +197,16 @@ class MultilingualDSLImpl implements MultilingualDSL {
   }
 
   parseWithConfidence(input: string, language: string): { node: SemanticNode; confidence: number } {
+    // Try explicit bracket syntax first (language-agnostic)
+    if (isExplicitSyntax(input)) {
+      const node = parseExplicit(input, { schemaLookup: this.schemaLookup });
+      // Only match if this DSL has a schema for the action
+      if (!this.schemaLookup.getSchema(node.action)) {
+        throw new Error(`No schema for action "${node.action}" in this DSL`);
+      }
+      return { node, confidence: 1.0 };
+    }
+
     // Get tokenizer for language
     const tokenizer = this.registry.getTokenizer(language);
     if (!tokenizer) {
@@ -245,6 +264,8 @@ class MultilingualDSLImpl implements MultilingualDSL {
   }
 
   translate(input: string, fromLanguage: string, toLanguage: string): string {
+    // Explicit syntax is language-agnostic — return unchanged
+    if (isExplicitSyntax(input)) return input;
     // Use injected grammar transformer
     return this.transformer.transform(input, fromLanguage, toLanguage);
   }

package/src/api/dispatcher.test.ts

@@ -333,4 +333,70 @@ describe('CrossDomainDispatcher', () => {
       expect(callCount).toBe(1);
     });
   });
+
+  // ---------------------------------------------------------------------------
+  // compileComposite()
+  // ---------------------------------------------------------------------------
+
+  describe('compileComposite', () => {
+    it('compiles multi-line input across domains', async () => {
+      registry.register(createDescriptor('sql', 'select'));
+      registry.register(createDescriptor('bdd', 'given'));
+      const dispatcher = new CrossDomainDispatcher(registry);
+
+      const result = await dispatcher.compileComposite(
+        'select name from users\ngiven page http://example.com'
+      );
+
+      expect(result.statements).toHaveLength(2);
+      expect(result.errors).toHaveLength(0);
+      expect(result.statements[0].domain).toBe('sql');
+      expect(result.statements[0].ok).toBe(true);
+      expect(result.statements[0].code).toContain('compiled:select');
+      expect(result.statements[1].domain).toBe('bdd');
+      expect(result.statements[1].ok).toBe(true);
+    });
+
+    it('skips comments and empty lines', async () => {
+      registry.register(createDescriptor('sql', 'select'));
+      const dispatcher = new CrossDomainDispatcher(registry);
+
+      const result = await dispatcher.compileComposite(
+        '// comment\n\nselect name\n-- another comment\n'
+      );
+
+      expect(result.statements).toHaveLength(1);
+      expect(result.statements[0].line).toBe(3);
+      expect(result.errors).toHaveLength(0);
+    });
+
+    it('collects unmatched lines as errors', async () => {
+      registry.register(createDescriptor('sql', 'select'));
+      const dispatcher = new CrossDomainDispatcher(registry);
+
+      const result = await dispatcher.compileComposite('select name\nunknown command\nselect id');
+
+      expect(result.statements).toHaveLength(2);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].line).toBe(2);
+      expect(result.errors[0].message).toBe('No domain matched this input');
+    });
+
+    it('handles empty input', async () => {
+      const dispatcher = new CrossDomainDispatcher(registry);
+      const result = await dispatcher.compileComposite('');
+      expect(result.statements).toHaveLength(0);
+      expect(result.errors).toHaveLength(0);
+    });
+
+    it('preserves line numbers', async () => {
+      registry.register(createDescriptor('sql', 'select'));
+      const dispatcher = new CrossDomainDispatcher(registry);
+
+      const result = await dispatcher.compileComposite('// header\n\nselect name\n\nselect id');
+
+      expect(result.statements[0].line).toBe(3);
+      expect(result.statements[1].line).toBe(5);
+    });
+  });
 });

package/src/api/dispatcher.ts

@@ -71,6 +71,24 @@ export interface CompositeParseResult {
   readonly errors: readonly CompositeError[];
 }
 
+/** A single compiled statement in a composite compile result */
+export interface CompositeCompileStatement {
+  readonly line: number;
+  readonly input: string;
+  readonly domain: string;
+  readonly ok: boolean;
+  readonly code?: string | undefined;
+  readonly errors?: string[] | undefined;
+}
+
+/** Result of multi-domain composite compilation */
+export interface CompositeCompileResult {
+  /** Successfully detected and compiled statements */
+  readonly statements: readonly CompositeCompileStatement[];
+  /** Lines that could not be matched to any domain */
+  readonly errors: readonly CompositeError[];
+}
+
 /** Dispatcher configuration options */
 export interface DispatcherOptions {
   /** Minimum confidence to accept a match (default: 0.5) */
@@ -123,26 +141,37 @@ export class CrossDomainDispatcher {
     const candidates: DispatchResult[] = [];
 
     for (const name of domainNames) {
-      const dsl = await this.registry.getDSLForDomain(name);
-      if (!dsl) continue;
-
       try {
+        const dsl = await this.registry.getDSLForDomain(name);
+        if (!dsl) continue;
+
         const { node, confidence } = dsl.parseWithConfidence(input, language);
         if (confidence >= this.minConfidence) {
           candidates.push({ domain: name, node, confidence, dsl });
         }
       } catch {
-        // Domain could not parse this input — skip
+        // Domain could not be loaded or parse this input — skip
       }
     }
 
     if (candidates.length === 0) return null;
 
-    // Sort by confidence descending, break ties by priority order
-    candidates.sort((a, b) => {
-      if (b.confidence !== a.confidence) return b.confidence - a.confidence;
-      return this.getPriorityIndex(a.domain) - this.getPriorityIndex(b.domain);
-    });
+    // Sort by priority first, then confidence within each priority tier.
+    // Simpler schemas (fewer roles) often score higher confidence than more
+    // specific ones — e.g., voice's "select [patient:greedy]" gets 1.0 while
+    // sql's "select [columns] from [source]" gets 0.71 for the same input.
+    // Priority ordering ensures domain specificity wins when both match.
+    if (this.priority.length > 0) {
+      candidates.sort((a, b) => {
+        const pa = this.getPriorityIndex(a.domain);
+        const pb = this.getPriorityIndex(b.domain);
+        if (pa !== pb) return pa - pb;
+        return b.confidence - a.confidence;
+      });
+    } else {
+      // No priority configured: pure confidence ordering
+      candidates.sort((a, b) => b.confidence - a.confidence);
+    }
 
     return candidates[0];
   }
@@ -198,6 +227,45 @@ export class CrossDomainDispatcher {
     return { ...result, domain: detected.domain };
   }
 
+  /**
+   * Compile multi-line input where each line may belong to a different domain.
+   * Detects the domain for each line and compiles it, returning per-line results.
+   * Empty lines and comment lines are skipped.
+   */
+  async compileComposite(input: string, language = 'en'): Promise<CompositeCompileResult> {
+    const lines = input.split('\n');
+    const statements: CompositeCompileStatement[] = [];
+    const errors: CompositeError[] = [];
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      if (isSkippableLine(line)) continue;
+
+      const trimmed = line.trim();
+      const detected = await this.detect(trimmed, language);
+
+      if (detected) {
+        const result = detected.dsl.compile(trimmed, language);
+        statements.push({
+          line: i + 1,
+          input: trimmed,
+          domain: detected.domain,
+          ok: result.ok,
+          code: result.code,
+          errors: result.errors,
+        });
+      } else {
+        errors.push({
+          line: i + 1,
+          input: trimmed,
+          message: 'No domain matched this input',
+        });
+      }
+    }
+
+    return { statements, errors };
+  }
+
   /**
    * Validate input against all domains, returning the best match.
    */

package/src/api/domain-registry.ts

@@ -33,6 +33,16 @@
 import type { SemanticNode } from '../core/types';
 import type { MultilingualDSL, CompileResult, ValidationResult } from './create-dsl';
 import type { NaturalLanguageRenderer } from '../generation/renderer';
+import type { CommandSchema } from '../schema/command-schema';
+import type { SchemaLookup } from '../ir/types';
+import type { Diagnostic } from '../generation/diagnostics';
+import type { GeneratedPrompt, PromptGeneratorConfig } from '../prompts/types';
+import type { SynthesisConfig, SynthesisResult } from '../training/types';
+import type { LSEFeedback } from '../feedback/types';
+import { generatePrompt as _generatePrompt } from '../prompts/prompt-generator';
+import { synthesizeFromSchemas as _synthesizeFromSchemas } from '../training/schema-synthesizer';
+import { buildFeedback as _buildFeedback } from '../feedback/feedback-formatter';
+import { parseExplicit as _parseExplicit } from '../ir/explicit-parser';
 
 // =============================================================================
 // Descriptor Interface
@@ -60,6 +70,9 @@ export interface DomainDescriptor {
   /** Description of the primary input parameter */
   readonly inputDescription: string;
 
+  /** What compile_{name} produces (e.g., "a SQL query string"). Used in tool description. */
+  readonly outputDescription?: string;
+
   /**
    * Factory to lazily create the DSL instance.
    * Called once on first tool use. The result is cached.
@@ -80,6 +93,13 @@ export interface DomainDescriptor {
    */
   readonly tools?: readonly ('parse' | 'compile' | 'validate' | 'translate')[];
 
+  /**
+   * Optional command schemas for this domain.
+   * When provided, enables system prompt generation, training data synthesis,
+   * and structured feedback via the registry's convenience methods.
+   */
+  readonly schemas?: readonly CommandSchema[];
+
   /**
    * Optional scan configuration for AOT compilation and Vite plugin.
    * Declares which HTML attributes/script types this domain uses.
@@ -150,6 +170,19 @@ export class DomainRegistry {
     this.descriptors.set(descriptor.name, descriptor);
   }
 
+  /**
+   * Attach schemas to an already-registered domain.
+   * Useful for lazy loading schemas after initial registration.
+   */
+  setSchemas(name: string, schemas: readonly CommandSchema[]): void {
+    const existing = this.descriptors.get(name);
+    if (!existing) {
+      throw new Error(`Domain not registered: ${name}`);
+    }
+    // Create a new descriptor with schemas attached
+    this.descriptors.set(name, { ...existing, schemas });
+  }
+
   /**
    * Get all registered domain names.
    */
@@ -230,6 +263,89 @@ export class DomainRegistry {
     return this.getDSL(descriptor);
   }
 
+  // ---------------------------------------------------------------------------
+  // LLM ↔ LSE Convenience Methods
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Generate an LLM system prompt for a domain from its schemas.
+   * Returns null if the domain is not registered or has no schemas.
+   */
+  generatePrompt(domain: string, config?: Partial<PromptGeneratorConfig>): GeneratedPrompt | null {
+    const descriptor = this.descriptors.get(domain);
+    if (!descriptor?.schemas || descriptor.schemas.length === 0) return null;
+
+    return _generatePrompt({
+      domain,
+      description: descriptor.description,
+      schemas: descriptor.schemas,
+      ...config,
+    });
+  }
+
+  /**
+   * Synthesize training data for a domain from its schemas.
+   * Returns null if the domain is not registered or has no schemas.
+   */
+  generateTrainingData(domain: string, config?: Partial<SynthesisConfig>): SynthesisResult | null {
+    const descriptor = this.descriptors.get(domain);
+    if (!descriptor?.schemas || descriptor.schemas.length === 0) return null;
+
+    return _synthesizeFromSchemas(descriptor.schemas, {
+      domain,
+      ...config,
+    });
+  }
+
+  /**
+   * Build structured feedback for a failed LSE input in a domain.
+   * Returns null if the domain is not registered or has no schemas.
+   */
+  buildFeedback(
+    domain: string,
+    input: string,
+    inputFormat: 'explicit' | 'json' | 'natural',
+    diagnostics: readonly Diagnostic[]
+  ): LSEFeedback | null {
+    const descriptor = this.descriptors.get(domain);
+    if (!descriptor?.schemas || descriptor.schemas.length === 0) return null;
+
+    const schemaLookup = this.createSchemaLookup(descriptor.schemas);
+
+    // Try to extract the action from the input
+    let action: string | undefined;
+    try {
+      if (inputFormat === 'explicit') {
+        const node = _parseExplicit(input, { schemaLookup });
+        action = node.action;
+      }
+    } catch {
+      // Action detection failed — still build feedback without it
+    }
+
+    return _buildFeedback(input, inputFormat, diagnostics, schemaLookup, action);
+  }
+
+  /**
+   * Get schemas for a domain (if available).
+   */
+  getSchemas(domain: string): readonly CommandSchema[] | null {
+    const descriptor = this.descriptors.get(domain);
+    if (!descriptor?.schemas) return null;
+    return descriptor.schemas;
+  }
+
+  /**
+   * Create a SchemaLookup from a domain's schemas.
+   */
+  private createSchemaLookup(schemas: readonly CommandSchema[]): SchemaLookup {
+    const map = new Map<string, CommandSchema>();
+    for (const s of schemas) {
+      map.set(s.action, s);
+    }
+    return { getSchema: (action: string) => map.get(action) };
+  }
+
   // ---------------------------------------------------------------------------
   // Private: DSL/renderer lifecycle
   // ---------------------------------------------------------------------------
@@ -380,7 +496,7 @@ function generateToolDefinitions(desc: DomainDescriptor): MCPToolDefinition[] {
       case 'parse':
         tools.push({
           name: `parse_${desc.name}`,
-          description: `Parse a ${desc.description} input into a semantic representation. Supports: ${langList}.`,
+          description: `Parse a ${desc.description} input into a semantic representation (action, roles, confidence). Supports: ${langList}.`,
           inputSchema: {
             type: 'object',
             properties: {
@@ -402,7 +518,7 @@ function generateToolDefinitions(desc: DomainDescriptor): MCPToolDefinition[] {
       case 'compile':
         tools.push({
           name: `compile_${desc.name}`,
-          description: `Compile a ${desc.description} input to target code. Supports: ${langList}.`,
+          description: `Compile a ${desc.description} input to ${desc.outputDescription ?? 'target code'}. Supports: ${langList}.`,
           inputSchema: {
             type: 'object',
             properties: {
@@ -424,7 +540,7 @@ function generateToolDefinitions(desc: DomainDescriptor): MCPToolDefinition[] {
       case 'validate':
         tools.push({
           name: `validate_${desc.name}`,
-          description: `Validate ${desc.description} syntax. Returns whether it parses successfully and any errors. Supports: ${langList}.`,
+          description: `Check ${desc.description} syntax for errors. Returns {valid, errors[]}. Supports: ${langList}.`,
           inputSchema: {
             type: 'object',
             properties: {
@@ -446,7 +562,7 @@ function generateToolDefinitions(desc: DomainDescriptor): MCPToolDefinition[] {
       case 'translate':
         tools.push({
           name: `translate_${desc.name}`,
-          description: `Translate ${desc.description} input between natural languages. Parses in source language and renders in target language.`,
+          description: `Translate ${desc.description} input between natural languages (requires 'from' and 'to' codes). Supports: ${langList}.`,
           inputSchema: {
             type: 'object',
             properties: {

package/src/core/pattern-matching/pattern-matcher.test.ts

@@ -300,6 +300,137 @@ describe('PatternMatcher', () => {
         expect(result).not.toBeNull();
         expect(result?.captured.has('patient')).toBe(true);
       });
+
+      it('should stop greedy role inside group at parent verb keyword (SOV WHERE)', () => {
+        // SOV pattern: [source] から [columns] [GROUP: 条件 condition(greedy)] 選択
+        // Input: users から name 条件 age > 18 選択
+        // The greedy condition role must stop at 選択 (the verb keyword outside the group)
+        const schema = defineCommand({
+          action: 'select',
+          roles: [
+            defineRole({
+              role: 'columns',
+              required: true,
+              expectedTypes: ['expression'],
+              svoPosition: 2,
+              sovPosition: 1,
+            }),
+            defineRole({
+              role: 'source',
+              required: true,
+              expectedTypes: ['expression'],
+              svoPosition: 1,
+              sovPosition: 2,
+              markerOverride: { ja: 'から' },
+            }),
+            defineRole({
+              role: 'condition',
+              required: false,
+              expectedTypes: ['expression'],
+              greedy: true,
+              svoPosition: 0,
+              sovPosition: 0,
+              markerOverride: { ja: '条件' },
+            }),
+          ],
+        });
+
+        const profile: PatternGenLanguageProfile = {
+          code: 'ja',
+          wordOrder: 'SOV',
+          keywords: {
+            select: { primary: '選択' },
+          },
+          roleMarkers: {
+            condition: { primary: '条件', position: 'before' },
+          },
+        };
+
+        const pattern = generatePattern(schema, profile);
+
+        // Input: users から name 条件 age > 18 選択
+        const tokens = createMockTokenStream([
+          createToken({ kind: 'identifier', value: 'users', position: createPosition(0, 5) }),
+          createToken({ kind: 'keyword', value: 'から', position: createPosition(6, 8) }),
+          createToken({ kind: 'identifier', value: 'name', position: createPosition(9, 13) }),
+          createToken({ kind: 'keyword', value: '条件', position: createPosition(14, 16) }),
+          createToken({ kind: 'identifier', value: 'age', position: createPosition(17, 20) }),
+          createToken({ kind: 'operator', value: '>', position: createPosition(21, 22) }),
+          createToken({ kind: 'literal', value: '18', position: createPosition(23, 25) }),
+          createToken({ kind: 'keyword', value: '選択', position: createPosition(26, 28) }),
+        ]);
+
+        const result = matcher.matchPattern(tokens, pattern);
+
+        expect(result).not.toBeNull();
+        expect(result?.captured.has('source')).toBe(true);
+        expect(result?.captured.get('source')).toMatchObject({ raw: 'users' });
+        expect(result?.captured.has('columns')).toBe(true);
+        expect(result?.captured.get('columns')).toMatchObject({ raw: 'name' });
+        expect(result?.captured.has('condition')).toBe(true);
+        expect(result?.captured.get('condition')).toMatchObject({ raw: 'age > 18' });
+      });
+
+      it('should match SOV pattern without optional greedy group', () => {
+        // Same schema but input has no WHERE clause
+        // Input: users から name 選択
+        const schema = defineCommand({
+          action: 'select',
+          roles: [
+            defineRole({
+              role: 'columns',
+              required: true,
+              expectedTypes: ['expression'],
+              svoPosition: 2,
+              sovPosition: 1,
+            }),
+            defineRole({
+              role: 'source',
+              required: true,
+              expectedTypes: ['expression'],
+              svoPosition: 1,
+              sovPosition: 2,
+              markerOverride: { ja: 'から' },
+            }),
+            defineRole({
+              role: 'condition',
+              required: false,
+              expectedTypes: ['expression'],
+              greedy: true,
+              svoPosition: 0,
+              sovPosition: 0,
+              markerOverride: { ja: '条件' },
+            }),
+          ],
+        });
+
+        const profile: PatternGenLanguageProfile = {
+          code: 'ja',
+          wordOrder: 'SOV',
+          keywords: {
+            select: { primary: '選択' },
+          },
+          roleMarkers: {
+            condition: { primary: '条件', position: 'before' },
+          },
+        };
+
+        const pattern = generatePattern(schema, profile);
+
+        const tokens = createMockTokenStream([
+          createToken({ kind: 'identifier', value: 'users', position: createPosition(0, 5) }),
+          createToken({ kind: 'keyword', value: 'から', position: createPosition(6, 8) }),
+          createToken({ kind: 'identifier', value: 'name', position: createPosition(9, 13) }),
+          createToken({ kind: 'keyword', value: '選択', position: createPosition(14, 16) }),
+        ]);
+
+        const result = matcher.matchPattern(tokens, pattern);
+
+        expect(result).not.toBeNull();
+        expect(result?.captured.has('source')).toBe(true);
+        expect(result?.captured.has('columns')).toBe(true);
+        expect(result?.captured.has('condition')).toBe(false);
+      });
     });
   });