@lokascript/framework 2.0.0 → 2.2.0

package/src/integration/llm-lse-integration.test.ts

@@ -0,0 +1,457 @@
+/**
+ * End-to-End Integration Tests — LLM ↔ LSE Pipeline
+ *
+ * Tests the full integration of system prompts, training data,
+ * feedback loops, and registry convenience methods.
+ */
+
+import { describe, it, expect, beforeAll } from 'vitest';
+import {
+  DomainRegistry,
+  generatePrompt,
+  synthesizeFromSchemas,
+  toJSONL,
+  parseJSONL,
+  buildFeedback,
+  needsDisambiguation,
+  buildDisambiguation,
+  PatternTracker,
+  parseExplicit,
+  renderExplicit,
+  semanticNodeToJSON,
+  createCommandNode,
+  createSelector,
+  createLiteral,
+  createExpression,
+  defineCommand,
+  defineRole,
+} from '../index';
+
+// =============================================================================
+// Test schemas (minimal — for fast, self-contained tests)
+// =============================================================================
+
+const testSchemas = [
+  defineCommand({
+    action: 'fetch',
+    description: 'Fetch data from a URL',
+    category: 'source',
+    primaryRole: 'source',
+    roles: [
+      defineRole({
+        role: 'source',
+        description: 'URL to fetch from',
+        required: true,
+        expectedTypes: ['expression'],
+      }),
+      defineRole({
+        role: 'style',
+        description: 'Response format',
+        required: false,
+        expectedTypes: ['expression'],
+      }),
+      defineRole({
+        role: 'destination',
+        description: 'Target element',
+        required: false,
+        expectedTypes: ['selector'],
+      }),
+    ],
+  }),
+  defineCommand({
+    action: 'toggle',
+    description: 'Toggle a class on an element',
+    category: 'mutation',
+    primaryRole: 'patient',
+    roles: [
+      defineRole({
+        role: 'patient',
+        description: 'CSS class or attribute to toggle',
+        required: true,
+        expectedTypes: ['selector'],
+      }),
+      defineRole({
+        role: 'destination',
+        description: 'Target element',
+        required: false,
+        expectedTypes: ['selector'],
+      }),
+    ],
+  }),
+];
+
+// =============================================================================
+// Pillar 1: System Prompt Generation → Parse round-trip
+// =============================================================================
+
+describe('Pillar 1: System Prompt Generation', () => {
+  it('generates prompt containing all commands', () => {
+    const prompt = generatePrompt({
+      domain: 'test',
+      description: 'Test domain',
+      schemas: testSchemas,
+    });
+
+    expect(prompt.text).toContain('fetch');
+    expect(prompt.text).toContain('toggle');
+    expect(prompt.metadata.commandCount).toBe(2);
+    expect(prompt.metadata.domain).toBe('test');
+  });
+
+  it('prompt includes role documentation', () => {
+    const prompt = generatePrompt({
+      domain: 'test',
+      description: 'Test domain',
+      schemas: testSchemas,
+    });
+
+    expect(prompt.text).toContain('source');
+    expect(prompt.text).toContain('patient');
+    expect(prompt.text).toContain('URL to fetch from');
+  });
+
+  it('prompt includes examples in bracket syntax', () => {
+    const prompt = generatePrompt({
+      domain: 'test',
+      description: 'Test domain',
+      schemas: testSchemas,
+      outputFormat: 'explicit',
+    });
+
+    expect(prompt.text).toContain('[fetch');
+    expect(prompt.text).toContain('[toggle');
+  });
+
+  it('prompt includes JSON format examples', () => {
+    const prompt = generatePrompt({
+      domain: 'test',
+      description: 'Test domain',
+      schemas: testSchemas,
+      outputFormat: 'json',
+    });
+
+    expect(prompt.text).toContain('"action"');
+    expect(prompt.text).toContain('"roles"');
+  });
+
+  it('respects token budget', () => {
+    const small = generatePrompt({
+      domain: 'test',
+      description: 'Test domain',
+      schemas: testSchemas,
+      maxTokens: 200,
+    });
+
+    const large = generatePrompt({
+      domain: 'test',
+      description: 'Test domain',
+      schemas: testSchemas,
+      maxTokens: 10000,
+    });
+
+    expect(small.text.length).toBeLessThan(large.text.length);
+  });
+});
+
+// =============================================================================
+// Pillar 2: Training Data → JSONL Round-trip
+// =============================================================================
+
+describe('Pillar 2: Training Data Generation', () => {
+  it('synthesizes pairs from schemas', () => {
+    const result = synthesizeFromSchemas(testSchemas, { domain: 'test' });
+
+    expect(result.pairs.length).toBeGreaterThan(0);
+    expect(result.metadata.domain).toBe('test');
+    expect(result.metadata.commandCount).toBe(2);
+  });
+
+  it('each pair has required fields', () => {
+    const result = synthesizeFromSchemas(testSchemas, { domain: 'test' });
+
+    for (const pair of result.pairs) {
+      expect(pair.domain).toBe('test');
+      expect(pair.action).toBeTruthy();
+      expect(pair.explicit).toContain('[');
+      expect(pair.natural).toBeTruthy();
+      expect(pair.quality).toBeGreaterThanOrEqual(0);
+      expect(pair.quality).toBeLessThanOrEqual(1);
+    }
+  });
+
+  it('JSONL round-trips correctly', () => {
+    const result = synthesizeFromSchemas(testSchemas, { domain: 'test' });
+    const jsonl = toJSONL(result.pairs);
+    const parsed = parseJSONL(jsonl);
+
+    expect(parsed.length).toBe(result.pairs.length);
+    for (let i = 0; i < parsed.length; i++) {
+      expect(parsed[i].metadata.action).toBe(result.pairs[i].action);
+      expect(parsed[i].completion).toBe(result.pairs[i].explicit);
+    }
+  });
+
+  it('synthetic examples parse back at full confidence', () => {
+    const result = synthesizeFromSchemas(testSchemas, { domain: 'test' });
+
+    for (const pair of result.pairs) {
+      // Each explicit syntax should parse without error
+      const node = parseExplicit(pair.explicit);
+      expect(node.action).toBe(pair.action);
+    }
+  });
+
+  it('respects maxPairsPerCommand', () => {
+    const result = synthesizeFromSchemas(testSchemas, {
+      domain: 'test',
+      maxPairsPerCommand: 2,
+    });
+
+    // Count pairs per action
+    const counts = new Map<string, number>();
+    for (const pair of result.pairs) {
+      counts.set(pair.action, (counts.get(pair.action) || 0) + 1);
+    }
+
+    for (const count of counts.values()) {
+      expect(count).toBeLessThanOrEqual(2);
+    }
+  });
+});
+
+// =============================================================================
+// Pillar 3: Feedback Loop
+// =============================================================================
+
+describe('Pillar 3: Feedback Loop', () => {
+  // --- 3A: Structured Error Feedback ---
+
+  it('invalid bracket syntax → structured feedback with hints', () => {
+    const feedback = buildFeedback(
+      '[fetch]',
+      'explicit',
+      [{ severity: 'error', message: 'Required role source missing', code: 'missing-role' }],
+      { getSchema: a => testSchemas.find(s => s.action === a) },
+      'fetch'
+    );
+
+    expect(feedback.accepted).toBe(false);
+    expect(feedback.diagnostics[0].fixType).toBe('missing_role');
+    expect(feedback.hints.length).toBeGreaterThan(0);
+    expect(feedback.schema?.requiredRoles).toContain('source');
+    expect(feedback.correctedExample?.explicit).toContain('[fetch');
+  });
+
+  it('accepted input → accepted: true, no corrected example', () => {
+    const feedback = buildFeedback(
+      '[fetch source:/api/users]',
+      'explicit',
+      [],
+      { getSchema: a => testSchemas.find(s => s.action === a) },
+      'fetch'
+    );
+
+    expect(feedback.accepted).toBe(true);
+    expect(feedback.correctedExample).toBeUndefined();
+  });
+
+  // --- 3B: Confidence-Aware Disambiguation ---
+
+  it('borderline confidence → disambiguation', () => {
+    expect(needsDisambiguation(0.6)).toBe(true);
+    expect(needsDisambiguation(0.9)).toBe(false);
+    expect(needsDisambiguation(0.3)).toBe(false);
+  });
+
+  it('buildDisambiguation produces sorted candidates', () => {
+    const nodeA = createCommandNode('toggle', { patient: createSelector('#button') });
+    const nodeB = createCommandNode('toggle', { patient: createSelector('.button') });
+
+    const result = buildDisambiguation('toggle button', [
+      { action: 'toggle', confidence: 0.55, node: nodeB },
+      { action: 'toggle', confidence: 0.65, node: nodeA },
+    ]);
+
+    expect(result.candidates[0].confidence).toBe(0.65);
+    expect(result.candidates[1].confidence).toBe(0.55);
+    expect(result.question).toContain('Which did you mean?');
+  });
+
+  // --- 3C: Pattern Hit-Rate Tracking ---
+
+  it('tracker records events and computes hit rates', () => {
+    const tracker = new PatternTracker();
+
+    tracker.record({
+      timestamp: Date.now(),
+      domain: 'flow',
+      action: 'fetch',
+      language: 'en',
+      inputFormat: 'explicit',
+      confidence: 0.95,
+      outcome: 'accepted',
+    });
+    tracker.record({
+      timestamp: Date.now(),
+      domain: 'flow',
+      action: 'fetch',
+      language: 'en',
+      inputFormat: 'explicit',
+      confidence: 0.4,
+      outcome: 'rejected',
+      diagnosticCodes: ['missing-role'],
+    });
+
+    const summary = tracker.getSummary();
+    expect(summary.totalEvents).toBe(2);
+    expect(summary.byCommand['fetch'].rate).toBe(0.5);
+    expect(summary.byOutcome['accepted']).toBe(1);
+    expect(summary.byOutcome['rejected']).toBe(1);
+    expect(summary.topFailures[0].code).toBe('missing-role');
+  });
+
+  it('tracker exports as JSONL', () => {
+    const tracker = new PatternTracker();
+    tracker.record({
+      timestamp: 1000,
+      domain: 'flow',
+      action: 'fetch',
+      language: 'en',
+      inputFormat: 'explicit',
+      confidence: 0.9,
+      outcome: 'accepted',
+    });
+
+    const jsonl = tracker.exportJSONL();
+    const parsed = JSON.parse(jsonl);
+    expect(parsed.action).toBe('fetch');
+  });
+});
+
+// =============================================================================
+// Pillar 4: Registry Convenience Methods
+// =============================================================================
+
+describe('Pillar 4: Registry Convenience Methods', () => {
+  let registry: DomainRegistry;
+
+  beforeAll(() => {
+    registry = new DomainRegistry();
+    registry.register({
+      name: 'test-domain',
+      description: 'Integration test domain',
+      languages: ['en'],
+      inputLabel: 'command',
+      inputDescription: 'Test command',
+      schemas: testSchemas,
+      getDSL: () => {
+        throw new Error('Not implemented for this test');
+      },
+    });
+  });
+
+  it('generatePrompt returns prompt for domain with schemas', () => {
+    const prompt = registry.generatePrompt('test-domain');
+    expect(prompt).not.toBeNull();
+    expect(prompt!.text).toContain('fetch');
+    expect(prompt!.text).toContain('toggle');
+    expect(prompt!.metadata.commandCount).toBe(2);
+  });
+
+  it('generatePrompt returns null for unknown domain', () => {
+    expect(registry.generatePrompt('nonexistent')).toBeNull();
+  });
+
+  it('generateTrainingData returns pairs for domain with schemas', () => {
+    const result = registry.generateTrainingData('test-domain');
+    expect(result).not.toBeNull();
+    expect(result!.pairs.length).toBeGreaterThan(0);
+    expect(result!.metadata.domain).toBe('test-domain');
+  });
+
+  it('generateTrainingData returns null for unknown domain', () => {
+    expect(registry.generateTrainingData('nonexistent')).toBeNull();
+  });
+
+  it('buildFeedback returns structured feedback', () => {
+    const feedback = registry.buildFeedback(
+      'test-domain',
+      '[fetch source:/api/users]',
+      'explicit',
+      [{ severity: 'error', message: 'Required role source missing', code: 'missing-role' }]
+    );
+
+    expect(feedback).not.toBeNull();
+    expect(feedback!.accepted).toBe(false);
+    expect(feedback!.diagnostics[0].fixType).toBe('missing_role');
+    // Schema info is included because action was detected from input
+    expect(feedback!.schema?.action).toBe('fetch');
+    expect(feedback!.hints.length).toBeGreaterThan(0);
+  });
+
+  it('buildFeedback returns null for unknown domain', () => {
+    expect(registry.buildFeedback('nonexistent', '[test]', 'explicit', [])).toBeNull();
+  });
+
+  it('getSchemas returns schemas for domain', () => {
+    const schemas = registry.getSchemas('test-domain');
+    expect(schemas).not.toBeNull();
+    expect(schemas!.length).toBe(2);
+  });
+
+  it('setSchemas attaches schemas post-registration', () => {
+    const reg = new DomainRegistry();
+    reg.register({
+      name: 'lazy',
+      description: 'Lazy schema domain',
+      languages: ['en'],
+      inputLabel: 'input',
+      inputDescription: 'Test input',
+      getDSL: () => {
+        throw new Error('Not implemented');
+      },
+    });
+
+    // Initially no schemas
+    expect(reg.getSchemas('lazy')).toBeNull();
+
+    // Attach schemas
+    reg.setSchemas('lazy', testSchemas);
+
+    expect(reg.getSchemas('lazy')!.length).toBe(2);
+    const prompt = reg.generatePrompt('lazy');
+    expect(prompt).not.toBeNull();
+  });
+});
+
+// =============================================================================
+// Cross-Cutting: Explicit Syntax ↔ JSON Round-trip
+// =============================================================================
+
+describe('Cross-Cutting: Explicit ↔ JSON Round-trip', () => {
+  it('node → explicit → parse → node preserves structure', () => {
+    const original = createCommandNode('fetch', {
+      source: createExpression('/api/users'),
+      style: createLiteral('json'),
+      destination: createSelector('#list'),
+    });
+
+    const explicit = renderExplicit(original);
+    const parsed = parseExplicit(explicit);
+
+    expect(parsed.action).toBe('fetch');
+    expect(parsed.roles.get('source')).toBeDefined();
+    expect(parsed.roles.get('style')).toBeDefined();
+    expect(parsed.roles.get('destination')).toBeDefined();
+  });
+
+  it('node → JSON → node preserves action and roles', () => {
+    const original = createCommandNode('toggle', {
+      patient: createSelector('.active'),
+    });
+
+    const json = semanticNodeToJSON(original);
+    expect(json.action).toBe('toggle');
+    expect(json.roles['patient']).toBeDefined();
+  });
+});

package/src/ir/conformance.test.ts

@@ -0,0 +1,120 @@
+/**
+ * Conformance Fixture Integration Tests
+ *
+ * Drives the protocol test fixtures through the bracket-syntax
+ * parseCompound/renderExplicit round-trip pipeline.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { parseCompound, parseDocument } from './explicit-parser';
+import { renderExplicit, renderDocument } from './explicit-renderer';
+import { fromProtocolJSON, toProtocolJSON } from './protocol-json';
+import type { CompoundSemanticNode } from '../core/types';
+
+import compoundFixtures from '../../../../protocol/test-fixtures/compound.json';
+import annotationFixtures from '../../../../protocol/test-fixtures/annotations.json';
+import versionFixtures from '../../../../protocol/test-fixtures/version-envelope.json';
+
+// =============================================================================
+// Compound Fixtures
+// =============================================================================
+
+describe('conformance: compound.json', () => {
+  for (const fixture of compoundFixtures) {
+    it(`${fixture.id}: ${fixture.description}`, () => {
+      // Verify rendering matches expected output
+      const node = fromProtocolJSON(fixture.node as any);
+      const rendered = renderExplicit(node);
+      expect(rendered).toBe(fixture.rendered);
+
+      // Now verify we can PARSE it back (previously renderOnly)
+      const reparsed = parseCompound(fixture.rendered);
+      expect(reparsed.kind).toBe('compound');
+      const compound = reparsed as CompoundSemanticNode;
+      expect(compound.chainType).toBe(fixture.node.chainType);
+      expect(compound.statements).toHaveLength(fixture.node.statements!.length);
+
+      for (let i = 0; i < compound.statements.length; i++) {
+        expect(compound.statements[i].action).toBe(fixture.node.statements![i].action);
+      }
+    });
+  }
+});
+
+// =============================================================================
+// Annotation Fixtures
+// =============================================================================
+
+describe('conformance: annotations.json', () => {
+  for (const fixture of annotationFixtures) {
+    it(`${fixture.id}: ${fixture.description}`, () => {
+      // Convert JSON → SemanticNode → render → parse → compare
+      const node = fromProtocolJSON(fixture.jsonInput as any);
+      const rendered = renderExplicit(node);
+      const reparsed = parseCompound(rendered);
+
+      // Verify action preserved
+      expect(reparsed.action).toBe(fixture.jsonInput.action);
+
+      // Verify annotations round-trip
+      if (fixture.jsonInput.annotations && fixture.jsonInput.annotations.length > 0) {
+        expect(reparsed.annotations).toHaveLength(fixture.jsonInput.annotations.length);
+        for (let i = 0; i < fixture.jsonInput.annotations.length; i++) {
+          expect(reparsed.annotations![i].name).toBe(fixture.jsonInput.annotations[i].name);
+          if (fixture.jsonInput.annotations[i].value !== undefined) {
+            expect(reparsed.annotations![i].value).toBe(fixture.jsonInput.annotations[i].value);
+          }
+        }
+      } else {
+        // ann-006: no annotations → should be undefined after parse
+        expect(reparsed.annotations).toBeUndefined();
+      }
+
+      // Verify re-render produces same output
+      const rerendered = renderExplicit(reparsed);
+      expect(rerendered).toBe(rendered);
+    });
+  }
+});
+
+// =============================================================================
+// Version Envelope Fixtures (streaming format)
+// =============================================================================
+
+describe('conformance: version-envelope.json (streaming)', () => {
+  for (const fixture of versionFixtures) {
+    const f = fixture as any;
+    if (!f.streamingInput) continue;
+
+    // ver-007 describes v1.0/v1.1 fallback behavior — our parser is v1.2,
+    // so we parse the version header correctly instead of treating it as a comment.
+    if (f.v1Fallback) {
+      it(`${f.id}: v1.2 parser parses version header — ${f.description}`, () => {
+        const envelope = parseDocument(f.streamingInput);
+        // v1.2 parser extracts the version (v1.0 parsers would ignore it)
+        expect(envelope.lseVersion).toBe('1.2');
+        expect(envelope.nodes).toHaveLength(f.v1Fallback.expectedNodeCount);
+      });
+      continue;
+    }
+
+    it(`${f.id}: ${f.description}`, () => {
+      const envelope = parseDocument(f.streamingInput);
+      expect(envelope.lseVersion).toBe(f.expectedVersion);
+      expect(envelope.nodes).toHaveLength(f.expectedNodeCount);
+    });
+  }
+
+  // Also test JSON-based envelope round-trips
+  for (const fixture of versionFixtures) {
+    const f = fixture as any;
+    if (f.streamingInput) continue;
+    if (!f.expectedRoundTrip) continue;
+
+    it(`${f.id}: JSON envelope round-trip — ${f.description}`, () => {
+      // For JSON envelopes: verify node count and version
+      expect(f.jsonInput.lseVersion).toBe(f.expectedVersion);
+      expect(f.jsonInput.nodes).toHaveLength(f.expectedNodeCount);
+    });
+  }
+});