@lokascript/framework 2.0.0 → 2.1.0

package/src/training/jsonl-writer.test.ts

@@ -0,0 +1,138 @@
+/**
+ * Tests for JSONL Writer
+ */
+
+import { describe, it, expect } from 'vitest';
+import { toJSONL, toJSONLRow, parseJSONL } from './jsonl-writer';
+import type { TrainingPair } from './types';
+import type { SemanticJSON } from '../ir/types';
+
+// =============================================================================
+// Test Data
+// =============================================================================
+
+function makePair(overrides?: Partial<TrainingPair>): TrainingPair {
+  return {
+    id: 'test-001',
+    natural: 'fetch /api/users as json into #list',
+    language: 'en',
+    explicit: '[fetch source:/api/users style:json destination:#list]',
+    json: {
+      action: 'fetch',
+      roles: {
+        source: { type: 'expression', value: '/api/users' },
+        style: { type: 'expression', value: 'json' },
+        destination: { type: 'selector', value: '#list' },
+      },
+    } as SemanticJSON,
+    domain: 'flow',
+    action: 'fetch',
+    source: 'synthetic',
+    quality: 1.0,
+    ...overrides,
+  };
+}
+
+// =============================================================================
+// toJSONLRow()
+// =============================================================================
+
+describe('toJSONLRow', () => {
+  it('converts pair to JSONL row', () => {
+    const row = toJSONLRow(makePair());
+    expect(row.id).toBe('test-001');
+    expect(row.prompt).toBe('fetch /api/users as json into #list');
+    expect(row.completion).toBe('[fetch source:/api/users style:json destination:#list]');
+  });
+
+  it('includes metadata', () => {
+    const row = toJSONLRow(makePair());
+    expect(row.metadata.domain).toBe('flow');
+    expect(row.metadata.action).toBe('fetch');
+    expect(row.metadata.language).toBe('en');
+    expect(row.metadata.source).toBe('synthetic');
+    expect(row.metadata.quality).toBe(1.0);
+  });
+
+  it('includes confidence when present', () => {
+    const row = toJSONLRow(makePair({ confidence: 0.95 }));
+    expect(row.metadata.confidence).toBe(0.95);
+  });
+
+  it('omits confidence when absent', () => {
+    const row = toJSONLRow(makePair());
+    expect(row.metadata.confidence).toBeUndefined();
+  });
+});
+
+// =============================================================================
+// toJSONL()
+// =============================================================================
+
+describe('toJSONL', () => {
+  it('produces one line per pair', () => {
+    const pairs = [makePair({ id: 'a' }), makePair({ id: 'b' }), makePair({ id: 'c' })];
+    const jsonl = toJSONL(pairs);
+    const lines = jsonl.split('\n');
+    expect(lines.length).toBe(3);
+  });
+
+  it('produces valid JSON on each line', () => {
+    const pairs = [makePair({ id: 'a' }), makePair({ id: 'b' })];
+    const jsonl = toJSONL(pairs);
+    for (const line of jsonl.split('\n')) {
+      expect(() => JSON.parse(line)).not.toThrow();
+    }
+  });
+
+  it('returns empty string for empty pairs', () => {
+    const jsonl = toJSONL([]);
+    expect(jsonl).toBe('');
+  });
+
+  it('preserves pair data through serialization', () => {
+    const pair = makePair();
+    const jsonl = toJSONL([pair]);
+    const parsed = JSON.parse(jsonl);
+    expect(parsed.id).toBe(pair.id);
+    expect(parsed.prompt).toBe(pair.natural);
+    expect(parsed.completion).toBe(pair.explicit);
+  });
+});
+
+// =============================================================================
+// parseJSONL()
+// =============================================================================
+
+describe('parseJSONL', () => {
+  it('roundtrips through toJSONL and parseJSONL', () => {
+    const pairs = [makePair({ id: 'a' }), makePair({ id: 'b' })];
+    const jsonl = toJSONL(pairs);
+    const rows = parseJSONL(jsonl);
+    expect(rows.length).toBe(2);
+    expect(rows[0].id).toBe('a');
+    expect(rows[1].id).toBe('b');
+  });
+
+  it('skips empty lines', () => {
+    const jsonl =
+      '{"id":"a","prompt":"x","completion":"y","metadata":{}}\n\n{"id":"b","prompt":"x","completion":"y","metadata":{}}';
+    const rows = parseJSONL(jsonl);
+    expect(rows.length).toBe(2);
+  });
+
+  it('handles single line', () => {
+    const jsonl = '{"id":"only","prompt":"x","completion":"y","metadata":{}}';
+    const rows = parseJSONL(jsonl);
+    expect(rows.length).toBe(1);
+    expect(rows[0].id).toBe('only');
+  });
+
+  it('preserves metadata through roundtrip', () => {
+    const pair = makePair({ confidence: 0.85 });
+    const jsonl = toJSONL([pair]);
+    const rows = parseJSONL(jsonl);
+    expect(rows[0].metadata.domain).toBe('flow');
+    expect(rows[0].metadata.confidence).toBe(0.85);
+  });
+});

package/src/training/jsonl-writer.ts

@@ -0,0 +1,67 @@
+/**
+ * JSONL Writer
+ *
+ * Exports training pairs in JSONL (JSON Lines) format,
+ * compatible with standard fine-tuning pipelines.
+ */
+
+import type { TrainingPair } from './types';
+
+/**
+ * Convert training pairs to JSONL format (one JSON object per line).
+ *
+ * Each line contains:
+ * - id: unique pair identifier
+ * - prompt: the natural language input
+ * - completion: the LSE bracket syntax output
+ * - metadata: domain, action, language, source, quality
+ */
+export function toJSONL(pairs: readonly TrainingPair[]): string {
+  return pairs.map(pair => JSON.stringify(toJSONLRow(pair))).join('\n');
+}
+
+/**
+ * Convert a single training pair to a JSONL row.
+ */
+export function toJSONLRow(pair: TrainingPair): JSONLRow {
+  return {
+    id: pair.id,
+    prompt: pair.natural,
+    completion: pair.explicit,
+    metadata: {
+      domain: pair.domain,
+      action: pair.action,
+      language: pair.language,
+      source: pair.source,
+      quality: pair.quality,
+      ...(pair.confidence !== undefined && { confidence: pair.confidence }),
+    },
+  };
+}
+
+/**
+ * Parse JSONL back into rows (for validation/roundtrip testing).
+ */
+export function parseJSONL(text: string): JSONLRow[] {
+  return text
+    .split('\n')
+    .filter(line => line.trim().length > 0)
+    .map(line => JSON.parse(line) as JSONLRow);
+}
+
+/**
+ * A single row in JSONL format.
+ */
+export interface JSONLRow {
+  readonly id: string;
+  readonly prompt: string;
+  readonly completion: string;
+  readonly metadata: {
+    readonly domain: string;
+    readonly action: string;
+    readonly language: string;
+    readonly source: string;
+    readonly quality: number;
+    readonly confidence?: number;
+  };
+}

package/src/training/schema-synthesizer.test.ts

@@ -0,0 +1,250 @@
+/**
+ * Tests for Schema Synthesizer
+ */
+
+import { describe, it, expect } from 'vitest';
+import { synthesizeFromSchemas } from './schema-synthesizer';
+import type { SynthesisConfig } from './types';
+import { defineCommand, defineRole } from '../schema/command-schema';
+import type { CommandSchema } from '../schema/command-schema';
+
+// =============================================================================
+// Test Schemas
+// =============================================================================
+
+const fetchSchema: CommandSchema = defineCommand({
+  action: 'fetch',
+  description: 'Fetch data from a URL',
+  category: 'source',
+  primaryRole: 'source',
+  roles: [
+    defineRole({
+      role: 'source',
+      description: 'URL to fetch from',
+      required: true,
+      expectedTypes: ['expression'],
+    }),
+    defineRole({
+      role: 'style',
+      description: 'Response format',
+      required: false,
+      expectedTypes: ['expression'],
+      markerOverride: { en: 'as', es: 'como' },
+    }),
+    defineRole({
+      role: 'destination',
+      description: 'Target element',
+      required: false,
+      expectedTypes: ['selector', 'expression'],
+      markerOverride: { en: 'into', es: 'en' },
+    }),
+  ],
+});
+
+const toggleSchema: CommandSchema = defineCommand({
+  action: 'toggle',
+  description: 'Toggle a class',
+  category: 'dom',
+  primaryRole: 'patient',
+  roles: [
+    defineRole({
+      role: 'patient',
+      description: 'CSS selector',
+      required: true,
+      expectedTypes: ['selector'],
+    }),
+  ],
+});
+
+const submitSchema: CommandSchema = defineCommand({
+  action: 'submit',
+  description: 'Submit form data',
+  category: 'action',
+  primaryRole: 'patient',
+  roles: [
+    defineRole({
+      role: 'patient',
+      description: 'Form element',
+      required: true,
+      expectedTypes: ['selector'],
+    }),
+    defineRole({
+      role: 'destination',
+      description: 'URL to submit to',
+      required: true,
+      expectedTypes: ['expression'],
+      markerOverride: { en: 'to', es: 'a' },
+    }),
+  ],
+});
+
+const testSchemas = [fetchSchema, toggleSchema, submitSchema];
+
+function makeConfig(overrides?: Partial<SynthesisConfig>): SynthesisConfig {
+  return {
+    domain: 'test',
+    ...overrides,
+  };
+}
+
+// =============================================================================
+// synthesizeFromSchemas()
+// =============================================================================
+
+describe('synthesizeFromSchemas', () => {
+  it('generates pairs for each schema', () => {
+    const result = synthesizeFromSchemas(testSchemas, makeConfig());
+    expect(result.pairs.length).toBeGreaterThan(0);
+    expect(result.metadata.commandCount).toBe(3);
+  });
+
+  it('includes metadata', () => {
+    const result = synthesizeFromSchemas(testSchemas, makeConfig());
+    expect(result.metadata.domain).toBe('test');
+    expect(result.metadata.pairCount).toBe(result.pairs.length);
+    expect(result.metadata.bySource.synthetic).toBe(result.pairs.length);
+  });
+
+  it('generates valid bracket syntax', () => {
+    const result = synthesizeFromSchemas(testSchemas, makeConfig());
+    for (const pair of result.pairs) {
+      expect(pair.explicit).toMatch(/^\[/);
+      expect(pair.explicit).toMatch(/\]$/);
+    }
+  });
+
+  it('generates valid JSON', () => {
+    const result = synthesizeFromSchemas(testSchemas, makeConfig());
+    for (const pair of result.pairs) {
+      expect(pair.json.action).toBeTruthy();
+      expect(typeof pair.json.roles).toBe('object');
+    }
+  });
+
+  it('includes required roles in every pair', () => {
+    const result = synthesizeFromSchemas([fetchSchema], makeConfig());
+    for (const pair of result.pairs) {
+      expect(pair.json.roles['source']).toBeDefined();
+    }
+  });
+
+  it('generates natural language form', () => {
+    const result = synthesizeFromSchemas([fetchSchema], makeConfig());
+    for (const pair of result.pairs) {
+      expect(pair.natural).toContain('fetch');
+      expect(pair.natural.length).toBeGreaterThan(0);
+    }
+  });
+
+  it('uses markers in natural language', () => {
+    const result = synthesizeFromSchemas([fetchSchema], makeConfig({ languages: ['en'] }));
+    // Find a pair with the style role
+    const withStyle = result.pairs.find(p => p.json.roles['style']);
+    if (withStyle) {
+      expect(withStyle.natural).toContain('as');
+    }
+  });
+
+  // --- Configuration ---
+
+  it('respects maxPairsPerCommand', () => {
+    const result = synthesizeFromSchemas([fetchSchema], makeConfig({ maxPairsPerCommand: 2 }));
+    expect(result.pairs.length).toBeLessThanOrEqual(2);
+  });
+
+  it('respects minQuality filter', () => {
+    const result = synthesizeFromSchemas(testSchemas, makeConfig({ minQuality: 0.9 }));
+    for (const pair of result.pairs) {
+      expect(pair.quality).toBeGreaterThanOrEqual(0.9);
+    }
+  });
+
+  it('generates for multiple languages', () => {
+    const result = synthesizeFromSchemas([toggleSchema], makeConfig({ languages: ['en', 'es'] }));
+    const languages = new Set(result.pairs.map(p => p.language));
+    expect(languages.has('en')).toBe(true);
+    expect(languages.has('es')).toBe(true);
+  });
+
+  it('generates shuffled role order variants', () => {
+    const result = synthesizeFromSchemas([submitSchema], makeConfig({ shuffleRoles: true }));
+    // submit has 2 required roles, so shuffled variant should exist
+    expect(result.pairs.length).toBeGreaterThanOrEqual(2);
+  });
+
+  it('disables shuffling when configured', () => {
+    const withShuffle = synthesizeFromSchemas(
+      [submitSchema],
+      makeConfig({ shuffleRoles: true, maxPairsPerCommand: 10 })
+    );
+    const noShuffle = synthesizeFromSchemas(
+      [submitSchema],
+      makeConfig({ shuffleRoles: false, maxPairsPerCommand: 10 })
+    );
+    expect(noShuffle.pairs.length).toBeLessThanOrEqual(withShuffle.pairs.length);
+  });
+
+  // --- Pair Quality ---
+
+  it('assigns quality based on role coverage', () => {
+    const result = synthesizeFromSchemas([fetchSchema], makeConfig());
+    for (const pair of result.pairs) {
+      expect(pair.quality).toBeGreaterThanOrEqual(0);
+      expect(pair.quality).toBeLessThanOrEqual(1);
+    }
+  });
+
+  it('gives quality 1.0 when all required roles present', () => {
+    const result = synthesizeFromSchemas([toggleSchema], makeConfig());
+    // toggle has 1 required role, always filled
+    expect(result.pairs[0].quality).toBe(1);
+  });
+
+  // --- Pair Metadata ---
+
+  it('tags pairs as synthetic source', () => {
+    const result = synthesizeFromSchemas(testSchemas, makeConfig());
+    for (const pair of result.pairs) {
+      expect(pair.source).toBe('synthetic');
+    }
+  });
+
+  it('assigns unique IDs', () => {
+    const result = synthesizeFromSchemas(testSchemas, makeConfig());
+    const ids = new Set(result.pairs.map(p => p.id));
+    expect(ids.size).toBe(result.pairs.length);
+  });
+
+  it('includes domain in each pair', () => {
+    const result = synthesizeFromSchemas(testSchemas, makeConfig());
+    for (const pair of result.pairs) {
+      expect(pair.domain).toBe('test');
+    }
+  });
+
+  it('includes action in each pair', () => {
+    const result = synthesizeFromSchemas(testSchemas, makeConfig());
+    const actions = new Set(result.pairs.map(p => p.action));
+    expect(actions.has('fetch')).toBe(true);
+    expect(actions.has('toggle')).toBe(true);
+    expect(actions.has('submit')).toBe(true);
+  });
+
+  // --- Edge Cases ---
+
+  it('handles empty schemas', () => {
+    const result = synthesizeFromSchemas([], makeConfig());
+    expect(result.pairs.length).toBe(0);
+    expect(result.metadata.commandCount).toBe(0);
+  });
+
+  it('handles schema with no roles', () => {
+    const noRoleSchema = defineCommand({
+      action: 'noop',
+      roles: [],
+      primaryRole: 'patient',
+    });
+    const result = synthesizeFromSchemas([noRoleSchema], makeConfig());
+    expect(result.pairs.length).toBeGreaterThanOrEqual(0);
+  });
+});