@lokascript/framework 2.0.0 → 2.2.0

package/src/feedback/pattern-tracker.test.ts

@@ -0,0 +1,198 @@
+/**
+ * Tests for Pattern Hit-Rate Tracker
+ */
+
+import { describe, it, expect, beforeEach } from 'vitest';
+import { PatternTracker } from './pattern-tracker';
+import type { PatternEvent } from './types';
+
+// =============================================================================
+// Test Helpers
+// =============================================================================
+
+function makeEvent(overrides?: Partial<PatternEvent>): PatternEvent {
+  return {
+    timestamp: Date.now(),
+    domain: 'flow',
+    action: 'fetch',
+    language: 'en',
+    inputFormat: 'explicit',
+    confidence: 0.95,
+    outcome: 'accepted',
+    ...overrides,
+  };
+}
+
+// =============================================================================
+// PatternTracker
+// =============================================================================
+
+describe('PatternTracker', () => {
+  let tracker: PatternTracker;
+
+  beforeEach(() => {
+    tracker = new PatternTracker();
+  });
+
+  // --- Basic recording ---
+
+  it('starts empty', () => {
+    expect(tracker.size).toBe(0);
+  });
+
+  it('records events', () => {
+    tracker.record(makeEvent());
+    expect(tracker.size).toBe(1);
+  });
+
+  it('records multiple events', () => {
+    tracker.record(makeEvent());
+    tracker.record(makeEvent({ action: 'toggle' }));
+    tracker.record(makeEvent({ action: 'add' }));
+    expect(tracker.size).toBe(3);
+  });
+
+  // --- Ring buffer ---
+
+  it('enforces ring buffer limit', () => {
+    const small = new PatternTracker(5);
+    for (let i = 0; i < 10; i++) {
+      small.record(makeEvent({ action: `action-${i}` }));
+    }
+    expect(small.size).toBe(5);
+  });
+
+  it('keeps newest events when buffer overflows', () => {
+    const small = new PatternTracker(3);
+    for (let i = 0; i < 5; i++) {
+      small.record(makeEvent({ action: `action-${i}` }));
+    }
+    const summary = small.getSummary();
+    // Should only have action-2, action-3, action-4
+    expect(summary.byCommand['action-0']).toBeUndefined();
+    expect(summary.byCommand['action-1']).toBeUndefined();
+    expect(summary.byCommand['action-2']).toBeDefined();
+  });
+
+  // --- Hit rate by command ---
+
+  it('computes hit rate by command', () => {
+    tracker.record(makeEvent({ action: 'fetch', outcome: 'accepted' }));
+    tracker.record(makeEvent({ action: 'fetch', outcome: 'accepted' }));
+    tracker.record(makeEvent({ action: 'fetch', outcome: 'rejected' }));
+    tracker.record(makeEvent({ action: 'toggle', outcome: 'accepted' }));
+
+    const rates = tracker.getHitRateByCommand();
+    expect(rates['fetch'].attempts).toBe(3);
+    expect(rates['fetch'].successes).toBe(2);
+    expect(rates['fetch'].rate).toBeCloseTo(2 / 3);
+    expect(rates['toggle'].rate).toBe(1);
+  });
+
+  // --- Hit rate by language ---
+
+  it('computes hit rate by language', () => {
+    tracker.record(makeEvent({ language: 'en', outcome: 'accepted' }));
+    tracker.record(makeEvent({ language: 'en', outcome: 'rejected' }));
+    tracker.record(makeEvent({ language: 'ja', outcome: 'accepted' }));
+
+    const rates = tracker.getHitRateByLanguage();
+    expect(rates['en'].attempts).toBe(2);
+    expect(rates['en'].rate).toBe(0.5);
+    expect(rates['ja'].rate).toBe(1);
+  });
+
+  // --- Top failures ---
+
+  it('computes top failures by diagnostic code', () => {
+    tracker.record(makeEvent({ outcome: 'rejected', diagnosticCodes: ['missing-role'] }));
+    tracker.record(makeEvent({ outcome: 'rejected', diagnosticCodes: ['missing-role'] }));
+    tracker.record(makeEvent({ outcome: 'rejected', diagnosticCodes: ['unknown-command'] }));
+
+    const failures = tracker.getTopFailures();
+    expect(failures[0].code).toBe('missing-role');
+    expect(failures[0].count).toBe(2);
+    expect(failures[1].code).toBe('unknown-command');
+    expect(failures[1].count).toBe(1);
+  });
+
+  it('tracks which actions trigger each failure', () => {
+    tracker.record(
+      makeEvent({ action: 'fetch', outcome: 'rejected', diagnosticCodes: ['missing-role'] })
+    );
+    tracker.record(
+      makeEvent({ action: 'toggle', outcome: 'rejected', diagnosticCodes: ['missing-role'] })
+    );
+
+    const failures = tracker.getTopFailures();
+    expect(failures[0].actions).toContain('fetch');
+    expect(failures[0].actions).toContain('toggle');
+  });
+
+  it('limits top failures to requested count', () => {
+    for (let i = 0; i < 20; i++) {
+      tracker.record(makeEvent({ outcome: 'rejected', diagnosticCodes: [`code-${i}`] }));
+    }
+
+    const failures = tracker.getTopFailures(5);
+    expect(failures.length).toBe(5);
+  });
+
+  it('ignores non-rejected events for failures', () => {
+    tracker.record(makeEvent({ outcome: 'accepted', diagnosticCodes: ['some-code'] }));
+    const failures = tracker.getTopFailures();
+    expect(failures.length).toBe(0);
+  });
+
+  it('ignores rejected events without diagnostic codes', () => {
+    tracker.record(makeEvent({ outcome: 'rejected' }));
+    const failures = tracker.getTopFailures();
+    expect(failures.length).toBe(0);
+  });
+
+  // --- Summary ---
+
+  it('produces a complete summary', () => {
+    tracker.record(makeEvent({ outcome: 'accepted' }));
+    tracker.record(makeEvent({ outcome: 'rejected', diagnosticCodes: ['err'] }));
+    tracker.record(makeEvent({ outcome: 'disambiguated' }));
+
+    const summary = tracker.getSummary();
+    expect(summary.totalEvents).toBe(3);
+    expect(summary.byOutcome['accepted']).toBe(1);
+    expect(summary.byOutcome['rejected']).toBe(1);
+    expect(summary.byOutcome['disambiguated']).toBe(1);
+    expect(summary.byCommand).toBeDefined();
+    expect(summary.byLanguage).toBeDefined();
+    expect(summary.topFailures.length).toBe(1);
+  });
+
+  // --- JSONL export ---
+
+  it('exports events as JSONL', () => {
+    tracker.record(makeEvent({ action: 'fetch' }));
+    tracker.record(makeEvent({ action: 'toggle' }));
+
+    const jsonl = tracker.exportJSONL();
+    const lines = jsonl.split('\n');
+    expect(lines.length).toBe(2);
+
+    const parsed0 = JSON.parse(lines[0]);
+    expect(parsed0.action).toBe('fetch');
+    const parsed1 = JSON.parse(lines[1]);
+    expect(parsed1.action).toBe('toggle');
+  });
+
+  it('exports empty string when no events', () => {
+    expect(tracker.exportJSONL()).toBe('');
+  });
+
+  // --- Clear ---
+
+  it('clears all events', () => {
+    tracker.record(makeEvent());
+    tracker.record(makeEvent());
+    tracker.clear();
+    expect(tracker.size).toBe(0);
+  });
+});

package/src/feedback/pattern-tracker.ts

@@ -0,0 +1,140 @@
+/**
+ * Pattern Hit-Rate Tracker
+ *
+ * Records pattern events in a ring buffer and computes hit-rate
+ * statistics by command, language, and diagnostic code. Patterns
+ * with high rejected/attempts ratios indicate miscalibrated confidence.
+ */
+
+import type { PatternEvent, HitRate, PatternTrackerSummary, TopFailure } from './types';
+
+// =============================================================================
+// Public API
+// =============================================================================
+
+export class PatternTracker {
+  private events: PatternEvent[] = [];
+  private readonly maxEvents: number;
+
+  constructor(maxEvents: number = 1000) {
+    this.maxEvents = maxEvents;
+  }
+
+  /**
+   * Record a pattern event.
+   */
+  record(event: PatternEvent): void {
+    this.events.push(event);
+    // Ring buffer: drop oldest when full
+    if (this.events.length > this.maxEvents) {
+      this.events = this.events.slice(this.events.length - this.maxEvents);
+    }
+  }
+
+  /**
+   * Get hit rate by command action.
+   */
+  getHitRateByCommand(): Record<string, HitRate> {
+    return this.computeHitRate(e => e.action);
+  }
+
+  /**
+   * Get hit rate by language.
+   */
+  getHitRateByLanguage(): Record<string, HitRate> {
+    return this.computeHitRate(e => e.language);
+  }
+
+  /**
+   * Get the most frequent failure diagnostic codes.
+   */
+  getTopFailures(limit: number = 10): TopFailure[] {
+    const codeMap = new Map<string, { count: number; actions: Set<string> }>();
+
+    for (const event of this.events) {
+      if (event.outcome !== 'rejected' || !event.diagnosticCodes) continue;
+      for (const code of event.diagnosticCodes) {
+        const entry = codeMap.get(code) || { count: 0, actions: new Set() };
+        entry.count++;
+        entry.actions.add(event.action);
+        codeMap.set(code, entry);
+      }
+    }
+
+    return Array.from(codeMap.entries())
+      .map(([code, { count, actions }]) => ({
+        code,
+        count,
+        actions: Array.from(actions),
+      }))
+      .sort((a, b) => b.count - a.count)
+      .slice(0, limit);
+  }
+
+  /**
+   * Get a complete summary of tracking data.
+   */
+  getSummary(): PatternTrackerSummary {
+    const byOutcome: Record<string, number> = {};
+    for (const event of this.events) {
+      byOutcome[event.outcome] = (byOutcome[event.outcome] || 0) + 1;
+    }
+
+    return {
+      totalEvents: this.events.length,
+      byOutcome,
+      byCommand: this.getHitRateByCommand(),
+      byLanguage: this.getHitRateByLanguage(),
+      topFailures: this.getTopFailures(),
+    };
+  }
+
+  /**
+   * Export all events as JSONL.
+   */
+  exportJSONL(): string {
+    return this.events.map(e => JSON.stringify(e)).join('\n');
+  }
+
+  /**
+   * Get the number of recorded events.
+   */
+  get size(): number {
+    return this.events.length;
+  }
+
+  /**
+   * Clear all events.
+   */
+  clear(): void {
+    this.events = [];
+  }
+
+  // ---------------------------------------------------------------------------
+  // Private
+  // ---------------------------------------------------------------------------
+
+  private computeHitRate(keyFn: (e: PatternEvent) => string): Record<string, HitRate> {
+    const map = new Map<string, { attempts: number; successes: number }>();
+
+    for (const event of this.events) {
+      const key = keyFn(event);
+      const entry = map.get(key) || { attempts: 0, successes: 0 };
+      entry.attempts++;
+      if (event.outcome === 'accepted') {
+        entry.successes++;
+      }
+      map.set(key, entry);
+    }
+
+    const result: Record<string, HitRate> = {};
+    for (const [key, { attempts, successes }] of map) {
+      result[key] = {
+        attempts,
+        successes,
+        rate: attempts > 0 ? successes / attempts : 0,
+      };
+    }
+    return result;
+  }
+}

package/src/feedback/types.ts

@@ -0,0 +1,149 @@
+/**
+ * Feedback Loop Types
+ *
+ * Types for structured error feedback, confidence-aware disambiguation,
+ * and pattern hit-rate tracking in the LLM ↔ LSE integration.
+ */
+
+import type { SemanticJSON } from '../ir/types';
+
+// =============================================================================
+// Structured Error Feedback (Level 1)
+// =============================================================================
+
+/**
+ * Structured feedback for an LSE validation attempt.
+ */
+export interface LSEFeedback {
+  /** Whether the input was accepted (no errors) */
+  readonly accepted: boolean;
+
+  /** The original input */
+  readonly input: {
+    readonly format: 'explicit' | 'json' | 'natural';
+    readonly text: string;
+  };
+
+  /** Machine-actionable diagnostics */
+  readonly diagnostics: readonly FeedbackDiagnostic[];
+
+  /** Human-readable hints for retrying */
+  readonly hints: readonly string[];
+
+  /** Valid schema structure for the attempted command (if identifiable) */
+  readonly schema?: {
+    readonly action: string;
+    readonly requiredRoles: readonly string[];
+    readonly optionalRoles: readonly string[];
+    readonly roleDescriptions: Readonly<Record<string, string>>;
+  };
+
+  /** Corrected example (if fixable) */
+  readonly correctedExample?: {
+    readonly explicit: string;
+    readonly json: SemanticJSON;
+  };
+}
+
+/**
+ * A single diagnostic with machine-actionable fix type.
+ */
+export interface FeedbackDiagnostic {
+  readonly severity: 'error' | 'warning' | 'info';
+  readonly code: string;
+  readonly message: string;
+  readonly suggestion?: string;
+  readonly fixType?: FixType;
+}
+
+/**
+ * Machine-actionable fix type for LLM retry logic.
+ */
+export type FixType =
+  | 'missing_role'
+  | 'invalid_type'
+  | 'unknown_command'
+  | 'unknown_role'
+  | 'syntax_error';
+
+// =============================================================================
+// Confidence-Aware Disambiguation (Level 2)
+// =============================================================================
+
+/**
+ * A disambiguation request presented when confidence is borderline.
+ */
+export interface DisambiguationRequest {
+  /** The original input that was ambiguous */
+  readonly input: string;
+
+  /** Candidate interpretations ranked by confidence */
+  readonly candidates: readonly DisambiguationCandidate[];
+
+  /** Human/LLM-readable disambiguation prompt */
+  readonly question: string;
+}
+
+/**
+ * A single candidate interpretation.
+ */
+export interface DisambiguationCandidate {
+  /** Command action */
+  readonly action: string;
+
+  /** Confidence score */
+  readonly confidence: number;
+
+  /** Canonical LSE for this interpretation */
+  readonly explicit: string;
+
+  /** Human-readable description */
+  readonly description: string;
+}
+
+// =============================================================================
+// Pattern Hit-Rate Instrumentation (Level 3)
+// =============================================================================
+
+/**
+ * A single pattern event for hit-rate tracking.
+ */
+export interface PatternEvent {
+  readonly timestamp: number;
+  readonly domain: string;
+  readonly action: string;
+  readonly language: string;
+  readonly inputFormat: 'explicit' | 'json' | 'natural';
+  readonly confidence: number;
+  readonly outcome: 'accepted' | 'rejected' | 'disambiguated';
+  readonly diagnosticCodes?: readonly string[];
+}
+
+/**
+ * Hit-rate statistics for a category.
+ */
+export interface HitRate {
+  readonly attempts: number;
+  readonly successes: number;
+  readonly rate: number;
+}
+
+/**
+ * Summary of pattern tracking data.
+ */
+export interface PatternTrackerSummary {
+  readonly totalEvents: number;
+  readonly byOutcome: Readonly<Record<string, number>>;
+  readonly byCommand: Readonly<Record<string, HitRate>>;
+  readonly byLanguage: Readonly<Record<string, HitRate>>;
+  readonly topFailures: readonly TopFailure[];
+}
+
+/**
+ * A frequently failing diagnostic code.
+ */
+export interface TopFailure {
+  readonly code: string;
+  readonly count: number;
+  readonly actions: readonly string[];
+}

package/src/generation/pattern-generator.ts

@@ -165,9 +165,9 @@ function addRoleWithMarker(
 
   if (marker) {
     const markerInfo = profile.roleMarkers?.[roleSpec.role];
-    // Default: SOV languages use postpositions (after), others prepositions (before)
+    // Priority: schema markerPosition → profile roleMarkers position → word-order default
     const defaultPosition = profile.wordOrder === 'SOV' ? 'after' : 'before';
-    const position = markerInfo?.position ?? defaultPosition;
+    const position = roleSpec.markerPosition ?? markerInfo?.position ?? defaultPosition;
 
     if (isOptional) {
       // Optional role: wrap marker + role in optional group

package/src/index.ts

@@ -71,6 +71,12 @@ export type {
   TokenStream,
   LanguageTokenizer,
   LanguagePattern,
+  // v1.2 types
+  Annotation,
+  ProtocolDiagnostic,
+  AsyncVariant,
+  MatchArm,
+  LSEEnvelope,
 } from './core/types';
 
 export type { CommandSchema, RoleSpec } from './schema';
@@ -135,12 +141,35 @@ export {
   createLoopNode,
   extractValue,
   extractRoleValue,
+  // v1.2 factories
+  createTryNode,
+  createAsyncNode,
+  createMatchNode,
 } from './core/types';
 
 export { defineCommand, defineRole } from './schema';
 export { createSimpleTokenizer } from './core/tokenization/base-tokenizer';
 export type { SimpleTokenizerConfig } from './core/tokenization/base-tokenizer';
 
+// Morphological normalization base class and types
+export { BaseMorphologicalNormalizer } from './core/tokenization/morphology/base-normalizer';
+export type {
+  ConjugationEnding,
+  NormalizerConfig,
+} from './core/tokenization/morphology/base-normalizer';
+
+// IR (Intermediate Representation) — explicit syntax, JSON conversion, reference validation
+export * from './ir';
+
+// LLM Prompt Generation — auto-generate system prompts from CommandSchema[]
+export * from './prompts';
+
+// Training Data Generation — synthesize (natural_language, LSE) pairs from schemas
+export * from './training';
+
+// Feedback Loop — structured errors, disambiguation, pattern tracking
+export * from './feedback';
+
 // Multi-statement parser
 export { createMultiStatementParser, accumulateBlocks } from './parsing/multi-statement';
 export type {