@symerian/symi 3.0.17 → 3.0.19

package/extensions/learning-loop/src/test/integration.test.ts

@@ -1,312 +0,0 @@
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { createMetricsAggregator } from "../analytics/metrics-aggregator.js";
-import { createRunTracker } from "../capture/run-tracker.js";
-import { normalizeCompletedRun } from "../capture/serializer.js";
-import { createDatabaseManager } from "../db.js";
-import { createExplicitFeedbackHandler } from "../feedback/explicit-feedback.js";
-import { createImplicitSignals } from "../feedback/implicit-signals.js";
-import { createContextInjector } from "../injection/context-injector.js";
-import { createLearningExtractor } from "../learning/learning-extractor.js";
-import { createLearningStore } from "../learning/learning-store.js";
-import { createQualityEngine } from "../scoring/quality-engine.js";
-import type { LearningLoopConfig } from "../types.js";
-
-const CONFIG: LearningLoopConfig = {
-  capture: { embedPrompts: false, maxRuns: 1000 },
-  scoring: {
-    weights: {
-      taskCompletion: 0.35,
-      toolEfficiency: 0.25,
-      responseAppropriateLength: 0.1,
-      latencyRelative: 0.1,
-      userFeedback: 0.2,
-    },
-  },
-  injection: { maxLearnings: 5, minRelevance: 0.1, maxTokens: 500, cacheTtlMs: 60000 },
-  decay: { halfLifeDays: 30 },
-};
-
-const logger = {
-  info: () => {},
-  warn: () => {},
-  error: () => {},
-  debug: () => {},
-};
-
-describe("Integration: Full Lifecycle", () => {
-  let tmpDir: string;
-  let db: ReturnType<typeof createDatabaseManager>;
-
-  beforeEach(() => {
-    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "learning-loop-integration-"));
-    db = createDatabaseManager({ stateDir: tmpDir, config: CONFIG, logger });
-  });
-
-  afterEach(() => {
-    db.close();
-    fs.rmSync(tmpDir, { recursive: true, force: true });
-  });
-
-  it("should capture, score, and persist a full run lifecycle", () => {
-    const tracker = createRunTracker();
-    const qualityEngine = createQualityEngine({ config: CONFIG, db });
-    const learningStore = createLearningStore({ db });
-    const metrics = createMetricsAggregator({ db });
-
-    // Simulate llm_input
-    tracker.onLlmInput({
-      runId: "run-integration-1",
-      sessionId: "sess-1",
-      provider: "openai",
-      model: "gpt-4",
-      prompt: "Write a function to sort an array",
-      sessionKey: "sk-1",
-      agentId: "agent-1",
-    });
-
-    // Simulate after_tool_call
-    tracker.onToolCall({
-      toolName: "code_search",
-      durationMs: 50,
-      success: true,
-      paramHash: "h1",
-      sessionKey: "sk-1",
-    });
-    tracker.onToolCall({
-      toolName: "file_write",
-      durationMs: 30,
-      success: true,
-      paramHash: "h2",
-      sessionKey: "sk-1",
-    });
-
-    // Simulate llm_output
-    tracker.onLlmOutput({
-      runId: "run-integration-1",
-      sessionId: "sess-1",
-      assistantTexts: ["Here is a sort function implementation..."],
-      usage: { input: 100, output: 200, total: 300 },
-    });
-
-    // Simulate agent_end
-    const completedRaw = tracker.finalize({
-      sessionKey: "sk-1",
-      agentId: "agent-1",
-      sessionId: "sess-1",
-      success: true,
-      durationMs: 1500,
-    });
-
-    expect(completedRaw).not.toBeNull();
-    const completed = normalizeCompletedRun(completedRaw!);
-
-    // Score the run
-    const score = qualityEngine.scoreRun(completed);
-    expect(score.score).toBeGreaterThan(0);
-    expect(score.score).toBeLessThanOrEqual(1);
-    expect(score.signals).toHaveLength(5);
-
-    // Persist
-    db.insertRun(completed, score);
-    metrics.recordRun(completed, score);
-
-    // Verify persistence
-    const savedRun = db.getRun("run-integration-1");
-    expect(savedRun).toBeDefined();
-    expect(savedRun!.provider).toBe("openai");
-    expect(savedRun!.quality_score).toBeCloseTo(score.score, 5);
-
-    const savedToolCalls = db.getToolCalls("run-integration-1");
-    expect(savedToolCalls).toHaveLength(2);
-
-    // Verify metrics
-    const leaderboard = db.getModelLeaderboard();
-    expect(leaderboard.length).toBeGreaterThan(0);
-    expect(leaderboard[0]!.provider).toBe("openai");
-    expect(leaderboard[0]!.runCount).toBe(1);
-
-    // Verify stats
-    const stats = db.getStats();
-    expect(stats.runCount).toBe(1);
-  });
-
-  it("should handle explicit feedback submission", () => {
-    const tracker = createRunTracker();
-    const qualityEngine = createQualityEngine({ config: CONFIG, db });
-    const feedbackHandler = createExplicitFeedbackHandler({ db });
-
-    // Create a run first
-    tracker.onLlmInput({
-      runId: "run-fb-1",
-      sessionId: "sess-1",
-      provider: "claude",
-      model: "claude-3",
-      prompt: "Test prompt",
-      sessionKey: "sk-1",
-    });
-    tracker.onLlmOutput({
-      runId: "run-fb-1",
-      sessionId: "sess-1",
-      assistantTexts: ["Response"],
-      usage: { input: 10, output: 20, total: 30 },
-    });
-
-    const completed = normalizeCompletedRun(
-      tracker.finalize({ sessionKey: "sk-1", success: true, durationMs: 500 })!,
-    );
-    const score = qualityEngine.scoreRun(completed);
-    db.insertRun(completed, score);
-
-    // Submit feedback
-    const feedback = feedbackHandler.submitFeedback("run-fb-1", 5);
-    expect(feedback).not.toBeNull();
-    expect(feedback!.source).toBe("explicit");
-    expect(feedback!.score).toBe(5);
-
-    // Verify feedback persisted
-    const savedFeedback = db.getFeedback("run-fb-1");
-    expect(savedFeedback).toHaveLength(1);
-  });
-
-  it("should detect implicit feedback signals", () => {
-    const implicit = createImplicitSignals({ db, config: CONFIG });
-    const tracker = createRunTracker();
-    const qualityEngine = createQualityEngine({ config: CONFIG, db });
-
-    // Create and persist a run
-    tracker.onLlmInput({
-      runId: "run-impl-1",
-      sessionId: "sess-1",
-      provider: "openai",
-      model: "gpt-4",
-      prompt: "How do I sort an array in JavaScript?",
-      sessionKey: "sk-1",
-    });
-    tracker.onLlmOutput({
-      runId: "run-impl-1",
-      sessionId: "sess-1",
-      assistantTexts: ["Use Array.sort()"],
-      usage: { input: 10, output: 20, total: 30 },
-    });
-    const completed = normalizeCompletedRun(
-      tracker.finalize({ sessionKey: "sk-1", success: true, durationMs: 200 })!,
-    );
-    const score = qualityEngine.scoreRun(completed);
-    db.insertRun(completed, score);
-
-    // Record the prompt for implicit tracking
-    implicit.recordPrompt("run-impl-1", "How do I sort an array in JavaScript?");
-
-    // Simulate topic change (positive signal)
-    const feedback = implicit.analyzeMessage("Now let me work on the database schema", Date.now());
-    expect(feedback).not.toBeNull();
-    expect(feedback!.score).toBe(1.0); // topic change = positive
-  });
-
-  it("should prune old runs when exceeding maxRuns", () => {
-    const qualityEngine = createQualityEngine({ config: CONFIG, db });
-
-    // Insert runs
-    for (let i = 0; i < 5; i++) {
-      const run = {
-        runId: `prune-run-${i}`,
-        sessionId: "sess-1",
-        sessionKey: "sk-1",
-        agentId: "agent-1",
-        provider: "openai",
-        model: "gpt-4",
-        promptHash: `hash-${i}`,
-        promptLength: 100,
-        responseLength: 200,
-        responseToolCallCount: 0,
-        usage: { input: 50, output: 100, total: 150 },
-        toolCalls: [],
-        success: true,
-        error: null,
-        durationMs: 1000,
-        startedAt: Date.now() - 5000 + i * 1000,
-        completedAt: Date.now() - 4000 + i * 1000,
-      };
-      const score = qualityEngine.scoreRun(run);
-      db.insertRun(run, score);
-    }
-
-    expect(db.getRunCount()).toBe(5);
-
-    // Prune to max 3
-    const pruned = db.pruneOldRuns(3);
-    expect(pruned).toBe(2);
-    expect(db.getRunCount()).toBe(3);
-  });
-
-  it("should track EWMA state across runs", () => {
-    // Update EWMA for latency
-    const state1 = db.updateEwmaState("openai", "gpt-4", "latency", 1000, 0.1);
-    expect(state1.value).toBe(1000);
-    expect(state1.count).toBe(1);
-
-    const state2 = db.updateEwmaState("openai", "gpt-4", "latency", 2000, 0.1);
-    // 0.1 * 2000 + 0.9 * 1000 = 1100
-    expect(state2.value).toBeCloseTo(1100, 5);
-    expect(state2.count).toBe(2);
-
-    // Verify persistence
-    const loaded = db.getEwmaState("openai", "gpt-4", "latency");
-    expect(loaded.value).toBeCloseTo(1100, 5);
-  });
-
-  it("should track metrics across multiple models", () => {
-    const qualityEngine = createQualityEngine({ config: CONFIG, db });
-    const metrics = createMetricsAggregator({ db });
-
-    // Run with model A
-    const runA = {
-      runId: "model-a-1",
-      sessionId: "sess-1",
-      sessionKey: "sk-1",
-      agentId: "agent-1",
-      provider: "openai",
-      model: "gpt-4",
-      promptHash: "hA",
-      promptLength: 100,
-      responseLength: 200,
-      responseToolCallCount: 0,
-      usage: { input: 50, output: 100, total: 150 },
-      toolCalls: [],
-      success: true,
-      error: null,
-      durationMs: 1000,
-      startedAt: Date.now() - 1000,
-      completedAt: Date.now(),
-    };
-
-    // Run with model B
-    const runB = {
-      ...runA,
-      runId: "model-b-1",
-      provider: "claude",
-      model: "claude-3",
-      promptHash: "hB",
-      durationMs: 500,
-    };
-
-    const scoreA = qualityEngine.scoreRun(runA);
-    db.insertRun(runA, scoreA);
-    metrics.recordRun(runA, scoreA);
-
-    const scoreB = qualityEngine.scoreRun(runB);
-    db.insertRun(runB, scoreB);
-    metrics.recordRun(runB, scoreB);
-
-    // Verify leaderboard has both models
-    const leaderboard = metrics.getLeaderboard();
-    expect(leaderboard).toHaveLength(2);
-
-    // Verify model-specific metrics
-    const modelMetrics = metrics.getModelMetrics();
-    expect(modelMetrics).toHaveLength(2);
-  });
-});

package/extensions/learning-loop/src/test/learning-store.test.ts

@@ -1,191 +0,0 @@
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { createDatabaseManager } from "../db.js";
-import { createLearningStore } from "../learning/learning-store.js";
-import type { LearningLoopConfig, CompletedRun, QualityScore } from "../types.js";
-
-const TEST_CONFIG: LearningLoopConfig = {
-  capture: { embedPrompts: false, maxRuns: 1000 },
-  scoring: {
-    weights: {
-      taskCompletion: 0.35,
-      toolEfficiency: 0.25,
-      responseAppropriateLength: 0.1,
-      latencyRelative: 0.1,
-      userFeedback: 0.2,
-    },
-  },
-  injection: { maxLearnings: 5, minRelevance: 0.3, maxTokens: 500, cacheTtlMs: 60000 },
-  decay: { halfLifeDays: 30 },
-};
-
-const logger = {
-  info: () => {},
-  warn: () => {},
-  error: () => {},
-};
-
-function makeRun(id: string): CompletedRun {
-  return {
-    runId: id,
-    sessionId: "sess-1",
-    sessionKey: "sk-1",
-    agentId: "agent-1",
-    provider: "openai",
-    model: "gpt-4",
-    promptHash: "hash-" + id,
-    promptLength: 100,
-    responseLength: 200,
-    responseToolCallCount: 0,
-    usage: { input: 50, output: 100, total: 150 },
-    toolCalls: [],
-    success: true,
-    error: null,
-    durationMs: 1000,
-    startedAt: Date.now() - 1000,
-    completedAt: Date.now(),
-  };
-}
-
-const testScore: QualityScore = {
-  score: 0.85,
-  signals: [],
-  algorithmVersion: 1,
-};
-
-describe("LearningStore", () => {
-  let tmpDir: string;
-  let db: ReturnType<typeof createDatabaseManager>;
-  let store: ReturnType<typeof createLearningStore>;
-
-  beforeEach(() => {
-    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "learning-loop-test-"));
-    db = createDatabaseManager({ stateDir: tmpDir, config: TEST_CONFIG, logger });
-    store = createLearningStore({ db });
-
-    // Insert a run for foreign key constraint
-    db.insertRun(makeRun("run-1"), testScore);
-  });
-
-  afterEach(() => {
-    db.close();
-    fs.rmSync(tmpDir, { recursive: true, force: true });
-  });
-
-  it("should add and retrieve a learning", () => {
-    const id = store.addLearning({
-      runId: "run-1",
-      category: "tool_pattern",
-      content: "Use search before read for better results",
-      embedding: null,
-      confidence: 0.8,
-    });
-
-    expect(id).not.toBeNull();
-
-    const learning = store.getLearning(id!);
-    expect(learning).not.toBeNull();
-    expect(learning!.category).toBe("tool_pattern");
-    expect(learning!.content).toContain("search before read");
-  });
-
-  it("should deduplicate identical content", () => {
-    const id1 = store.addLearning({
-      runId: "run-1",
-      category: "tool_pattern",
-      content: "Exact duplicate content",
-      embedding: null,
-      confidence: 0.8,
-    });
-
-    const id2 = store.addLearning({
-      runId: "run-1",
-      category: "tool_pattern",
-      content: "Exact duplicate content",
-      embedding: null,
-      confidence: 0.8,
-    });
-
-    expect(id1).not.toBeNull();
-    expect(id2).toBeNull();
-  });
-
-  it("should list learnings by category", () => {
-    store.addLearning({
-      runId: "run-1",
-      category: "tool_pattern",
-      content: "Pattern A",
-      embedding: null,
-      confidence: 0.8,
-    });
-    store.addLearning({
-      runId: "run-1",
-      category: "error_recovery",
-      content: "Recovery B",
-      embedding: null,
-      confidence: 0.7,
-    });
-
-    const patterns = store.listLearnings({ category: "tool_pattern" });
-    expect(patterns).toHaveLength(1);
-    expect(patterns[0]!.category).toBe("tool_pattern");
-
-    const all = store.listLearnings();
-    expect(all).toHaveLength(2);
-  });
-
-  it("should increment applied count", () => {
-    const id = store.addLearning({
-      runId: "run-1",
-      category: "tool_pattern",
-      content: "Test learning",
-      embedding: null,
-      confidence: 0.8,
-    })!;
-
-    store.recordApplication(id);
-    store.recordApplication(id);
-
-    const learning = store.getLearning(id);
-    expect(learning!.appliedCount).toBe(2);
-  });
-
-  it("should remove learnings", () => {
-    const id = store.addLearning({
-      runId: "run-1",
-      category: "anti_pattern",
-      content: "Bad pattern to remove",
-      embedding: null,
-      confidence: 0.5,
-    })!;
-
-    store.removeLearning(id);
-    expect(store.getLearning(id)).toBeNull();
-  });
-
-  it("should deduplicate by cosine similarity when embeddings provided", () => {
-    const embedding1 = [0.1, 0.2, 0.3, 0.4, 0.5];
-    const embedding2 = [0.1, 0.2, 0.3, 0.4, 0.5]; // identical = cosine 1.0
-
-    const id1 = store.addLearning({
-      runId: "run-1",
-      category: "tool_pattern",
-      content: "First version of similar learning",
-      embedding: embedding1,
-      confidence: 0.8,
-    });
-
-    const id2 = store.addLearning({
-      runId: "run-1",
-      category: "tool_pattern",
-      content: "Slightly different but same embedding",
-      embedding: embedding2,
-      confidence: 0.8,
-    });
-
-    expect(id1).not.toBeNull();
-    expect(id2).toBeNull(); // deduplicated
-  });
-});

package/extensions/learning-loop/src/test/math.test.ts

@@ -1,148 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { wilsonScoreLowerBound, wilsonScoreInterval } from "../math/confidence-intervals.js";
-import { createEWMA, updateEWMA, getEWMAValue } from "../math/ewma.js";
-import { computeWeightedScore } from "../math/weighted-scorer.js";
-
-describe("EWMA", () => {
-  it("should initialize with zero value and count", () => {
-    const state = createEWMA();
-    expect(state.value).toBe(0);
-    expect(state.count).toBe(0);
-  });
-
-  it("should return fallback when no observations exist", () => {
-    const state = createEWMA();
-    expect(getEWMAValue(state, 42)).toBe(42);
-  });
-
-  it("should set first observation directly", () => {
-    const state = createEWMA();
-    const updated = updateEWMA(state, 100);
-    expect(updated.value).toBe(100);
-    expect(updated.count).toBe(1);
-  });
-
-  it("should apply exponential weighting on subsequent observations", () => {
-    let state = createEWMA();
-    state = updateEWMA(state, 100, 0.1);
-    state = updateEWMA(state, 200, 0.1);
-    // 0.1 * 200 + 0.9 * 100 = 110
-    expect(state.value).toBeCloseTo(110, 5);
-    expect(state.count).toBe(2);
-  });
-
-  it("should converge toward recent values with high alpha", () => {
-    let state = createEWMA();
-    state = updateEWMA(state, 100, 0.9);
-    state = updateEWMA(state, 200, 0.9);
-    // 0.9 * 200 + 0.1 * 100 = 190
-    expect(state.value).toBeCloseTo(190, 5);
-  });
-
-  it("should ignore non-finite observations", () => {
-    let state = createEWMA();
-    state = updateEWMA(state, 100);
-    const unchanged = updateEWMA(state, NaN);
-    expect(unchanged.value).toBe(100);
-    expect(unchanged.count).toBe(1);
-  });
-
-  it("should clamp alpha to [0, 1]", () => {
-    let state = createEWMA();
-    state = updateEWMA(state, 100, 2.0);
-    state = updateEWMA(state, 200, 2.0);
-    // alpha clamped to 1.0: 1.0 * 200 + 0.0 * 100 = 200
-    expect(state.value).toBeCloseTo(200, 5);
-  });
-});
-
-describe("Wilson Score Interval", () => {
-  it("should return 0 for zero total", () => {
-    expect(wilsonScoreLowerBound(0, 0)).toBe(0);
-  });
-
-  it("should return 0 for zero successes", () => {
-    const lb = wilsonScoreLowerBound(0, 10);
-    expect(lb).toBeGreaterThanOrEqual(0);
-    expect(lb).toBeLessThan(0.05);
-  });
-
-  it("should return close to 1 for all successes with many trials", () => {
-    const lb = wilsonScoreLowerBound(1000, 1000);
-    expect(lb).toBeGreaterThan(0.99);
-  });
-
-  it("should produce lower bound < upper bound", () => {
-    const [lower, upper] = wilsonScoreInterval(7, 10);
-    expect(lower).toBeLessThan(upper);
-    expect(lower).toBeGreaterThanOrEqual(0);
-    expect(upper).toBeLessThanOrEqual(1);
-  });
-
-  it("should increase lower bound with more observations at same rate", () => {
-    const lb10 = wilsonScoreLowerBound(8, 10);
-    const lb100 = wilsonScoreLowerBound(80, 100);
-    expect(lb100).toBeGreaterThan(lb10);
-  });
-
-  it("should return [0, 0] for zero total in interval", () => {
-    const [lower, upper] = wilsonScoreInterval(0, 0);
-    expect(lower).toBe(0);
-    expect(upper).toBe(0);
-  });
-});
-
-describe("Weighted Scorer", () => {
-  it("should return 0 for empty signals", () => {
-    expect(computeWeightedScore([])).toBe(0);
-  });
-
-  it("should return 0 when all confidences are 0", () => {
-    expect(
-      computeWeightedScore([
-        { value: 1.0, weight: 0.5, confidence: 0 },
-        { value: 0.8, weight: 0.5, confidence: 0 },
-      ]),
-    ).toBe(0);
-  });
-
-  it("should compute correct weighted average for uniform confidence", () => {
-    const score = computeWeightedScore([
-      { value: 1.0, weight: 0.5, confidence: 1.0 },
-      { value: 0.0, weight: 0.5, confidence: 1.0 },
-    ]);
-    expect(score).toBeCloseTo(0.5, 5);
-  });
-
-  it("should weight by confidence", () => {
-    const score = computeWeightedScore([
-      { value: 1.0, weight: 0.5, confidence: 1.0 },
-      { value: 0.0, weight: 0.5, confidence: 0.0 }, // excluded
-    ]);
-    expect(score).toBeCloseTo(1.0, 5);
-  });
-
-  it("should clamp values to [0, 1]", () => {
-    const score = computeWeightedScore([{ value: 2.0, weight: 1.0, confidence: 1.0 }]);
-    expect(score).toBeCloseTo(1.0, 5);
-  });
-
-  it("should handle mixed confidences correctly", () => {
-    // w1*c1*v1 + w2*c2*v2 / (w1*c1 + w2*c2)
-    // 0.35*1.0*1.0 + 0.25*0.5*0.6 / (0.35*1.0 + 0.25*0.5)
-    // = 0.35 + 0.075 / (0.35 + 0.125) = 0.425 / 0.475 = 0.8947...
-    const score = computeWeightedScore([
-      { value: 1.0, weight: 0.35, confidence: 1.0 },
-      { value: 0.6, weight: 0.25, confidence: 0.5 },
-    ]);
-    expect(score).toBeCloseTo(0.425 / 0.475, 4);
-  });
-
-  it("should skip signals with non-finite values", () => {
-    const score = computeWeightedScore([
-      { value: NaN, weight: 0.5, confidence: 1.0 },
-      { value: 0.8, weight: 0.5, confidence: 1.0 },
-    ]);
-    expect(score).toBeCloseTo(0.8, 5);
-  });
-});