@symerian/symi 3.0.18 → 3.0.19

package/extensions/learning-loop/src/test/quality-engine.test.ts

@@ -1,231 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { combineSignals } from "../scoring/normalization.js";
-import {
-  extractTaskCompletion,
-  extractToolEfficiency,
-  extractResponseAppropriateness,
-  extractLatencyRelative,
-  extractUserFeedback,
-} from "../scoring/signal-extractors.js";
-import type { CompletedRun } from "../types.js";
-
-function makeRun(overrides: Partial<CompletedRun> = {}): CompletedRun {
-  return {
-    runId: "test-run",
-    sessionId: "sess-1",
-    sessionKey: "sk-1",
-    agentId: "agent-1",
-    provider: "openai",
-    model: "gpt-4",
-    promptHash: "abc123",
-    promptLength: 100,
-    responseLength: 200,
-    responseToolCallCount: 0,
-    usage: { input: 50, output: 100, total: 150 },
-    toolCalls: [],
-    success: true,
-    error: null,
-    durationMs: 1000,
-    startedAt: Date.now() - 1000,
-    completedAt: Date.now(),
-    ...overrides,
-  };
-}
-
-describe("Signal Extractors", () => {
-  describe("Task Completion", () => {
-    it("should return 1.0 for success without error", () => {
-      const signal = extractTaskCompletion(makeRun({ success: true, error: null }), 0.35);
-      expect(signal.value).toBe(1.0);
-      expect(signal.confidence).toBe(1.0);
-    });
-
-    it("should return 0.5 for success with error", () => {
-      const signal = extractTaskCompletion(makeRun({ success: true, error: "warning" }), 0.35);
-      expect(signal.value).toBe(0.5);
-    });
-
-    it("should return 0.0 for failure", () => {
-      const signal = extractTaskCompletion(makeRun({ success: false }), 0.35);
-      expect(signal.value).toBe(0.0);
-    });
-  });
-
-  describe("Tool Efficiency", () => {
-    it("should return 1.0 with no tool calls", () => {
-      const signal = extractToolEfficiency(makeRun({ toolCalls: [] }), 0.25);
-      expect(signal.value).toBe(1.0);
-      expect(signal.confidence).toBe(0.3);
-    });
-
-    it("should return 1.0 for all successful unique tool calls", () => {
-      const signal = extractToolEfficiency(
-        makeRun({
-          toolCalls: [
-            { toolName: "a", durationMs: 10, success: true, error: null, paramHash: "h1" },
-            { toolName: "b", durationMs: 20, success: true, error: null, paramHash: "h2" },
-          ],
-        }),
-        0.25,
-      );
-      expect(signal.value).toBe(1.0);
-    });
-
-    it("should penalize errors", () => {
-      const signal = extractToolEfficiency(
-        makeRun({
-          toolCalls: [
-            { toolName: "a", durationMs: 10, success: false, error: "fail", paramHash: "h1" },
-            { toolName: "b", durationMs: 20, success: true, error: null, paramHash: "h2" },
-          ],
-        }),
-        0.25,
-      );
-      // 1.0 - 0.6*(1/2) - 0.4*(0/2) = 0.7
-      expect(signal.value).toBeCloseTo(0.7, 5);
-    });
-
-    it("should penalize redundant calls", () => {
-      const signal = extractToolEfficiency(
-        makeRun({
-          toolCalls: [
-            { toolName: "a", durationMs: 10, success: true, error: null, paramHash: "h1" },
-            { toolName: "a", durationMs: 10, success: true, error: null, paramHash: "h1" },
-          ],
-        }),
-        0.25,
-      );
-      // 1.0 - 0.6*(0/2) - 0.4*(1/2) = 0.8
-      expect(signal.value).toBeCloseTo(0.8, 5);
-    });
-
-    it("should increase confidence with more tool calls", () => {
-      const few = extractToolEfficiency(
-        makeRun({
-          toolCalls: [
-            { toolName: "a", durationMs: 10, success: true, error: null, paramHash: "h1" },
-          ],
-        }),
-        0.25,
-      );
-      const many = extractToolEfficiency(
-        makeRun({
-          toolCalls: Array.from({ length: 5 }, (_, i) => ({
-            toolName: `t${i}`,
-            durationMs: 10,
-            success: true,
-            error: null,
-            paramHash: `h${i}`,
-          })),
-        }),
-        0.25,
-      );
-      expect(many.confidence).toBeGreaterThan(few.confidence);
-    });
-  });
-
-  describe("Response Appropriateness", () => {
-    it("should return 0.5 for zero prompt length", () => {
-      const signal = extractResponseAppropriateness(makeRun({ promptLength: 0 }), 0.1);
-      expect(signal.value).toBe(0.5);
-    });
-
-    it("should return ~1.0 for equal prompt and response length", () => {
-      const signal = extractResponseAppropriateness(
-        makeRun({ promptLength: 100, responseLength: 100 }),
-        0.1,
-      );
-      // ratio = 1.0, ln(1) = 0, exp(-0) = 1.0
-      expect(signal.value).toBeCloseTo(1.0, 5);
-    });
-
-    it("should penalize very long responses", () => {
-      const signal = extractResponseAppropriateness(
-        makeRun({ promptLength: 10, responseLength: 10000 }),
-        0.1,
-      );
-      expect(signal.value).toBeLessThan(0.5);
-    });
-  });
-
-  describe("Latency Relative", () => {
-    it("should return 1.0 for zero duration", () => {
-      const signal = extractLatencyRelative(makeRun({ durationMs: 0 }), 0.1, {
-        value: 0,
-        count: 0,
-      });
-      expect(signal.value).toBe(1.0);
-    });
-
-    it("should return 1.0 when faster than EWMA", () => {
-      const signal = extractLatencyRelative(makeRun({ durationMs: 500 }), 0.1, {
-        value: 1000,
-        count: 10,
-      });
-      expect(signal.value).toBe(1.0);
-    });
-
-    it("should degrade when slower than EWMA", () => {
-      const signal = extractLatencyRelative(makeRun({ durationMs: 2000 }), 0.1, {
-        value: 1000,
-        count: 10,
-      });
-      expect(signal.value).toBeCloseTo(0.5, 5);
-    });
-  });
-
-  describe("User Feedback", () => {
-    it("should return confidence 0 when no feedback", () => {
-      const signal = extractUserFeedback(0.2);
-      expect(signal.confidence).toBe(0.0);
-    });
-
-    it("should map explicit 5 to 1.0", () => {
-      const signal = extractUserFeedback(0.2, { source: "explicit", score: 5 });
-      expect(signal.value).toBe(1.0);
-      expect(signal.confidence).toBe(1.0);
-    });
-
-    it("should map explicit 1 to 0.0", () => {
-      const signal = extractUserFeedback(0.2, { source: "explicit", score: 1 });
-      expect(signal.value).toBe(0.0);
-    });
-
-    it("should map explicit 3 to 0.5", () => {
-      const signal = extractUserFeedback(0.2, { source: "explicit", score: 3 });
-      expect(signal.value).toBe(0.5);
-    });
-
-    it("should use lower confidence for implicit feedback", () => {
-      const signal = extractUserFeedback(0.2, { source: "implicit", score: 0.8 });
-      expect(signal.confidence).toBe(0.6);
-    });
-  });
-});
-
-describe("Signal Combination", () => {
-  it("should produce score in [0, 1]", () => {
-    const score = combineSignals([
-      { name: "taskCompletion", value: 1.0, confidence: 1.0, weight: 0.35 },
-      { name: "toolEfficiency", value: 0.8, confidence: 0.7, weight: 0.25 },
-      { name: "responseAppropriateLength", value: 0.9, confidence: 0.5, weight: 0.1 },
-      { name: "latencyRelative", value: 0.7, confidence: 0.3, weight: 0.1 },
-      { name: "userFeedback", value: 0.5, confidence: 0.0, weight: 0.2 },
-    ]);
-    expect(score.score).toBeGreaterThanOrEqual(0);
-    expect(score.score).toBeLessThanOrEqual(1);
-    expect(score.algorithmVersion).toBe(1);
-  });
-
-  it("should exclude zero-confidence signals", () => {
-    const withFeedback = combineSignals([
-      { name: "taskCompletion", value: 1.0, confidence: 1.0, weight: 0.35 },
-      { name: "userFeedback", value: 0.0, confidence: 0.0, weight: 0.2 },
-    ]);
-    const withoutFeedback = combineSignals([
-      { name: "taskCompletion", value: 1.0, confidence: 1.0, weight: 0.35 },
-    ]);
-    // Should be equal since feedback has 0 confidence
-    expect(withFeedback.score).toBeCloseTo(withoutFeedback.score, 5);
-  });
-});

package/extensions/learning-loop/src/test/run-tracker.test.ts

@@ -1,143 +0,0 @@
-import { describe, it, expect, beforeEach } from "vitest";
-import { createRunTracker } from "../capture/run-tracker.js";
-
-describe("RunTracker", () => {
-  let tracker: ReturnType<typeof createRunTracker>;
-
-  beforeEach(() => {
-    tracker = createRunTracker();
-  });
-
-  it("should start with no active runs", () => {
-    expect(tracker.getActiveRunCount()).toBe(0);
-  });
-
-  it("should create a run on llm_input", () => {
-    tracker.onLlmInput({
-      runId: "run-1",
-      sessionId: "sess-1",
-      provider: "openai",
-      model: "gpt-4",
-      prompt: "Hello world",
-      sessionKey: "sk-1",
-      agentId: "agent-1",
-    });
-    expect(tracker.getActiveRunCount()).toBe(1);
-  });
-
-  it("should bind session to run", () => {
-    tracker.onLlmInput({
-      runId: "run-1",
-      sessionId: "sess-1",
-      provider: "openai",
-      model: "gpt-4",
-      prompt: "Hello",
-      sessionKey: "sk-1",
-    });
-    expect(tracker.getRunIdForSession("sk-1")).toBe("run-1");
-  });
-
-  it("should accumulate tool calls", () => {
-    tracker.onLlmInput({
-      runId: "run-1",
-      sessionId: "sess-1",
-      provider: "openai",
-      model: "gpt-4",
-      prompt: "Hello",
-      sessionKey: "sk-1",
-    });
-
-    tracker.onToolCall({
-      toolName: "search",
-      durationMs: 50,
-      success: true,
-      paramHash: "h1",
-      sessionKey: "sk-1",
-    });
-
-    tracker.onToolCall({
-      toolName: "read",
-      durationMs: 30,
-      success: false,
-      error: "not found",
-      paramHash: "h2",
-      sessionKey: "sk-1",
-    });
-
-    const result = tracker.finalize({
-      sessionKey: "sk-1",
-      success: true,
-      durationMs: 500,
-    });
-
-    expect(result).not.toBeNull();
-    expect(result!.toolCalls).toHaveLength(2);
-    expect(result!.toolCalls[0]!.toolName).toBe("search");
-    expect(result!.toolCalls[1]!.success).toBe(false);
-  });
-
-  it("should capture usage from llm_output", () => {
-    tracker.onLlmInput({
-      runId: "run-1",
-      sessionId: "sess-1",
-      provider: "claude",
-      model: "claude-3",
-      prompt: "Test prompt",
-      sessionKey: "sk-1",
-    });
-
-    tracker.onLlmOutput({
-      runId: "run-1",
-      sessionId: "sess-1",
-      assistantTexts: ["Response text here"],
-      usage: { input: 10, output: 20, total: 30 },
-    });
-
-    const result = tracker.finalize({
-      sessionKey: "sk-1",
-      success: true,
-      durationMs: 200,
-    });
-
-    expect(result).not.toBeNull();
-    expect(result!.responseLength).toBe("Response text here".length);
-    expect(result!.usage.input).toBe(10);
-    expect(result!.usage.output).toBe(20);
-  });
-
-  it("should return null for finalize without llm_input", () => {
-    const result = tracker.finalize({
-      sessionKey: "sk-unknown",
-      success: true,
-    });
-    expect(result).toBeNull();
-  });
-
-  it("should clean up after finalize", () => {
-    tracker.onLlmInput({
-      runId: "run-1",
-      sessionId: "sess-1",
-      provider: "openai",
-      model: "gpt-4",
-      prompt: "Hello",
-      sessionKey: "sk-1",
-    });
-
-    tracker.finalize({ sessionKey: "sk-1", success: true });
-    expect(tracker.getActiveRunCount()).toBe(0);
-    expect(tracker.getRunIdForSession("sk-1")).toBeUndefined();
-  });
-
-  it("should clear all state", () => {
-    tracker.onLlmInput({
-      runId: "run-1",
-      sessionId: "sess-1",
-      provider: "openai",
-      model: "gpt-4",
-      prompt: "Hello",
-      sessionKey: "sk-1",
-    });
-    tracker.clear();
-    expect(tracker.getActiveRunCount()).toBe(0);
-  });
-});

package/extensions/learning-loop/src/types.ts

@@ -1,281 +0,0 @@
-export type RunId = string;
-export type SessionId = string;
-export type SessionKey = string;
-export type AgentId = string;
-
-export type UsageData = {
-  input?: number;
-  output?: number;
-  cacheRead?: number;
-  cacheWrite?: number;
-  total?: number;
-};
-
-export type ToolCallRecord = {
-  toolName: string;
-  durationMs: number | null;
-  success: boolean;
-  error: string | null;
-  paramHash: string;
-};
-
-export type PartialRun = {
-  runId: RunId;
-  sessionId?: SessionId;
-  sessionKey?: SessionKey;
-  agentId?: AgentId;
-  provider?: string;
-  model?: string;
-  promptHash?: string;
-  promptLength?: number;
-  responseLength?: number;
-  responseToolCallCount?: number;
-  usage?: UsageData;
-  toolCalls: ToolCallRecord[];
-  success?: boolean;
-  error?: string;
-  durationMs?: number;
-  startedAt: number;
-};
-
-export type CompletedRun = {
-  runId: RunId;
-  sessionId: string;
-  sessionKey: string;
-  agentId: string;
-  provider: string;
-  model: string;
-  promptHash: string;
-  promptLength: number;
-  responseLength: number;
-  responseToolCallCount: number;
-  usage: UsageData;
-  toolCalls: ToolCallRecord[];
-  success: boolean;
-  error: string | null;
-  durationMs: number;
-  startedAt: number;
-  completedAt: number;
-};
-
-export type QualitySignalName =
-  | "taskCompletion"
-  | "toolEfficiency"
-  | "responseAppropriateLength"
-  | "latencyRelative"
-  | "userFeedback";
-
-export type QualitySignal = {
-  name: QualitySignalName;
-  value: number;
-  confidence: number;
-  weight: number;
-};
-
-export type QualityScore = {
-  score: number;
-  signals: QualitySignal[];
-  algorithmVersion: number;
-};
-
-export type LearningCategory =
-  | "tool_pattern"
-  | "error_recovery"
-  | "model_affinity"
-  | "anti_pattern";
-
-export type LearningRecord = {
-  id: string;
-  runId: RunId;
-  category: LearningCategory;
-  content: string;
-  embedding: number[] | null;
-  confidence: number;
-  appliedCount: number;
-  createdAt: number;
-  updatedAt: number;
-};
-
-export type FeedbackRecord = {
-  id: string;
-  runId: RunId;
-  source: "explicit" | "implicit";
-  score: number;
-  createdAt: number;
-};
-
-export type MetricsBucket = {
-  provider: string;
-  model: string;
-  bucketHour: string;
-  runCount: number;
-  successCount: number;
-  qualitySum: number;
-  qualitySumSq: number;
-  latencySum: number;
-  latencySumSq: number;
-  tokenInputSum: number;
-  tokenOutputSum: number;
-};
-
-export type RunRow = {
-  run_id: string;
-  session_id: string;
-  session_key: string;
-  agent_id: string;
-  provider: string;
-  model: string;
-  prompt_hash: string;
-  prompt_length: number;
-  response_length: number;
-  response_tool_call_count: number;
-  usage_input: number;
-  usage_output: number;
-  usage_cache_read: number;
-  usage_cache_write: number;
-  usage_total: number;
-  success: number;
-  error: string | null;
-  duration_ms: number;
-  quality_score: number | null;
-  algorithm_version: number | null;
-  started_at: number;
-  completed_at: number;
-};
-
-export type ToolCallRow = {
-  id: number;
-  run_id: string;
-  tool_name: string;
-  duration_ms: number | null;
-  success: number;
-  error: string | null;
-  param_hash: string;
-};
-
-export type LearningRow = {
-  id: string;
-  run_id: string;
-  category: string;
-  content: string;
-  embedding: string | null;
-  confidence: number;
-  applied_count: number;
-  created_at: number;
-  updated_at: number;
-};
-
-export type FeedbackRow = {
-  id: string;
-  run_id: string;
-  source: string;
-  score: number;
-  created_at: number;
-};
-
-export type MetricsBucketRow = {
-  provider: string;
-  model: string;
-  bucket_hour: string;
-  run_count: number;
-  success_count: number;
-  quality_sum: number;
-  quality_sum_sq: number;
-  latency_sum: number;
-  latency_sum_sq: number;
-  token_input_sum: number;
-  token_output_sum: number;
-};
-
-export type ScoringWeights = {
-  taskCompletion: number;
-  toolEfficiency: number;
-  responseAppropriateLength: number;
-  latencyRelative: number;
-  userFeedback: number;
-};
-
-export type InjectionConfig = {
-  maxLearnings: number;
-  minRelevance: number;
-  maxTokens: number;
-  cacheTtlMs: number;
-};
-
-export type CaptureConfig = {
-  embedPrompts: boolean;
-  maxRuns: number;
-};
-
-export type DecayConfig = {
-  halfLifeDays: number;
-};
-
-export type LearningLoopConfig = {
-  capture: CaptureConfig;
-  scoring: { weights: ScoringWeights };
-  injection: InjectionConfig;
-  decay: DecayConfig;
-};
-
-const DEFAULT_CONFIG: LearningLoopConfig = {
-  capture: { embedPrompts: false, maxRuns: 10_000 },
-  scoring: {
-    weights: {
-      taskCompletion: 0.35,
-      toolEfficiency: 0.25,
-      responseAppropriateLength: 0.1,
-      latencyRelative: 0.1,
-      userFeedback: 0.2,
-    },
-  },
-  injection: {
-    maxLearnings: 5,
-    minRelevance: 0.3,
-    maxTokens: 500,
-    cacheTtlMs: 60_000,
-  },
-  decay: { halfLifeDays: 30 },
-};
-
-export function resolveConfig(pluginConfig?: Record<string, unknown>): LearningLoopConfig {
-  if (!pluginConfig) return { ...DEFAULT_CONFIG };
-
-  const raw = pluginConfig as Partial<{
-    capture: Partial<CaptureConfig>;
-    scoring: Partial<{ weights: Partial<ScoringWeights> }>;
-    injection: Partial<InjectionConfig>;
-    decay: Partial<DecayConfig>;
-  }>;
-
-  return {
-    capture: { ...DEFAULT_CONFIG.capture, ...raw.capture },
-    scoring: {
-      weights: { ...DEFAULT_CONFIG.scoring.weights, ...raw.scoring?.weights },
-    },
-    injection: { ...DEFAULT_CONFIG.injection, ...raw.injection },
-    decay: { ...DEFAULT_CONFIG.decay, ...raw.decay },
-  };
-}
-
-export type EdgeType = "T" | "S" | "C" | "U" | "X" | "R";
-
-export interface LearningEdge {
-  id: number;
-  sourceId: string;
-  targetId: string;
-  edgeType: EdgeType;
-  weight: number;
-  createdAt: number;
-}
-
-export interface EdgeRow {
-  id: number;
-  source_id: string;
-  target_id: string;
-  edge_type: string;
-  weight: number;
-  created_at: number;
-}
-
-export const ALGORITHM_VERSION = 1;

package/extensions/learning-loop/symi.plugin.json

@@ -1,46 +0,0 @@
-{
-  "id": "learning-loop",
-  "configSchema": {
-    "type": "object",
-    "additionalProperties": false,
-    "properties": {
-      "capture": {
-        "type": "object",
-        "properties": {
-          "embedPrompts": { "type": "boolean" },
-          "maxRuns": { "type": "number" }
-        }
-      },
-      "scoring": {
-        "type": "object",
-        "properties": {
-          "weights": {
-            "type": "object",
-            "properties": {
-              "taskCompletion": { "type": "number" },
-              "toolEfficiency": { "type": "number" },
-              "responseAppropriateLength": { "type": "number" },
-              "latencyRelative": { "type": "number" },
-              "userFeedback": { "type": "number" }
-            }
-          }
-        }
-      },
-      "injection": {
-        "type": "object",
-        "properties": {
-          "maxLearnings": { "type": "number" },
-          "minRelevance": { "type": "number" },
-          "maxTokens": { "type": "number" },
-          "cacheTtlMs": { "type": "number" }
-        }
-      },
-      "decay": {
-        "type": "object",
-        "properties": {
-          "halfLifeDays": { "type": "number" }
-        }
-      }
-    }
-  }
-}