@mhingston5/lasso 0.1.0

package/test/helpers/runCompiledWorkflow.ts

@@ -0,0 +1,208 @@
+import { execFileSync } from "node:child_process";
+import type { CompiledHarnessResult, CompiledHarnessWorkflow } from "../../src/compiler/compile.js";
+
+/** Minimal input shape required by the test runner — only `repoPath` is needed. */
+export type WorkflowInput = unknown;
+
+export interface RunCompiledWorkflowOptions {
+  llmResult?: unknown;
+  humanResponse?: unknown;
+  maxContinuations?: number;
+}
+
+type YieldItem =
+  | { kind: "tool-call"; name: string; args: { command: string } }
+  | { kind: "llm-call"; messages: unknown[]; options?: unknown }
+  | { kind: "wait-for-event"; eventName: string }
+  | { kind: "subworkflow"; name: string; input: unknown }
+  | { kind: "all"; tasks: YieldItem[] }
+  | { kind: "timer"; delayMs: number };
+
+export async function runCompiledWorkflow(
+  compiled: CompiledHarnessWorkflow,
+  input: WorkflowInput,
+  options: RunCompiledWorkflowOptions,
+): Promise<CompiledHarnessResult> {
+  const maxContinuations = options.maxContinuations ?? 0;
+  let currentInput: unknown = input;
+  let continuationCount = 0;
+  let lastResult: CompiledHarnessResult | undefined;
+
+  while (continuationCount <= maxContinuations) {
+    const context = createRuntimeContext(continuationCount < maxContinuations);
+    const iterator = compiled.workflows[0].generator(context as any, currentInput);
+
+    let next = iterator.next();
+    let continuationTriggered = false;
+
+    while (!next.done) {
+      try {
+        const resolved = executeYieldItem(next.value as YieldItem, currentInput, options, (nextInput) => {
+          continuationTriggered = true;
+          currentInput = nextInput;
+        });
+        next = iterator.next(resolved);
+      } catch (error) {
+        if (error instanceof ContinueAsNewError) {
+          continuationTriggered = true;
+          currentInput = error.input;
+          break;
+        }
+        if (!iterator.throw) {
+          throw error;
+        }
+        next = iterator.throw(error);
+      }
+    }
+
+    if (!continuationTriggered) {
+      return next.value;
+    }
+
+    lastResult = next.value;
+    continuationCount++;
+  }
+
+  if (lastResult) {
+    return lastResult;
+  }
+
+  throw new Error("Workflow did not complete");
+}
+
+class ContinueAsNewError extends Error {
+  constructor(public input: unknown) {
+    super("continueAsNew called");
+  }
+}
+
+function createRuntimeContext(allowContinueAsNew: boolean = false) {
+  return {
+    scheduleActivity: () => {
+      throw new Error("scheduleActivity is not used in reference workflow tests");
+    },
+    scheduleActivityWithRetry: () => {
+      throw new Error("scheduleActivityWithRetry is not used in reference workflow tests");
+    },
+    scheduleTimer: (delayMs: number) => ({ kind: "timer", delayMs }),
+    waitForEvent: (eventName: string) => ({ kind: "wait-for-event", eventName }),
+    scheduleSubOrchestration: (name: string, input: unknown) => ({ kind: "subworkflow", name, input }),
+    all: (tasks: YieldItem[]) => ({ kind: "all", tasks }),
+    race: () => {
+      throw new Error("race is not used in reference workflow tests");
+    },
+    utcNow: () => 0,
+    newGuid: () => "guid-1",
+    continueAsNew: (input: unknown) => {
+      if (!allowContinueAsNew) {
+        throw new Error("continueAsNew is not enabled in this test context");
+      }
+      throw new ContinueAsNewError(input);
+    },
+    setCustomStatus: () => {},
+    traceInfo: () => {},
+    traceWarn: () => {},
+    traceError: () => {},
+    traceDebug: () => {},
+    kv: {
+      get: () => undefined,
+      set: () => undefined,
+      clear: () => undefined,
+    },
+    pi: {
+      tool: (name: string, args: { command: string }) => ({ kind: "tool-call", name, args }),
+      llm: (messages: unknown[], options?: unknown) => ({ kind: "llm-call", messages, options }),
+      skill: () => {
+        throw new Error("skill is not used in reference workflow tests");
+      },
+      sendMessage: () => {
+        throw new Error("sendMessage is not used in reference workflow tests");
+      },
+      prompt: () => {
+        throw new Error("prompt is not used in reference workflow tests");
+      },
+    },
+  };
+}
+
+function executeYieldItem(
+  item: YieldItem,
+  input: WorkflowInput,
+  options: RunCompiledWorkflowOptions,
+  onContinueAsNew?: (input: unknown) => void,
+): unknown {
+  switch (item.kind) {
+    case "tool-call":
+      return executeToolCall(item.name, item.args.command, resolveRepoPath(input));
+    case "llm-call":
+      return options.llmResult ?? { approved: true };
+    case "wait-for-event":
+      return options.humanResponse ?? { approved: true };
+    case "subworkflow":
+      return {
+        name: item.name,
+        input: item.input,
+      };
+    case "all":
+      return item.tasks.map(task => executeYieldItem(task, input, options, onContinueAsNew));
+    case "timer":
+      return { delayMs: item.delayMs };
+  }
+}
+
+function resolveRepoPath(input: WorkflowInput): string | undefined {
+  if (!input || typeof input !== "object") {
+    return undefined;
+  }
+
+  const record = input as Record<string, unknown>;
+  if (typeof record.repoPath === "string") {
+    return record.repoPath;
+  }
+
+  if (record.input && typeof record.input === "object") {
+    const nested = record.input as Record<string, unknown>;
+    if (typeof nested.repoPath === "string") {
+      return nested.repoPath;
+    }
+  }
+
+  return undefined;
+}
+
+function executeToolCall(name: string, command: string, cwd?: string): unknown {
+  if (name !== "bash") {
+    throw new Error(`Unsupported tool in reference workflow tests: ${name}`);
+  }
+
+  try {
+    const stdout = execFileSync(
+      "bash",
+      ["-lc", command],
+      {
+        ...(cwd ? { cwd } : {}),
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "pipe"],
+      },
+    ).trim();
+
+    if (stdout.length === 0) {
+      return { stdout: "" };
+    }
+
+    try {
+      return JSON.parse(stdout);
+    } catch {
+      return { stdout };
+    }
+  } catch (error) {
+    const result = error as {
+      stderr?: Buffer | string;
+      stdout?: Buffer | string;
+      message?: string;
+    };
+    const stderr = result.stderr?.toString().trim();
+    const stdout = result.stdout?.toString().trim();
+    throw new Error(stderr || stdout || result.message || "bash command failed");
+  }
+}

package/test/memory/advisor.test.ts

@@ -0,0 +1,332 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { adviseFromMemory } from "../../src/memory/advisor.js";
+import { FileMemoryStore } from "../../src/memory/store.js";
+import type { MemoryStore, HarnessMemory, MemoryAdvice } from "../../src/memory/types.js";
+import type { HarnessSpec } from "../../src/spec/types.js";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+function makeSpec(name: string = "test-harness"): HarnessSpec {
+  return {
+    name,
+    graph: {
+      entryNodeId: "start",
+      nodes: [
+        { id: "start", kind: "tool" as const, tool: "bash", args: ["echo", "start"] },
+        { id: "deploy", kind: "tool" as const, tool: "bash", args: ["echo", "deploy"] },
+      ],
+      edges: [{ from: "start", to: "deploy" }],
+    },
+  };
+}
+
+function makeMemory(taskId: string, overrides?: Partial<HarnessMemory>): HarnessMemory {
+  return {
+    taskId,
+    taskEmbedding: `hash-${taskId}`,
+    successfulPatterns: [],
+    failedPatterns: [],
+    mutationHistory: [],
+    effectivenessScore: 0.5,
+    lastUpdated: Date.now(),
+    ...overrides,
+  };
+}
+
+describe("memory/adviseFromMemory", () => {
+  let storeDir: string;
+  let store: MemoryStore;
+
+  beforeEach(async () => {
+    storeDir = await mkdtemp(join(tmpdir(), "advisor-test-"));
+    store = new FileMemoryStore(storeDir);
+  });
+
+  afterEach(async () => {
+    await rm(storeDir, { recursive: true, force: true });
+  });
+
+  describe("basic advice generation", () => {
+    it("should return empty advice when no memories exist", async () => {
+      const advice = await adviseFromMemory("new-task", store);
+
+      expect(advice.suggestions).toEqual([]);
+      expect(advice.warnings).toEqual([]);
+    });
+
+    it("should return advice when matching memories exist", async () => {
+      const memory = makeMemory("similar-task", {
+        taskEmbedding: "hash-similar",
+        successfulPatterns: ["auth-check-before-deploy"],
+        effectivenessScore: 0.8,
+      });
+      await store.saveMemory(memory);
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-similar",
+      });
+
+      expect(advice.suggestions.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe("successful pattern suggestions", () => {
+    it("should suggest successful patterns from similar tasks", async () => {
+      const memory = makeMemory("task-with-success", {
+        taskEmbedding: "hash-deploy",
+        successfulPatterns: ["auth-check-before-deploy", "verify-after-deploy"],
+        effectivenessScore: 0.9,
+      });
+      await store.saveMemory(memory);
+
+      const advice = await adviseFromMemory("new-deploy-task", store, {
+        taskSignature: "hash-deploy",
+      });
+
+      expect(advice.suggestions.some(s => s.includes("auth-check-before-deploy"))).toBe(true);
+      expect(advice.suggestions.some(s => s.includes("verify-after-deploy"))).toBe(true);
+    });
+
+    it("should include effectiveness context in suggestions", async () => {
+      const memory = makeMemory("high-perf-task", {
+        taskEmbedding: "hash-perf",
+        successfulPatterns: ["retry-with-backoff"],
+        effectivenessScore: 0.95,
+      });
+      await store.saveMemory(memory);
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-perf",
+      });
+
+      expect(advice.suggestions.some(s => s.includes("95") || s.includes("0.95"))).toBe(true);
+    });
+
+    it("should aggregate patterns from multiple matching memories", async () => {
+      await store.saveMemory(makeMemory("task-a", {
+        taskEmbedding: "hash-multi",
+        successfulPatterns: ["auth-check-before-deploy"],
+        effectivenessScore: 0.7,
+      }));
+      await store.saveMemory(makeMemory("task-b", {
+        taskEmbedding: "hash-multi",
+        successfulPatterns: ["verify-after-deploy"],
+        effectivenessScore: 0.8,
+      }));
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-multi",
+      });
+
+      expect(advice.suggestions.length).toBeGreaterThanOrEqual(2);
+    });
+  });
+
+  describe("failed pattern warnings", () => {
+    it("should warn about failed patterns from similar tasks", async () => {
+      const memory = makeMemory("task-with-failures", {
+        taskEmbedding: "hash-fail",
+        failedPatterns: ["deploy-without-auth", "skip-verification"],
+        effectivenessScore: 0.3,
+      });
+      await store.saveMemory(memory);
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-fail",
+      });
+
+      expect(advice.warnings.some(w => w.includes("deploy-without-auth"))).toBe(true);
+      expect(advice.warnings.some(w => w.includes("skip-verification"))).toBe(true);
+    });
+
+    it("should include failure context in warnings", async () => {
+      const memory = makeMemory("failing-task", {
+        taskEmbedding: "hash-warn",
+        failedPatterns: ["deploy-without-auth"],
+        mutationHistory: [
+          {
+            mutation: "add-node:auth-check",
+            triggeredBy: "auth-failure",
+            timestamp: Date.now(),
+            outcome: "improved",
+          },
+        ],
+        effectivenessScore: 0.4,
+      });
+      await store.saveMemory(memory);
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-warn",
+      });
+
+      expect(advice.warnings.some(w => w.includes("deploy-without-auth"))).toBe(true);
+    });
+  });
+
+  describe("effectiveness-based filtering", () => {
+    it("should prioritize high-effectiveness memories", async () => {
+      await store.saveMemory(makeMemory("low-perf", {
+        taskEmbedding: "hash-priority",
+        successfulPatterns: ["basic-check"],
+        effectivenessScore: 0.2,
+      }));
+      await store.saveMemory(makeMemory("high-perf", {
+        taskEmbedding: "hash-priority",
+        successfulPatterns: ["advanced-check"],
+        effectivenessScore: 0.9,
+      }));
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-priority",
+      });
+
+      expect(advice.suggestions[0]).toBeDefined();
+      expect(advice.suggestions[0]).toContain("advanced-check");
+    });
+
+    it("should not suggest from low-effectiveness memories when minEffectiveness is set", async () => {
+      await store.saveMemory(makeMemory("low-perf", {
+        taskEmbedding: "hash-filter",
+        successfulPatterns: ["bad-pattern"],
+        effectivenessScore: 0.1,
+      }));
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-filter",
+        minEffectiveness: 0.5,
+      });
+
+      expect(advice.suggestions.some(s => s.includes("bad-pattern"))).toBe(false);
+    });
+  });
+
+  describe("mutation history insights", () => {
+    it("should suggest mutations that previously improved outcomes", async () => {
+      const memory = makeMemory("mutation-task", {
+        taskEmbedding: "hash-mutation",
+        successfulPatterns: [],
+        mutationHistory: [
+          {
+            mutation: "add-node:auth-check",
+            triggeredBy: "auth-failure",
+            timestamp: Date.now(),
+            outcome: "improved",
+          },
+        ],
+        effectivenessScore: 0.7,
+      });
+      await store.saveMemory(memory);
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-mutation",
+      });
+
+      expect(advice.suggestions.some(s => s.includes("add-node:auth-check"))).toBe(true);
+    });
+
+    it("should warn about mutations that made things worse", async () => {
+      const memory = makeMemory("bad-mutation-task", {
+        taskEmbedding: "hash-bad-mutation",
+        mutationHistory: [
+          {
+            mutation: "remove-node:verification",
+            triggeredBy: "slow-execution",
+            timestamp: Date.now(),
+            outcome: "worse",
+          },
+        ],
+        effectivenessScore: 0.3,
+      });
+      await store.saveMemory(memory);
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-bad-mutation",
+      });
+
+      expect(advice.warnings.some(w => w.includes("remove-node:verification"))).toBe(true);
+    });
+  });
+
+  describe("spec-aware advice", () => {
+    it("should tailor suggestions based on current spec nodes", async () => {
+      const memory = makeMemory("spec-aware-task", {
+        taskEmbedding: "hash-spec",
+        successfulPatterns: ["auth-check-before-deploy"],
+        effectivenessScore: 0.8,
+      });
+      await store.saveMemory(memory);
+
+      const spec = makeSpec("deploy-harness");
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-spec",
+      }, spec);
+
+      expect(advice.suggestions.length).toBeGreaterThan(0);
+    });
+
+    it("should not suggest patterns for nodes that already exist", async () => {
+      const memory = makeMemory("existing-node-task", {
+        taskEmbedding: "hash-existing",
+        successfulPatterns: ["auth-check-before-deploy"],
+        effectivenessScore: 0.8,
+      });
+      await store.saveMemory(memory);
+
+      const spec: HarnessSpec = {
+        name: "harness-with-auth",
+        graph: {
+          entryNodeId: "auth-check",
+          nodes: [
+            { id: "auth-check", kind: "tool" as const, tool: "bash", args: ["echo", "auth"] },
+            { id: "deploy", kind: "tool" as const, tool: "bash", args: ["echo", "deploy"] },
+          ],
+          edges: [{ from: "auth-check", to: "deploy" }],
+        },
+      };
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-existing",
+      }, spec);
+
+      expect(advice.suggestions.some(s => s.includes("already"))).toBe(true);
+    });
+  });
+
+  describe("advice structure", () => {
+    it("should include source task IDs in advice", async () => {
+      const memory = makeMemory("source-task-123", {
+        taskEmbedding: "hash-source",
+        successfulPatterns: ["good-pattern"],
+        effectivenessScore: 0.75,
+      });
+      await store.saveMemory(memory);
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-source",
+      });
+
+      expect(advice.sourceTaskIds).toContain("source-task-123");
+    });
+
+    it("should include aggregate effectiveness score", async () => {
+      await store.saveMemory(makeMemory("task-a", {
+        taskEmbedding: "hash-agg",
+        successfulPatterns: ["pattern-a"],
+        effectivenessScore: 0.6,
+      }));
+      await store.saveMemory(makeMemory("task-b", {
+        taskEmbedding: "hash-agg",
+        successfulPatterns: ["pattern-b"],
+        effectivenessScore: 0.8,
+      }));
+
+      const advice = await adviseFromMemory("new-task", store, {
+        taskSignature: "hash-agg",
+      });
+
+      expect(advice.aggregateEffectiveness).toBeGreaterThan(0);
+      expect(advice.aggregateEffectiveness).toBeLessThanOrEqual(1);
+    });
+  });
+});