@mhingston5/lasso 0.1.1 → 0.2.0

package/test/failures/risk.test.ts

@@ -0,0 +1,285 @@
+import { describe, expect, it } from "vitest";
+import {
+  generateFailureModes,
+  assessRisks,
+  failureModeToRisk,
+  probabilityToNumber,
+  failureClassToImpact,
+} from "../../src/failures/generator.js";
+import type { FailureMode } from "../../src/failures/generator.js";
+import type { FailureClass } from "../../src/failures/ontology.js";
+
+function makeFailureMode(overrides: Partial<FailureMode> = {}): FailureMode {
+  return {
+    id: "test-mode-1",
+    description: "Test failure mode",
+    failureClass: "auth",
+    probability: "medium",
+    triggers: ["trigger A"],
+    mitigations: ["mitigation A"],
+    recoveryActions: ["recovery A"],
+    ...overrides,
+  };
+}
+
+describe("probabilityToNumber", () => {
+  it("should map low to 0.2", () => {
+    expect(probabilityToNumber("low")).toBe(0.2);
+  });
+
+  it("should map medium to 0.5", () => {
+    expect(probabilityToNumber("medium")).toBe(0.5);
+  });
+
+  it("should map high to 0.8", () => {
+    expect(probabilityToNumber("high")).toBe(0.8);
+  });
+});
+
+describe("failureClassToImpact", () => {
+  it("should map auth to 0.7", () => {
+    expect(failureClassToImpact("auth")).toBe(0.7);
+  });
+
+  it("should map network to 0.6", () => {
+    expect(failureClassToImpact("network")).toBe(0.6);
+  });
+
+  it("should map resource to 0.5", () => {
+    expect(failureClassToImpact("resource")).toBe(0.5);
+  });
+
+  it("should map semantic to 0.4", () => {
+    expect(failureClassToImpact("semantic")).toBe(0.4);
+  });
+
+  it("should map tool to 0.6", () => {
+    expect(failureClassToImpact("tool")).toBe(0.6);
+  });
+
+  it("should map environment-drift to 0.3", () => {
+    expect(failureClassToImpact("environment-drift")).toBe(0.3);
+  });
+
+  it("should map unknown to 0.3", () => {
+    expect(failureClassToImpact("unknown")).toBe(0.3);
+  });
+
+  it("should map human to 0.5", () => {
+    expect(failureClassToImpact("human")).toBe(0.5);
+  });
+});
+
+describe("failureModeToRisk", () => {
+  it("should convert a FailureMode to a Risk with correct probability", () => {
+    const mode = makeFailureMode({ probability: "high" });
+    const risk = failureModeToRisk(mode);
+
+    expect(risk.probability).toBe(0.8);
+  });
+
+  it("should compute score as probability * impact", () => {
+    const mode = makeFailureMode({ failureClass: "auth", probability: "high" });
+    const risk = failureModeToRisk(mode);
+
+    expect(risk.score).toBeCloseTo(0.8 * 0.7);
+  });
+
+  it("should map triggers to signals", () => {
+    const mode = makeFailureMode({ triggers: ["trigger A", "trigger B"] });
+    const risk = failureModeToRisk(mode);
+
+    expect(risk.signals).toEqual(["trigger A", "trigger B"]);
+  });
+
+  it("should map mitigations to HarnessMutation objects", () => {
+    const mode = makeFailureMode({ mitigations: ["mitigation A"] });
+    const risk = failureModeToRisk(mode);
+
+    expect(risk.mitigations).toHaveLength(1);
+    expect(risk.mitigations[0].type).toBe("add-verification");
+    expect(risk.mitigations[0].description).toBe("mitigation A");
+  });
+
+  it("should preserve failureClass from the FailureMode", () => {
+    const mode = makeFailureMode({ failureClass: "network" });
+    const risk = failureModeToRisk(mode);
+
+    expect(risk.failureClass).toBe("network");
+  });
+
+  it("should preserve description from the FailureMode", () => {
+    const mode = makeFailureMode({ description: "Something broke" });
+    const risk = failureModeToRisk(mode);
+
+    expect(risk.description).toBe("Something broke");
+  });
+
+  it("should use the FailureMode id as the Risk id", () => {
+    const mode = makeFailureMode({ id: "my-mode-42" });
+    const risk = failureModeToRisk(mode);
+
+    expect(risk.id).toBe("my-mode-42");
+  });
+});
+
+describe("assessRisks", () => {
+  it("should return empty risks and 0 overallScore for empty input", () => {
+    const assessment = assessRisks([]);
+
+    expect(assessment.risks).toEqual([]);
+    expect(assessment.overallScore).toBe(0);
+    expect(assessment.risksAboveThreshold).toEqual([]);
+  });
+
+  it("should compute overallScore as average of risk scores", () => {
+    const risks = [
+      failureModeToRisk(makeFailureMode({ id: "a", failureClass: "auth", probability: "high" })),
+      failureModeToRisk(makeFailureMode({ id: "b", failureClass: "tool", probability: "low" })),
+    ];
+
+    const assessment = assessRisks(risks);
+
+    expect(assessment.overallScore).toBeCloseTo((0.8 * 0.7 + 0.2 * 0.6) / 2);
+  });
+
+  it("should filter risksAboveThreshold using default threshold 0.7", () => {
+    const risks = [
+      failureModeToRisk(makeFailureMode({ id: "high-one", failureClass: "auth", probability: "high" })),
+      failureModeToRisk(makeFailureMode({ id: "low-one", failureClass: "unknown", probability: "low" })),
+    ];
+
+    const assessment = assessRisks(risks);
+
+    // auth+high score = 0.8 * 0.7 = 0.56, below 0.7 default threshold
+    expect(assessment.risksAboveThreshold).toHaveLength(0);
+  });
+
+  it("should include risks above a custom threshold", () => {
+    const risks = [
+      failureModeToRisk(makeFailureMode({ id: "high-one", failureClass: "auth", probability: "high" })),
+      failureModeToRisk(makeFailureMode({ id: "low-one", failureClass: "unknown", probability: "low" })),
+    ];
+
+    const assessment = assessRisks(risks, { highRiskThreshold: 0.5 });
+
+    // auth+high = 0.56 > 0.5, unknown+low = 0.06 < 0.5
+    expect(assessment.risksAboveThreshold).toHaveLength(1);
+    expect(assessment.risksAboveThreshold[0].id).toBe("high-one");
+  });
+
+  it("should respect custom highRiskThreshold", () => {
+    const risks = [
+      failureModeToRisk(makeFailureMode({ id: "a", failureClass: "auth", probability: "medium" })),
+      failureModeToRisk(makeFailureMode({ id: "b", failureClass: "tool", probability: "high" })),
+    ];
+
+    const assessment = assessRisks(risks, { highRiskThreshold: 0.4 });
+
+    expect(assessment.risksAboveThreshold.length).toBeGreaterThanOrEqual(1);
+  });
+
+  it("should include all risks in the output", () => {
+    const risks = [
+      failureModeToRisk(makeFailureMode({ id: "a" })),
+      failureModeToRisk(makeFailureMode({ id: "b" })),
+      failureModeToRisk(makeFailureMode({ id: "c" })),
+    ];
+
+    const assessment = assessRisks(risks);
+
+    expect(assessment.risks).toHaveLength(3);
+  });
+
+  it("should use default highRiskThreshold of 0.7", () => {
+    const assessment = assessRisks([]);
+
+    expect(assessment.highRiskThreshold).toBe(0.7);
+  });
+});
+
+describe("generateFailureModes — risks integration", () => {
+  it("should include risks array in the result", () => {
+    const result = generateFailureModes("deploy application");
+
+    expect(result.risks).toBeDefined();
+    expect(Array.isArray(result.risks)).toBe(true);
+    expect(result.risks.length).toBeGreaterThan(0);
+  });
+
+  it("should produce one risk per failure mode", () => {
+    const result = generateFailureModes("deploy application");
+
+    expect(result.risks.length).toBe(result.failureModes.length);
+  });
+
+  it("should produce risks with numeric probability values", () => {
+    const result = generateFailureModes("deploy application");
+
+    for (const risk of result.risks) {
+      expect(typeof risk.probability).toBe("number");
+      expect(risk.probability).toBeGreaterThanOrEqual(0);
+      expect(risk.probability).toBeLessThanOrEqual(1);
+    }
+  });
+
+  it("should produce risks with numeric impact values", () => {
+    const result = generateFailureModes("deploy application");
+
+    for (const risk of result.risks) {
+      expect(typeof risk.impact).toBe("number");
+      expect(risk.impact).toBeGreaterThanOrEqual(0);
+      expect(risk.impact).toBeLessThanOrEqual(1);
+    }
+  });
+
+  it("should produce risks with score = probability * impact", () => {
+    const result = generateFailureModes("deploy application");
+
+    for (const risk of result.risks) {
+      expect(risk.score).toBeCloseTo(risk.probability * risk.impact);
+    }
+  });
+
+  it("should produce risks with HarnessMutation mitigations", () => {
+    const result = generateFailureModes("deploy application");
+
+    for (const risk of result.risks) {
+      expect(Array.isArray(risk.mitigations)).toBe(true);
+      for (const m of risk.mitigations) {
+        expect(m.type).toBeDefined();
+        expect(typeof m.type).toBe("string");
+      }
+    }
+  });
+
+  it("should produce risks with signals from triggers", () => {
+    const result = generateFailureModes("deploy application");
+
+    for (const risk of result.risks) {
+      expect(Array.isArray(risk.signals)).toBe(true);
+      expect(risk.signals.length).toBeGreaterThan(0);
+    }
+  });
+
+  it("should produce risks matching failure classes", () => {
+    const result = generateFailureModes("deploy application");
+
+    for (const risk of result.risks) {
+      expect(typeof risk.failureClass).toBe("string");
+    }
+  });
+
+  it("should remain backwards-compatible — failureModes still has original shape", () => {
+    const result = generateFailureModes("deploy application");
+
+    for (const mode of result.failureModes) {
+      expect(typeof mode.id).toBe("string");
+      expect(typeof mode.description).toBe("string");
+      expect(["low", "medium", "high"]).toContain(mode.probability);
+      expect(Array.isArray(mode.triggers)).toBe(true);
+      expect(Array.isArray(mode.mitigations)).toBe(true);
+      expect(Array.isArray(mode.recoveryActions)).toBe(true);
+    }
+  });
+});

package/test/metaharness/synthesize-from-trace.test.ts

@@ -0,0 +1,372 @@
+import { describe, expect, it } from "vitest";
+import { DefaultMetaHarness } from "../../src/metaharness/engine.js";
+import type {
+  ExecutionTrace,
+  CompletedNode,
+  FailedNode,
+  MetaHarnessConfig,
+} from "../../src/metaharness/types.js";
+import type { HarnessSpec } from "../../src/spec/types.js";
+import type { EnvironmentModel } from "../../src/environment/types.js";
+
+function makeMinimalSpec(overrides?: Partial<HarnessSpec>): HarnessSpec {
+  return {
+    name: "test-harness",
+    graph: {
+      entryNodeId: "node-a",
+      nodes: [
+        {
+          id: "node-a",
+          label: "Node A",
+          kind: "tool",
+          tool: "bash",
+          args: ["echo hello"],
+        },
+        {
+          id: "node-b",
+          label: "Node B",
+          kind: "tool",
+          tool: "bash",
+          args: ["echo world"],
+        },
+      ],
+      edges: [{ from: "node-a", to: "node-b" }],
+    },
+    ...overrides,
+  };
+}
+
+function makeMinimalEnv(overrides?: Partial<EnvironmentModel>): EnvironmentModel {
+  return {
+    tools: [
+      { name: "bash", version: "5.0", available: true },
+      { name: "git", version: "2.39", available: true },
+    ],
+    resources: [
+      { name: "disk", type: "disk", available: true, limit: "500GB", usage: "45%" },
+    ],
+    constraints: [],
+    authState: [],
+    externalSystems: [],
+    discoveredAt: Date.now(),
+    ...overrides,
+  };
+}
+
+function makeCompletedNode(overrides: Partial<CompletedNode>): CompletedNode {
+  return {
+    nodeId: "node-a",
+    startedAt: Date.now() - 1000,
+    completedAt: Date.now(),
+    output: "ok",
+    ...overrides,
+  };
+}
+
+function makeFailedNode(overrides: Partial<FailedNode>): FailedNode {
+  return {
+    nodeId: "node-a",
+    startedAt: Date.now() - 1000,
+    failedAt: Date.now(),
+    error: "command not found",
+    retryCount: 0,
+    ...overrides,
+  };
+}
+
+function makeTrace(overrides?: Partial<ExecutionTrace>): ExecutionTrace {
+  return {
+    completedNodes: [],
+    failedNodes: [],
+    capturedAt: Date.now(),
+    ...overrides,
+  };
+}
+
+describe("synthesizeFromTrace", () => {
+  const harness = new DefaultMetaHarness({});
+  const env = makeMinimalEnv();
+  const spec = makeMinimalSpec();
+
+  describe("empty trace", () => {
+    it("returns spec unchanged with no mutations when trace is empty", async () => {
+      const trace = makeTrace();
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.mutations).toHaveLength(0);
+      expect(result.spec).toEqual(spec);
+      expect(result.rationale).toHaveLength(0);
+      expect(result.decision).toBe("continue");
+    });
+  });
+
+  describe("completed nodes only", () => {
+    it("returns continue decision when all nodes completed successfully", async () => {
+      const trace = makeTrace({
+        completedNodes: [
+          makeCompletedNode({ nodeId: "node-a" }),
+          makeCompletedNode({ nodeId: "node-b" }),
+        ],
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.decision).toBe("continue");
+      expect(result.mutations).toHaveLength(0);
+      expect(result.rationale).toHaveLength(0);
+    });
+  });
+
+  describe("failure analysis", () => {
+    it("classifies failures and derives mutations for tool failures", async () => {
+      const trace = makeTrace({
+        failedNodes: [
+          makeFailedNode({
+            nodeId: "node-a",
+            error: "bash: command not found: kubectl",
+            retryCount: 0,
+          }),
+        ],
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.mutations.length).toBeGreaterThan(0);
+      expect(result.rationale.length).toBeGreaterThan(0);
+    });
+
+    it("adds retry policy for transient network failures", async () => {
+      const trace = makeTrace({
+        failedNodes: [
+          makeFailedNode({
+            nodeId: "node-a",
+            error: "ETIMEDOUT connection timeout",
+            retryCount: 0,
+          }),
+        ],
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      const retryMutations = result.mutations.filter(m => m.type === "modify-node");
+      expect(retryMutations.length).toBeGreaterThan(0);
+    });
+
+    it("detects repeated failures on same node", async () => {
+      const trace = makeTrace({
+        failedNodes: [
+          makeFailedNode({
+            nodeId: "node-a",
+            error: "connection refused",
+            retryCount: 2,
+            failedAt: Date.now() - 2000,
+          }),
+          makeFailedNode({
+            nodeId: "node-a",
+            error: "connection refused",
+            retryCount: 3,
+            failedAt: Date.now(),
+          }),
+        ],
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.rationale.some(r => r.includes("repeated") || r.includes("node-a"))).toBe(true);
+    });
+
+    it("detects slow nodes", async () => {
+      const now = Date.now();
+      const trace = makeTrace({
+        completedNodes: [
+          makeCompletedNode({
+            nodeId: "node-a",
+            startedAt: now - 60000,
+            completedAt: now,
+          }),
+        ],
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.rationale.some(r => r.includes("slow") || r.includes("duration"))).toBe(true);
+    });
+
+    it("detects cost spikes", async () => {
+      const trace = makeTrace({
+        completedNodes: [
+          makeCompletedNode({ nodeId: "node-a", costUsd: 5.0 }),
+          makeCompletedNode({ nodeId: "node-b", costUsd: 3.0 }),
+        ],
+        totalCostUsd: 8.0,
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.rationale.some(r => r.includes("cost") || r.includes("budget"))).toBe(true);
+    });
+  });
+
+  describe("decision logic", () => {
+    it("returns stop when all nodes failed", async () => {
+      const trace = makeTrace({
+        failedNodes: [
+          makeFailedNode({ nodeId: "node-a", error: "auth required" }),
+          makeFailedNode({ nodeId: "node-b", error: "auth required" }),
+        ],
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.decision).toBe("stop");
+    });
+
+    it("returns needs_operator_input for human-required failures", async () => {
+      const trace = makeTrace({
+        failedNodes: [
+          makeFailedNode({
+            nodeId: "node-a",
+            error: "approval required by operator",
+            retryCount: 0,
+          }),
+        ],
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.decision).toBe("needs_operator_input");
+    });
+
+    it("returns continue when failures are recoverable", async () => {
+      const trace = makeTrace({
+        completedNodes: [
+          makeCompletedNode({ nodeId: "node-a" }),
+        ],
+        failedNodes: [
+          makeFailedNode({
+            nodeId: "node-b",
+            error: "ETIMEDOUT",
+            retryCount: 0,
+          }),
+        ],
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.decision).toBe("continue");
+    });
+  });
+
+  describe("mutation application", () => {
+    it("applies mutations to spec and returns updated spec", async () => {
+      const trace = makeTrace({
+        failedNodes: [
+          makeFailedNode({
+            nodeId: "node-a",
+            error: "connection timeout",
+            retryCount: 0,
+          }),
+        ],
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.spec).not.toEqual(spec);
+      expect(result.spec.graph.nodes.length).toBeGreaterThanOrEqual(spec.graph.nodes.length);
+    });
+
+    it("does not mutate original spec", async () => {
+      const originalSpec = makeMinimalSpec();
+      const trace = makeTrace({
+        failedNodes: [
+          makeFailedNode({ nodeId: "node-a", error: "timeout" }),
+        ],
+      });
+
+      await harness.synthesizeFromTrace(trace, originalSpec, env);
+
+      expect(originalSpec.graph.nodes.length).toBe(2);
+    });
+  });
+
+  describe("mutation policy enforcement", () => {
+    it("respects mutation policy when provided", async () => {
+      const config: MetaHarnessConfig = {
+        mutationPolicy: {
+          allowedMutations: ["modify-node"],
+          maxMutations: 1,
+        },
+      };
+      const harnessWithPolicy = new DefaultMetaHarness(config);
+      const trace = makeTrace({
+        failedNodes: [
+          makeFailedNode({
+            nodeId: "node-a",
+            error: "ETIMEDOUT connection timeout",
+            retryCount: 0,
+          }),
+          makeFailedNode({
+            nodeId: "node-b",
+            error: "ETIMEDOUT connection timeout",
+            retryCount: 0,
+          }),
+        ],
+      });
+
+      const result = await harnessWithPolicy.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.mutations.length).toBeLessThanOrEqual(1);
+    });
+  });
+
+  describe("multiple failure patterns", () => {
+    it("handles mixed completed and failed nodes", async () => {
+      const trace = makeTrace({
+        completedNodes: [
+          makeCompletedNode({ nodeId: "node-a", costUsd: 0.5 }),
+        ],
+        failedNodes: [
+          makeFailedNode({
+            nodeId: "node-b",
+            error: "permission denied",
+            retryCount: 2,
+          }),
+        ],
+        totalCostUsd: 0.5,
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.mutations.length).toBeGreaterThan(0);
+      expect(result.rationale.length).toBeGreaterThan(0);
+      expect(["continue", "needs_operator_input", "stop"]).toContain(result.decision);
+    });
+
+    it("derives trace-based mutations when entries are populated via adapter", async () => {
+      const trace = makeTrace({
+        failedNodes: [
+          makeFailedNode({
+            nodeId: "node-a",
+            error: "connection refused",
+            retryCount: 3,
+          }),
+          makeFailedNode({
+            nodeId: "node-a",
+            error: "connection refused",
+            retryCount: 4,
+          }),
+          makeFailedNode({
+            nodeId: "node-a",
+            error: "connection refused",
+            retryCount: 5,
+          }),
+        ],
+      });
+
+      const result = await harness.synthesizeFromTrace(trace, spec, env);
+
+      expect(result.rationale.some(r => r.includes("trace"))).toBe(true);
+      expect(result.mutations.length).toBeGreaterThan(0);
+    });
+  });
+});