opencode-wait-a-minute 0.0.0-stage → 1.1.0-rc.8

package/src/context/context-routing-evaluation.js

@@ -0,0 +1,331 @@
+/**
+ * WAM Context Routing — Evaluation Suite
+ *
+ * Compares the existing WAM Context Pack against the shadow routing strategy.
+ * Prioritizes task correctness and verification over token reduction.
+ */
+
+import { ContextGraph } from "./context-graph.js";
+import { getCurrentContextPack, getProposedContext, hasRequiredContext, computeOverlap } from "./context-routing-shadow.js";
+import { persistTaskState } from "../skills/engine.js";
+import fs from "node:fs";
+import path from "node:path";
+
+// -- Types --
+
+/**
+ * @typedef {Object} EvaluationResult
+ * @property {string} scenario
+ * @property {string} strategy
+ * @property {boolean} taskSuccess
+ * @property {boolean} verificationSuccess
+ * @property {number} contextTokens
+ * @property {number} contextRecall
+ * @property {number} contextPrecision
+ * @property {number} dependencyCoverage
+ * @property {number} evidenceCoverage
+ * @property {string[]} missingDependencies
+ */
+
+// -- Token estimation --
+
+function estimateTokens(text = "") {
+  return Math.ceil((text || "").length / 4);
+}
+
+function nodeTokens(node) {
+  return estimateTokens(node.content) + 20;
+}
+
+// -- Metrics --
+
+function computeMetrics(result, expected, graph) {
+  const resultIds = new Set(result.nodeIds);
+  const expectedIds = new Set(expected.requiredNodes);
+
+  // Context Recall
+  let recall = 0;
+  if (expectedIds.size > 0) {
+    for (const id of expectedIds) {
+      if (resultIds.has(id)) recall++;
+    }
+    recall /= expectedIds.size;
+  }
+
+  // Context Precision
+  let precision = 0;
+  if (resultIds.size > 0) {
+    for (const id of resultIds) {
+      if (expectedIds.has(id)) precision++;
+    }
+    precision /= resultIds.size;
+  }
+
+  // Dependency Coverage
+  let depCoverage = 1;
+  if (expected.missingDeps && expected.missingDeps.length > 0) {
+    const foundMissing = result.gaps ? result.gaps.filter((g) => g.type === "missing_dependency") : [];
+    depCoverage = foundMissing.length > 0 ? 0 : 1;
+  }
+
+  // Evidence Coverage
+  const evidenceNodes = graph ? graph.getNodes("evidence") : [];
+  const requiredEvidence = evidenceNodes.filter((e) =>
+    [...expectedIds].some((id) => {
+      if (!graph) return false;
+      const edges = graph.getEdgesFrom(id);
+      return edges.some((edge) => edge.to === e.id);
+    })
+  );
+
+  let evidenceCoverage = 1;
+  if (requiredEvidence.length > 0) {
+    const foundEvidence = requiredEvidence.filter((e) => resultIds.has(e.id));
+    evidenceCoverage = foundEvidence.length / requiredEvidence.length;
+  }
+
+  // Task Success
+  const taskSuccess = expectedIds.size > 0 &&
+    [...expectedIds].every((id) => resultIds.has(id));
+
+  // Verification Success
+  const contradictions = result.gaps ? result.gaps.filter((g) => g.type === "contradiction") : [];
+  const verificationSuccess = contradictions.length === 0 && evidenceCoverage > 0.5;
+
+  return {
+    taskSuccess,
+    verificationSuccess,
+    contextTokens: result.tokenEstimate || 0,
+    contextRecall: recall,
+    contextPrecision: precision,
+    dependencyCoverage: depCoverage,
+    evidenceCoverage,
+    missingDependencies: result.gaps ? result.gaps.filter((g) => g.type === "missing_dependency").map((g) => g.description) : [],
+  };
+}
+
+// -- Scenarios --
+
+function buildNoiseScenario() {
+  const graph = new ContextGraph();
+  const taskId = "task-noise";
+
+  graph.addNode({ id: taskId, type: "task", content: "Build authentication system", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "auth-output", type: "output", content: "JWT auth module with bcrypt", verified: true, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "auth-evidence", type: "evidence", content: "Auth test passed with 100% coverage", verified: true, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "auth-decision", type: "decision", content: "Use JWT tokens with 24h expiry", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "auth-constraint", type: "constraint", content: "Must use bcrypt for password hashing", verified: true, createdAt: Date.now(), updatedAt: Date.now() });
+
+  graph.addEdge({ from: taskId, to: "auth-output", type: "produces" });
+  graph.addEdge({ from: taskId, to: "auth-decision", type: "produces" });
+  graph.addEdge({ from: "auth-evidence", to: "auth-output", type: "supports" });
+  graph.addEdge({ from: taskId, to: "auth-constraint", type: "requires_decision" });
+
+  for (let i = 0; i < 95; i++) {
+    graph.addNode({ id: `irr-${i}`, type: "observation", content: `Irrelevant observation ${i}: The weather is nice today`, verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  }
+
+  return {
+    name: "noise",
+    graph,
+    taskId,
+    expected: {
+      requiredNodes: [taskId, "auth-output", "auth-evidence", "auth-decision", "auth-constraint"],
+      missingDeps: [],
+    },
+  };
+}
+
+function buildHiddenDependencyScenario() {
+  const graph = new ContextGraph();
+  const taskId = "task-deploy";
+
+  graph.addNode({ id: "db-setup", type: "task", content: "Configure PostgreSQL connection pool", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "db-output", type: "output", content: "PostgreSQL connection string and pool config", verified: true, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addEdge({ from: "db-setup", to: "db-output", type: "produces" });
+
+  graph.addNode({ id: "api-task", type: "task", content: "Create REST endpoints for user management", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "api-output", type: "output", content: "Express routes with authentication middleware", verified: true, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addEdge({ from: "api-task", to: "api-output", type: "produces" });
+  graph.addEdge({ from: "api-task", to: "db-output", type: "requires_output" });
+
+  graph.addNode({ id: taskId, type: "task", content: "Deploy application to Kubernetes cluster", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addEdge({ from: taskId, to: "api-output", type: "requires_output" });
+
+  return {
+    name: "hidden-dependency",
+    graph,
+    taskId,
+    expected: {
+      requiredNodes: [taskId, "api-output", "db-output"],
+      missingDeps: [],
+    },
+  };
+}
+
+function buildContradictoryEvidenceScenario() {
+  const graph = new ContextGraph();
+  const taskId = "task-test";
+
+  graph.addNode({ id: taskId, type: "task", content: "Verify feature works", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "feature-output", type: "output", content: "Feature implementation", verified: true, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "evidence-pass", type: "evidence", content: "Unit test passed locally", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "evidence-fail", type: "evidence", content: "Integration test failed on CI", verified: true, createdAt: Date.now(), updatedAt: Date.now() });
+
+  graph.addEdge({ from: taskId, to: "feature-output", type: "requires_output" });
+  graph.addEdge({ from: taskId, to: "evidence-pass", type: "requires_evidence" });
+  graph.addEdge({ from: taskId, to: "evidence-fail", type: "requires_evidence" });
+  graph.addEdge({ from: "evidence-pass", to: "evidence-fail", type: "contradicts" });
+
+  return {
+    name: "contradictory-evidence",
+    graph,
+    taskId,
+    expected: {
+      requiredNodes: [taskId, "feature-output", "evidence-pass", "evidence-fail"],
+      missingDeps: [],
+    },
+  };
+}
+
+function buildStaleEvidenceScenario() {
+  const graph = new ContextGraph();
+  const taskId = "task-data";
+
+  graph.addNode({ id: "old-decision", type: "decision", content: "Use MongoDB for data storage", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "new-evidence", type: "evidence", content: "MongoDB performance degrades at scale", verified: true, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: taskId, type: "task", content: "Design data layer", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+
+  graph.addEdge({ from: "new-evidence", to: "old-decision", type: "invalidates" });
+  graph.addEdge({ from: taskId, to: "old-decision", type: "requires_decision" });
+
+  return {
+    name: "stale-evidence",
+    graph,
+    taskId,
+    expected: {
+      requiredNodes: [taskId, "old-decision", "new-evidence"],
+      missingDeps: [],
+    },
+  };
+}
+
+function buildMissingDependencyScenario() {
+  const graph = new ContextGraph();
+  const taskId = "task-missing";
+
+  graph.addNode({ id: taskId, type: "task", content: "Build feature that depends on missing service", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+
+  // Add edge manually to non-existent node
+  const edges = graph.edgesFrom.get(taskId) || [];
+  edges.push({ from: taskId, to: "output-missing", type: "requires_output", weight: 1.0 });
+  graph.edgesFrom.set(taskId, edges);
+  const toEdges = graph.edgesTo.get("output-missing") || [];
+  toEdges.push({ from: taskId, to: "output-missing", type: "requires_output", weight: 1.0 });
+  graph.edgesTo.set("output-missing", toEdges);
+
+  return {
+    name: "missing-dependency",
+    graph,
+    taskId,
+    expected: {
+      requiredNodes: [taskId],
+      missingDeps: ["output-missing"],
+    },
+  };
+}
+
+function buildPlaywrightChromiumScenario() {
+  const graph = new ContextGraph();
+  const taskId = "task-scrape";
+
+  graph.addNode({ id: taskId, type: "task", content: "Scrape Lider website with Playwright", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "browser-output", type: "output", content: "Scraped product data from Lider", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "chromium-dep", type: "artifact", content: "Chromium browser binary installed", verified: false, createdAt: Date.now(), updatedAt: Date.now() });
+  graph.addNode({ id: "playwright-dep", type: "artifact", content: "Playwright npm package installed", verified: true, createdAt: Date.now(), updatedAt: Date.now() });
+
+  graph.addEdge({ from: taskId, to: "browser-output", type: "produces" });
+  graph.addEdge({ from: taskId, to: "chromium-dep", type: "depends_on_artifact" });
+  graph.addEdge({ from: taskId, to: "playwright-dep", type: "depends_on_artifact" });
+
+  return {
+    name: "playwright-chromium",
+    graph,
+    taskId,
+    expected: {
+      requiredNodes: [taskId, "browser-output", "chromium-dep", "playwright-dep"],
+      missingDeps: [],
+    },
+  };
+}
+
+const SCENARIOS = [
+  buildNoiseScenario,
+  buildHiddenDependencyScenario,
+  buildContradictoryEvidenceScenario,
+  buildStaleEvidenceScenario,
+  buildMissingDependencyScenario,
+  buildPlaywrightChromiumScenario,
+];
+
+// -- Benchmark runner --
+
+function runScenarioWithStrategy(scenarioBuilder, strategyName, strategyFn, root) {
+  const { name, graph, taskId, expected } = scenarioBuilder();
+
+  // Create task state on disk
+  const dir = path.join(root || process.cwd(), ".wam", "tasks", taskId);
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(
+    path.join(dir, "state.yaml"),
+    JSON.stringify({
+      phase: "IMPLEMENTING",
+      lastAction: `Test: ${name}`,
+      contract: { objective: `Test ${name} scenario` },
+      requirements: [{ id: "req-1", title: "Test", status: "pending", evidence: [] }],
+      executions: [],
+    }, null, 2)
+  );
+
+  const result = strategyFn(taskId, graph, root);
+  const metrics = computeMetrics(result, expected, graph);
+
+  return {
+    scenario: name,
+    strategy: strategyName,
+    ...metrics,
+  };
+}
+
+export function runEvaluation(root) {
+  const strategies = [
+    { name: "current-context-pack", fn: (taskId, graph, root) => getCurrentContextPack(taskId, root) },
+    { name: "shadow-routing", fn: (taskId, graph, root) => getProposedContext(taskId, graph, root) },
+  ];
+
+  const results = [];
+  for (const strategy of strategies) {
+    for (const scenarioBuilder of SCENARIOS) {
+      const result = runScenarioWithStrategy(scenarioBuilder, strategy.name, strategy.fn, root);
+      results.push(result);
+    }
+  }
+
+  // Summary
+  const summary = {};
+  for (const r of results) {
+    if (!summary[r.strategy]) {
+      summary[r.strategy] = { totalTokens: 0, successes: 0, total: 0, avgRecall: 0 };
+    }
+    summary[r.strategy].totalTokens += r.contextTokens;
+    summary[r.strategy].total++;
+    if (r.taskSuccess) summary[r.strategy].successes++;
+    summary[r.strategy].avgRecall += r.contextRecall;
+  }
+
+  for (const key of Object.keys(summary)) {
+    summary[key].avgRecall /= summary[key].total;
+  }
+
+  return { results, summary };
+}

package/src/context/context-routing-observability.js

@@ -0,0 +1,296 @@
+/**
+ * WAM Context Routing — Observability
+ *
+ * Exposes why WAM included, excluded, deferred or rejected context during
+ * context routing. Every routing decision must be explainable.
+ *
+ * Decision types:
+ * - included
+ * - excluded
+ * - dependency-required
+ * - evidence-required
+ * - superseded
+ * - stale
+ * - invalidated
+ * - budget-exceeded
+ * - low-relevance
+ * - missing
+ */
+
+import { ContextGraph } from "./context-graph.js";
+import { getHistoricalCandidates } from "./context-history-relevance.js";
+import { getDependencies, detectContextGaps } from "../state/task-dependencies.js";
+
+/**
+ * @typedef {"included"|"excluded"|"dependency-required"|"evidence-required"|"superseded"|"stale"|"invalidated"|"budget-exceeded"|"low-relevance"|"missing"} RoutingDecisionType
+ */
+
+/**
+ * @typedef {Object} RoutingDecision
+ * @property {string} itemId
+ * @property {RoutingDecisionType} type
+ * @property {string} reason
+ * @property {string} [supersededBy]
+ * @property {string} [requiredBy]
+ */
+
+export function explainRoutingDecision(item, context, graph) {
+  const decisions = [];
+
+  // Check if item was included
+  if (context.included?.includes(item.id)) {
+    const reason = explainInclusion(item, context, graph);
+    decisions.push({
+      itemId: item.id,
+      type: "included",
+      reason,
+    });
+    return decisions;
+  }
+
+  // Check various exclusion reasons
+  if (item.status === "superseded") {
+    decisions.push({
+      itemId: item.id,
+      type: "superseded",
+      reason: `Superseded by ${item.supersededBy || "newer version"}`,
+      supersededBy: item.supersededBy,
+    });
+  } else if (item.status === "stale") {
+    decisions.push({
+      itemId: item.id,
+      type: "stale",
+      reason: "No longer relevant to current task",
+    });
+  } else if (item.status === "invalidated") {
+    decisions.push({
+      itemId: item.id,
+      type: "invalidated",
+      reason: item.invalidationReason || "Evidence or output was invalidated",
+    });
+  } else if (context.excluded?.includes(item.id)) {
+    const reason = explainExclusion(item, context, graph);
+    decisions.push({
+      itemId: item.id,
+      type: reason.type,
+      reason: reason.description,
+    });
+  } else if (
+    context.missing &&
+    context.missing.some(
+      (g) =>
+        g.requiredBy === item.id ||
+        (typeof g.description === "string" && g.description.includes(item.id)) ||
+        (g.type === "missing_dependency" && g.requiredBy)
+    )
+  ) {
+    const gap = context.missing.find(
+      (g) =>
+        g.requiredBy === item.id ||
+        (typeof g.description === "string" && g.description.includes(item.id)) ||
+        (g.type === "missing_dependency" && g.requiredBy)
+    );
+    decisions.push({
+      itemId: item.id,
+      type: "missing",
+      reason: gap.description || "Required but missing",
+      requiredBy: gap.requiredBy,
+    });
+  } else if (!context.included?.includes(item.id) && context.included) {
+    decisions.push({
+      itemId: item.id,
+      type: "low-relevance",
+      reason: `Relevance score ${item.relevanceScore || 0} below threshold`,
+    });
+  }
+
+  return decisions;
+}
+
+function explainInclusion(item, context, graph) {
+  if (item.dependencyOf) {
+    return `Required by ${item.dependencyOf}`;
+  }
+  if (item.supports) {
+    return `Supports evidence ${item.supports}`;
+  }
+  if (item.type === "decision") {
+    return "Active decision affecting current task";
+  }
+  if (item.type === "requirement" && item.status === "unresolved") {
+    return "Unresolved requirement from current task";
+  }
+  if (item.relevanceScore > 0.7) {
+    return `High relevance score: ${item.relevanceScore.toFixed(2)}`;
+  }
+  return "Included based on routing rules";
+}
+
+function explainExclusion(item, context, graph) {
+  // Check if required but missing
+  const gaps = context.missing || [];
+  const itemId = item.id;
+  const missingGap = gaps.find(
+    (g) =>
+      g.requiredBy === itemId ||
+      (typeof g.description === "string" && g.description.includes(itemId)) ||
+      (g.type === "missing_dependency" && g.requiredBy)
+  );
+  if (missingGap) {
+    return { type: "missing", description: missingGap.description };
+  }
+
+  // Check budget
+  if (context.tokenBudget && context.totalTokens > context.tokenBudget) {
+    return { type: "budget-exceeded", description: "Token budget exceeded" };
+  }
+
+  // Check if superseded
+  if (item.status === "superseded") {
+    return { type: "superseded", description: `Superseded by ${item.supersededBy || "newer version"}` };
+  }
+
+  // Check if stale
+  if (item.status === "stale") {
+    return { type: "stale", description: "No longer relevant" };
+  }
+
+  // Check if invalidated
+  if (item.status === "invalidated") {
+    return { type: "invalidated", description: "Content was invalidated" };
+  }
+
+  // Default: low relevance
+  return { type: "low-relevance", description: `Relevance score below threshold` };
+}
+
+export function explainContextRouting(taskId, graph, context, options = {}) {
+  const decisions = [];
+  const { includeExcluded = true, includeIncluded = true } = options;
+
+  // Process included items
+  if (includeIncluded && context.included) {
+    for (const itemId of context.included) {
+      const item = findItem(itemId, context, graph);
+      if (item) {
+        decisions.push(...explainRoutingDecision(item, context, graph));
+      }
+    }
+  }
+
+  // Process excluded items
+  if (includeExcluded && context.excluded) {
+    for (const itemId of context.excluded) {
+      const item = findItem(itemId, context, graph);
+      if (item) {
+        decisions.push(...explainRoutingDecision(item, context, graph));
+      }
+    }
+  }
+
+  // Process missing items
+  if (context.missing) {
+    for (const gap of context.missing) {
+      decisions.push({
+        itemId: gap.type,
+        type: "missing",
+        reason: gap.description,
+        requiredBy: gap.requiredBy,
+      });
+    }
+  }
+
+  return decisions;
+}
+
+function findItem(itemId, context, graph) {
+  // Check included items
+  if (context.includedItems) {
+    const item = context.includedItems.find((i) => i.id === itemId);
+    if (item) return item;
+  }
+
+  // Check graph
+  if (graph) {
+    const node = graph.getNode(itemId);
+    if (node) return node;
+  }
+
+  // Return minimal object
+  return { id: itemId };
+}
+
+export function formatRoutingReport(decisions) {
+  const byType = {
+    included: [],
+    excluded: [],
+    missing: [],
+    superseded: [],
+    stale: [],
+    invalidated: [],
+    budgetExceeded: [],
+    lowRelevance: [],
+  };
+
+  for (const d of decisions) {
+    if (d.type === "included") byType.included.push(d);
+    else if (d.type === "missing") byType.missing.push(d);
+    else if (d.type === "superseded") byType.superseded.push(d);
+    else if (d.type === "stale") byType.stale.push(d);
+    else if (d.type === "invalidated") byType.invalidated.push(d);
+    else if (d.type === "budget-exceeded") byType.budgetExceeded.push(d);
+    else byType.excluded.push(d);
+  }
+
+  const lines = ["# Routing Decision Report", ""];
+
+  if (byType.included.length > 0) {
+    lines.push("## Included");
+    for (const d of byType.included) {
+      lines.push(`- ${d.itemId}: ${d.reason}`);
+    }
+    lines.push("");
+  }
+
+  if (byType.missing.length > 0) {
+    lines.push("## Missing");
+    for (const d of byType.missing) {
+      lines.push(`- ${d.itemId}: ${d.reason}`);
+    }
+    lines.push("");
+  }
+
+  if (byType.excluded.length > 0) {
+    lines.push("## Excluded");
+    for (const d of byType.excluded) {
+      lines.push(`- ${d.itemId}: ${d.reason}`);
+    }
+    lines.push("");
+  }
+
+  if (byType.superseded.length > 0) {
+    lines.push("## Superseded");
+    for (const d of byType.superseded) {
+      lines.push(`- ${d.itemId}: ${d.reason}`);
+    }
+    lines.push("");
+  }
+
+  if (byType.stale.length > 0) {
+    lines.push("## Stale");
+    for (const d of byType.stale) {
+      lines.push(`- ${d.itemId}: ${d.reason}`);
+    }
+    lines.push("");
+  }
+
+  if (byType.invalidated.length > 0) {
+    lines.push("## Invalidated");
+    for (const d of byType.invalidated) {
+      lines.push(`- ${d.itemId}: ${d.reason}`);
+    }
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}