@bugmole/cli 0.4.0

package/src/runtime/pipeline.test.ts

@@ -0,0 +1,405 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+  derivePipelineState,
+  pipelineIdempotencyKey,
+  plannedJourneyIds,
+  type PipelineInputs,
+} from "./pipeline.js";
+
+const canonicalJourney = {
+  id: "flow:login",
+  actor: "user",
+  revision: 2,
+  nodeIds: ["screen:/", "screen:/login"],
+  stability: "stable",
+  linkedToGraph: true,
+};
+
+function inputs(overrides: Partial<PipelineInputs> = {}): PipelineInputs {
+  return {
+    projectId: "bugmole",
+    journeys: [canonicalJourney],
+    knownActors: ["user"],
+    caseSetJourneyIds: [],
+    plannedJourneyIds: [],
+    tasks: [],
+    hardStopCategories: [],
+    recordedBlockerCategories: [],
+    ...overrides,
+  };
+}
+
+test("with nothing discovered yet, the first thing to do is explore", () => {
+  const state = derivePipelineState(inputs({ journeys: [] }));
+  assert.equal(state.status, "running");
+  assert.equal(state.next?.stage, "explore");
+  assert.equal(state.next?.idempotencyKey, pipelineIdempotencyKey("bugmole", "explore"));
+});
+
+test("after exploration it moves on by itself — this is the stall being fixed", () => {
+  const state = derivePipelineState(inputs());
+  assert.equal(state.next?.stage, "audit");
+  assert.equal(state.next?.journeyId, "flow:login");
+});
+
+test("the audit task is what wires up the Storybook scan that populates the component library", () => {
+  const state = derivePipelineState(inputs());
+  assert.equal(state.next?.stage, "audit");
+  assert.match(state.next!.description, /bugmole_ui_re_start/);
+  assert.match(state.next!.description, /bugmole_ui_re_stage/);
+  assert.match(state.next!.description, /atoms/);
+  // It must not invent a catalogue when the project has no Storybook.
+  assert.match(state.next!.description, /blocked reason/i);
+});
+
+test("test cases only come after the audit, so a case is written against known components", () => {
+  const state = derivePipelineState(inputs({ auditedJourneyIds: ["flow:login"] }));
+  assert.equal(state.next?.stage, "testcases");
+});
+
+test("an actor missing from roles.yaml is reconciled instead of dead-ending in the planner", () => {
+  const state = derivePipelineState(inputs({ knownActors: ["owner"] }));
+  assert.equal(state.next?.stage, "roles");
+  assert.match(state.next!.description, /user/);
+});
+
+test("once cases exist the flow gets planned", () => {
+  const state = derivePipelineState(inputs({
+    auditedJourneyIds: ["flow:login"],
+    caseSetJourneyIds: ["flow:login"],
+  }));
+  assert.equal(state.next?.stage, "plan");
+});
+
+test("a fully satisfied project is complete and queues nothing further", () => {
+  const state = derivePipelineState(inputs({
+    auditedJourneyIds: ["flow:login"],
+    caseSetJourneyIds: ["flow:login"],
+    plannedJourneyIds: ["flow:login"],
+    publishedPlans: [{ planId: "flow_login", journeyId: "flow:login" }],
+    ranJourneyIds: ["flow:login"],
+    verdictJourneyIds: ["flow:login"],
+  }));
+  assert.equal(state.status, "complete");
+  assert.equal(state.next, undefined);
+  assert.equal(state.nextRun, undefined);
+});
+
+test("only one stage is queued at a time, so stages cannot race", () => {
+  const state = derivePipelineState(inputs({
+    tasks: [{ status: "running", idempotencyKey: pipelineIdempotencyKey("bugmole", "explore") }],
+  }));
+  assert.equal(state.status, "running");
+  assert.equal(state.next, undefined);
+});
+
+test("cancelling stops the whole pipeline, not just the one task", () => {
+  const state = derivePipelineState(inputs({
+    tasks: [{
+      status: "error",
+      idempotencyKey: pipelineIdempotencyKey("bugmole", "explore"),
+      completionSummary: "Cancelled",
+    }],
+  }));
+  assert.equal(state.status, "stopped");
+  assert.equal(state.next, undefined);
+  assert.match(state.reason, /cancelled/i);
+});
+
+test("an unrelated cancelled task does not stop the pipeline", () => {
+  const state = derivePipelineState(inputs({
+    tasks: [{ status: "error", idempotencyKey: "manual-thing", completionSummary: "Cancelled" }],
+  }));
+  assert.equal(state.status, "running");
+  assert.equal(state.next?.stage, "audit");
+});
+
+test("a recorded hard-stop blocker terminates the run with a visible reason", () => {
+  const state = derivePipelineState(inputs({
+    hardStopCategories: ["secret_value_requested"],
+    recordedBlockerCategories: ["secret_value_requested"],
+  }));
+  assert.equal(state.status, "stopped");
+  assert.match(state.reason, /secret_value_requested/);
+});
+
+test("a soft blocker does not stop the pipeline", () => {
+  const state = derivePipelineState(inputs({
+    hardStopCategories: ["secret_value_requested"],
+    recordedBlockerCategories: ["supabase_schema_missing"],
+  }));
+  assert.equal(state.status, "running");
+});
+
+test("an unanswered clarification pauses rather than guessing", () => {
+  const state = derivePipelineState(inputs({
+    pendingClarification: "Which account should the owner flow use?",
+  }));
+  assert.equal(state.status, "paused");
+  assert.equal(state.next, undefined);
+  assert.match(state.reason, /Which account/);
+});
+
+test("a discovered-but-unstabilised flow is not treated as ready to plan", () => {
+  const state = derivePipelineState(inputs({
+    journeys: [{ ...canonicalJourney, stability: "discovered", linkedToGraph: false }],
+  }));
+  // Explore is satisfied (the flow has nodes) but nothing downstream acts
+  // on a flow the graph does not consider canonical yet.
+  assert.notEqual(state.next?.stage, "plan");
+});
+
+test("stage keys are stable, so the agent's own enqueue and the reconciler converge", () => {
+  const a = derivePipelineState(inputs()).next!.idempotencyKey;
+  const b = derivePipelineState(inputs()).next!.idempotencyKey;
+  assert.equal(a, b);
+  assert.equal(a, pipelineIdempotencyKey("bugmole", "audit", "flow:login"));
+});
+
+test("a planned flow gets run — without this every case stays an unproven assumption", () => {
+  const state = derivePipelineState(inputs({
+    auditedJourneyIds: ["flow:login"],
+    caseSetJourneyIds: ["flow:login"],
+    plannedJourneyIds: ["flow:login"],
+    publishedPlans: [{ planId: "flow_login", journeyId: "flow:login" }],
+    environmentId: "local",
+  }));
+  assert.equal(state.status, "running");
+  assert.equal(state.nextRun?.stage, "run");
+  assert.equal(state.nextRun?.planId, "flow_login");
+  assert.equal(state.nextRun?.environmentId, "local");
+});
+
+test("a project is only complete once its plans have run AND judged their cases", () => {
+  const base = {
+    auditedJourneyIds: ["flow:login"],
+    caseSetJourneyIds: ["flow:login"],
+    plannedJourneyIds: ["flow:login"],
+    publishedPlans: [{ planId: "flow_login", journeyId: "flow:login" }],
+  };
+  assert.notEqual(derivePipelineState(inputs(base)).status, "complete");
+  // A run artifact on its own is not enough. This is the exact shape the
+  // real project was in: 20/20 flows "executed" while all 118 cases still
+  // had no status, and the pipeline called itself finished.
+  const ranOnly = derivePipelineState(inputs({ ...base, ranJourneyIds: ["flow:login"] }));
+  assert.notEqual(ranOnly.status, "complete", "running without recording verdicts is not coverage");
+  const done = derivePipelineState(inputs({
+    ...base,
+    ranJourneyIds: ["flow:login"],
+    verdictJourneyIds: ["flow:login"],
+  }));
+  assert.equal(done.status, "complete");
+});
+
+test("a flow that ran but judged nothing is queued to run again, not left as covered", () => {
+  const state = derivePipelineState(inputs({
+    auditedJourneyIds: ["flow:login"],
+    caseSetJourneyIds: ["flow:login"],
+    plannedJourneyIds: ["flow:login"],
+    publishedPlans: [{ planId: "flow_login", journeyId: "flow:login" }],
+    ranJourneyIds: ["flow:login"],
+  }));
+  assert.equal(state.status, "running");
+  assert.equal(state.nextRun?.stage, "run");
+  assert.equal(state.nextRun?.journeyId, "flow:login");
+});
+
+test("the run stage says how many flows ran but recorded nothing", () => {
+  const state = derivePipelineState(inputs({
+    auditedJourneyIds: ["flow:login"],
+    caseSetJourneyIds: ["flow:login"],
+    plannedJourneyIds: ["flow:login"],
+    publishedPlans: [{ planId: "flow_login", journeyId: "flow:login" }],
+    ranJourneyIds: ["flow:login"],
+  }));
+  const run = state.stages.find((stage) => stage.stage === "run")!;
+  assert.equal(run.satisfied, false);
+  assert.match(run.detail, /ran but recorded no verdicts/);
+});
+
+test("a judged flow with no registry run record does not report a negative count", () => {
+  // verdicts can be recorded by something other than a registry run, so
+  // ran minus judged went negative and printed "-1 recorded no verdicts".
+  const state = derivePipelineState(inputs({
+    auditedJourneyIds: ["flow:login"],
+    caseSetJourneyIds: ["flow:login"],
+    plannedJourneyIds: ["flow:login"],
+    publishedPlans: [{ planId: "flow_login", journeyId: "flow:login" }],
+    ranJourneyIds: [],
+    verdictJourneyIds: ["flow:login"],
+  }));
+  const run = state.stages.find((stage) => stage.stage === "run")!;
+  assert.equal(run.satisfied, true);
+  assert.doesNotMatch(run.detail, /-\d/, run.detail);
+});
+
+test("a plan that was never published pauses with a reason instead of idling silently", () => {
+  const state = derivePipelineState(inputs({
+    auditedJourneyIds: ["flow:login"],
+    caseSetJourneyIds: ["flow:login"],
+    plannedJourneyIds: ["flow:login"],
+    publishedPlans: [],
+  }));
+  assert.equal(state.status, "paused");
+  assert.match(state.reason, /no published plan/);
+});
+
+test("artifacts belonging to deleted flows do not count as progress", () => {
+  // Regenerating the graph renames journeys, orphaning everything written
+  // against the old ids. Counting those raw reported "4/7 flows have test
+  // cases" when no live flow had any, and marked planning satisfied while
+  // nothing was planned — so the stage was skipped entirely.
+  const state = derivePipelineState(inputs({
+    caseSetJourneyIds: ["flow:deleted-a", "flow:deleted-b"],
+    auditedJourneyIds: ["flow:deleted-a"],
+    plannedJourneyIds: ["flow:deleted-a"],
+  }));
+  const byStage = new Map(state.stages.map((stage) => [stage.stage, stage]));
+  assert.equal(byStage.get("testcases")!.satisfied, false);
+  assert.equal(byStage.get("testcases")!.detail, "0/1 flows have test case assumptions");
+  assert.equal(byStage.get("plan")!.satisfied, false, "planning must not look done when nothing live is planned");
+  assert.equal(byStage.get("audit")!.satisfied, false);
+});
+
+test("orphaned artifacts are reported, so a skewed-looking stage is explainable", () => {
+  const state = derivePipelineState(inputs({
+    caseSetJourneyIds: ["flow:login", "flow:deleted"],
+    plannedJourneyIds: ["flow:gone"],
+  }));
+  const kinds = state.orphanedArtifacts.map((entry) => `${entry.kind}:${entry.journeyId}`).sort();
+  assert.deepEqual(kinds, ["plan:flow:gone", "test cases:flow:deleted"]);
+  // The live flow's own case set still counts.
+  const testcases = state.stages.find((stage) => stage.stage === "testcases")!;
+  assert.equal(testcases.detail, "1/1 flows have test case assumptions");
+});
+
+test("a project with nothing orphaned reports an empty list", () => {
+  const state = derivePipelineState(inputs({ caseSetJourneyIds: ["flow:login"] }));
+  assert.deepEqual(state.orphanedArtifacts, []);
+});
+
+test("a stage that gave up asks for your input instead of being re-queued forever", () => {
+  // A blocked task with its attempts spent is neither in flight nor
+  // terminal, so the reconciler used to keep enqueueing it — getting the
+  // same task back every few seconds, making no progress and saying
+  // nothing. The one thing that can unblock it is an answer.
+  const state = derivePipelineState(inputs({
+    tasks: [{
+      status: "blocked",
+      idempotencyKey: pipelineIdempotencyKey("bugmole", "audit", "flow:login"),
+      title: "Audit the controls and components of flow:login",
+      attempts: 3,
+      errorMessage: "Needs a login for the owner account",
+    }],
+  }));
+  assert.equal(state.status, "paused");
+  assert.equal(state.next, undefined, "must not queue more work while stuck");
+  assert.match(state.reason, /need your input/);
+  assert.match(state.reason, /Needs a login for the owner account/);
+  assert.equal(state.blockedTasks?.length, 1);
+});
+
+test("a stage still inside its retry budget keeps going on its own", () => {
+  const state = derivePipelineState(inputs({
+    tasks: [{
+      status: "blocked",
+      idempotencyKey: pipelineIdempotencyKey("bugmole", "audit", "flow:login"),
+      attempts: 1,
+    }],
+  }));
+  // Retrying is the pipeline's job; only a spent budget is the user's.
+  assert.notEqual(state.status, "paused");
+  assert.equal(state.next?.stage, "audit");
+});
+
+test("work continues on other flows only once the stuck one is resolved", () => {
+  const stuck = {
+    status: "blocked",
+    idempotencyKey: pipelineIdempotencyKey("bugmole", "audit", "flow:login"),
+    title: "Audit flow:login",
+    attempts: 3,
+  };
+  assert.equal(derivePipelineState(inputs({ tasks: [stuck] })).status, "paused");
+  // Cleared (completed by a human or retried successfully) → back to work.
+  const resumed = derivePipelineState(inputs({
+    tasks: [{ ...stuck, status: "completed" }],
+    auditedJourneyIds: ["flow:login"],
+  }));
+  assert.equal(resumed.status, "running");
+  assert.equal(resumed.next?.stage, "testcases");
+});
+
+const runnable = {
+  auditedJourneyIds: ["flow:login"],
+  caseSetJourneyIds: ["flow:login"],
+  plannedJourneyIds: ["flow:login"],
+  publishedPlans: [{ planId: "flow_login", journeyId: "flow:login" }],
+};
+
+test("a finished run does not bind the flow to the same run forever", () => {
+  // The run key used to be stable per flow, so once any run existed the
+  // registry handed that one back on every reconcile. A finished run from
+  // hours earlier was returned each poll: the pipeline logged "started run"
+  // indefinitely while nothing executed and no verdict was ever recorded.
+  const first = derivePipelineState(inputs(runnable));
+  const afterOneRun = derivePipelineState(inputs({
+    ...runnable,
+    runs: [{ journeyId: "flow:login", status: "partial" }],
+  }));
+  assert.ok(first.nextRun);
+  assert.ok(afterOneRun.nextRun);
+  assert.notEqual(
+    afterOneRun.nextRun!.idempotencyKey,
+    first.nextRun!.idempotencyKey,
+    "a fresh attempt needs a key the finished run does not already own",
+  );
+});
+
+test("a run already in flight is not started again", () => {
+  const state = derivePipelineState(inputs({
+    ...runnable,
+    runs: [{ journeyId: "flow:login", status: "running" }],
+  }));
+  assert.equal(state.status, "running");
+  assert.equal(state.nextRun, undefined, "starting a second run would duplicate the work");
+  assert.match(state.reason, /already in flight/);
+});
+
+test("a flow that keeps running without recording verdicts stops instead of looping", () => {
+  const state = derivePipelineState(inputs({
+    ...runnable,
+    runs: Array.from({ length: 3 }, () => ({ journeyId: "flow:login", status: "success" })),
+  }));
+  assert.equal(state.status, "paused");
+  assert.equal(state.nextRun, undefined);
+  assert.match(state.reason, /without recording any verdict/);
+});
+
+test("the in-flight key is stable, so repeated reconciles do not duplicate a queued run", () => {
+  const a = derivePipelineState(inputs(runnable)).nextRun!.idempotencyKey;
+  const b = derivePipelineState(inputs(runnable)).nextRun!.idempotencyKey;
+  assert.equal(a, b);
+});
+
+test("a web plan pinned to the mobile driver does not count as planned", async () => {
+  // 18 plans were written with driver "maestro" against platform "web".
+  // Every run of them failed identically without checking anything, yet the
+  // plan stage read 20/20 satisfied — so the pipeline never re-planned them.
+  const fs = await import("node:fs/promises");
+  const os = await import("node:os");
+  const path = await import("node:path");
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-plans-"));
+  try {
+    const plansDir = path.join(root, "plans");
+    await fs.mkdir(plansDir, { recursive: true });
+    await fs.writeFile(path.join(plansDir, "flow_web.flow.yaml"), "driver: maestro\nplatform: web\n");
+    await fs.writeFile(path.join(plansDir, "flow_ok.flow.yaml"), "driver: playwright\nplatform: web\n");
+    await fs.writeFile(path.join(plansDir, "flow_mobile.flow.yaml"), "driver: maestro\nplatform: ios\n");
+
+    const planned = plannedJourneyIds(root, ["flow:web", "flow:ok", "flow:mobile"]);
+    assert.deepEqual(planned.sort(), ["flow:mobile", "flow:ok"]);
+  } finally {
+    await fs.rm(root, { recursive: true, force: true });
+  }
+});