@hasna/loops 0.4.26 → 0.4.28

package/dist/lib/store.d.ts

@@ -133,6 +133,8 @@ export interface WorkflowWorkItemRow {
     priority: number;
     status: string;
     attempts: number;
+    /** Nullable for rows read before migration 0011 backfills the column. */
+    gate_deaths: number | null;
     next_attempt_at: string | null;
     lease_expires_at: string | null;
     workflow_id: string | null;
@@ -368,6 +370,49 @@ export interface RecordGoalEventInput {
     rawResponse?: unknown;
 }
 export declare function workItemStatusForLoopRun(status: RunStatus, attempt: number, maxAttempts: number | undefined): WorkflowWorkItemStatus | undefined;
+/**
+ * `exit(75)` = `EX_TEMPFAIL`: the sysexits.h "temporary failure, retry later"
+ * code. A gate/worker step that exits 75 (e.g. an account-quota probe that is
+ * still dry) is signalling "not now", not a real failed attempt — so it must
+ * not burn the todos-task redispatch cap and must leave the work item
+ * requeueable rather than persisting as a terminal, dedupe-forever row.
+ */
+export declare const WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
+/**
+ * Workflow step ids that run BEFORE the worker does any real work. A failure in
+ * one of these (triage/planner gate, or the pre-step worktree preparation) that
+ * dies quickly is a "gate death": the run never executed the worker, so it must
+ * not count toward the redispatch cap (otherwise a purely infrastructural fault
+ * — e.g. a stale worktree registration — silently dead-letters a task that
+ * never actually ran).
+ */
+export declare const GATE_STEP_IDS: ReadonlySet<string>;
+/** A gate-step failure only counts as a gate death when it dies before any real
+ *  work could have happened. Worktree-prep deaths are always gate deaths (they
+ *  fail before the agent is spawned) regardless of this bound. */
+export declare const GATE_DEATH_MAX_DURATION_MS = 60000;
+/**
+ * Secondary ceiling for CONSECUTIVE gate deaths. Gate deaths refund their
+ * redispatch attempt (the worker never ran), which is correct for transient
+ * infrastructure faults — but a deterministic fault (e.g. a permanently broken
+ * repo path) would otherwise retry forever at the backoff floor. After this
+ * many consecutive gate deaths the work item is dead-lettered (visible in
+ * drain reports) instead of spinning; any run that reaches the worker resets
+ * the streak, and an operator requeue (attempts reset) re-arms it. At the
+ * ~2–4 minute refunded-attempts backoff this bounds a deterministic fault to
+ * roughly an hour of auto-retry before it demands an operator.
+ */
+export declare const GATE_DEATH_CEILING = 20;
+export type NonProductiveFailureKind = "tempfail" | "gate-death";
+type ClassifiableStepRun = Pick<WorkflowStepRun, "stepId" | "status" | "exitCode" | "durationMs" | "error">;
+/**
+ * Classify a just-finalized *failed* workflow run's decisive failing step. A
+ * non-productive finish (a tempfail retry-signal, or a gate death before the
+ * worker ran) must not count toward the todos-task redispatch cap. Returns the
+ * non-productive kind, or `undefined` when the run represents a real worker
+ * attempt that legitimately counts toward the cap. Pure/exported for tests.
+ */
+export declare function classifyNonProductiveStepFailure(steps: ClassifiableStepRun[]): NonProductiveFailureKind | undefined;
 export declare function scrubbedOrNull(value: string | undefined | null): string | null;
 /** Scrub secrets then bound size before persisting run stdout/stderr. */
 export declare function persistedRunOutput(value: string | undefined | null): string | null;
@@ -490,9 +535,41 @@ export declare class Store {
      * step on that account right now — exactly the concurrency to bound.
      */
     countRunningWorkflowStepsByAuthProfile(): Record<string, number>;
+    /**
+     * Requeue a terminal admission work item for the next task/event delivery.
+     * By default `attempts` is preserved (used by the bounded stale-terminal
+     * re-admission on the route path, which must keep counting toward the cap).
+     * Pass `resetAttempts: true` for the operator unwedge (`loops routes requeue`)
+     * so a manual requeue is DURABLE rather than one-shot: without the reset a
+     * capped item that finishes terminal once more re-caps instantly.
+     */
     requeueWorkflowWorkItem(id: string, patch?: {
         reason?: string;
+        resetAttempts?: boolean;
+    }): WorkflowWorkItem;
+    /**
+     * Transition a terminal admission work item to `dead_letter`. Used by the
+     * route path when a still-actionable todos task has exhausted the redispatch
+     * cap: instead of silently deduping the same terminal row forever (the "black
+     * hole" — `considered=N created=0` with no signal), the item is moved to a
+     * visible `dead_letter` state so drain reports can surface + count it and an
+     * operator can `loops routes requeue` it. Idempotent: a no-op on an item that
+     * is already dead-lettered.
+     */
+    deadLetterWorkflowWorkItem(id: string, patch?: {
+        reason?: string;
     }): WorkflowWorkItem;
+    /**
+     * Refund a redispatch attempt for a *failed* run that never did real work.
+     * Called from {@link finalizeWorkflowRun} the moment a work item is set
+     * `failed`, so the todos-task redispatch cap only ever counts real worker
+     * attempts. A tempfail (`exit 75`) is additionally made requeueable (dropped
+     * back to `queued`, bindings cleared) so its "retry later" contract fires on
+     * the next drain instead of persisting as a terminal, dedupe-forever row. A
+     * gate death stays `failed` (the bounded re-admission picks it up after
+     * backoff once the underlying infra fault clears). Both floor attempts at 0.
+     */
+    private demoteNonProductiveWorkItems;
     admitWorkflowWorkItem(id: string, patch: {
         workflowId: string;
         loopId: string;

package/dist/lib/store.js

@@ -1569,6 +1569,7 @@ var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
 var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
 var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
 var SCHEMA_USER_VERSION = 8;
+var BREAKING_SCHEMA_FLOOR = 7;
 var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
 var PRUNE_BATCH_SIZE = 400;
 var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
@@ -1711,6 +1712,7 @@ function rowToWorkflowWorkItem(row) {
     priority: row.priority,
     status: row.status,
     attempts: row.attempts,
+    gateDeaths: row.gate_deaths ?? 0,
     nextAttemptAt: row.next_attempt_at ?? undefined,
     leaseExpiresAt: row.lease_expires_at ?? undefined,
     workflowId: row.workflow_id ?? undefined,
@@ -1860,6 +1862,23 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
   }
   return;
 }
+var WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
+var GATE_STEP_IDS = new Set(["triage", "planner", "plan"]);
+var GATE_DEATH_MAX_DURATION_MS = 60000;
+var GATE_DEATH_CEILING = 20;
+function classifyNonProductiveStepFailure(steps) {
+  const failing = [...steps].reverse().find((step) => step.status === "failed" || step.status === "timed_out");
+  if (!failing)
+    return;
+  if (failing.exitCode === WORK_ITEM_TEMPFAIL_EXIT_CODE)
+    return "tempfail";
+  if (typeof failing.error === "string" && failing.error.includes("worktree preparation failed"))
+    return "gate-death";
+  const fast = failing.durationMs === undefined || failing.durationMs < GATE_DEATH_MAX_DURATION_MS;
+  if (GATE_STEP_IDS.has(failing.stepId) && fast)
+    return "gate-death";
+  return;
+}
 function scrubbedOrNull(value) {
   return value == null ? null : scrubSecrets(value);
 }
@@ -1960,11 +1979,21 @@ class Store {
         id TEXT PRIMARY KEY,
         applied_at TEXT NOT NULL
       );
+      CREATE TABLE IF NOT EXISTS schema_compat (
+        id INTEGER PRIMARY KEY CHECK (id = 1),
+        min_compatible_user_version INTEGER NOT NULL
+      );
     `);
     const versionRow = this.db.query("PRAGMA user_version").get();
     const userVersion = versionRow?.user_version ?? 0;
     if (userVersion > SCHEMA_USER_VERSION) {
-      throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
+      const floorRow = this.db.query("SELECT min_compatible_user_version FROM schema_compat WHERE id = 1").get();
+      if (!floorRow) {
+        throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}) and carries no compatibility floor; upgrade open-loops before opening this database`);
+      }
+      if (SCHEMA_USER_VERSION < floorRow.min_compatible_user_version) {
+        throw new Error(`loops database schema version ${userVersion} requires a binary with schema support >= ${floorRow.min_compatible_user_version} (this binary supports ${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
+      }
     }
     const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
     for (const migration of this.migrations()) {
@@ -1975,8 +2004,10 @@ class Store {
         this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
       }
     }
-    if (userVersion !== SCHEMA_USER_VERSION)
+    if (userVersion < SCHEMA_USER_VERSION)
       this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
+    this.db.query(`INSERT INTO schema_compat (id, min_compatible_user_version) VALUES (1, ?)
+         ON CONFLICT(id) DO UPDATE SET min_compatible_user_version = MAX(min_compatible_user_version, excluded.min_compatible_user_version)`).run(BREAKING_SCHEMA_FLOOR);
   }
   migrations() {
     return [
@@ -2043,6 +2074,12 @@ class Store {
           this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
           this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
         }
+      },
+      {
+        id: "0011_work_item_gate_deaths",
+        apply: () => {
+          this.addColumnIfMissing("workflow_work_items", "gate_deaths", "INTEGER NOT NULL DEFAULT 0");
+        }
       }
     ];
   }
@@ -3173,7 +3210,7 @@ class Store {
   }
   upsertWorkflowWorkItem(input) {
     const now = nowIso();
-    const id = genId();
+    const id = input.id ?? genId();
     const status = input.status ?? "queued";
     this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
           subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
@@ -3301,6 +3338,7 @@ class Store {
     const placeholders = requeueableStatuses.map(() => "?").join(",");
     const res = this.db.query(`UPDATE workflow_work_items
          SET status='queued', workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
+          ${patch.resetAttempts ? "attempts=0, gate_deaths=0," : ""}
           next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
          WHERE id=? AND status IN (${placeholders})`).run(reason, now, id, ...requeueableStatuses);
     const item = this.getWorkflowWorkItem(id);
@@ -3310,6 +3348,46 @@ class Store {
       throw new Error(`workflow work item was not requeued: ${id} status=${item.status}`);
     return item;
   }
+  deadLetterWorkflowWorkItem(id, patch = {}) {
+    const now = nowIso();
+    const reason = patch.reason?.trim() || "redispatch cap reached; dead-lettered";
+    this.db.query(`UPDATE workflow_work_items
+         SET status='dead_letter', next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
+         WHERE id=? AND status IN ('succeeded','failed','cancelled')`).run(reason, now, id);
+    const item = this.getWorkflowWorkItem(id);
+    if (!item)
+      throw new Error(`workflow work item not found after dead-letter: ${id}`);
+    return item;
+  }
+  demoteNonProductiveWorkItems(workflowRunId, finishedAt) {
+    const kind = classifyNonProductiveStepFailure(this.listWorkflowStepRuns(workflowRunId));
+    if (!kind) {
+      this.db.query("UPDATE workflow_work_items SET gate_deaths=0, updated_at=? WHERE workflow_run_id=? AND status='failed' AND gate_deaths > 0").run(finishedAt, workflowRunId);
+      return;
+    }
+    if (kind === "tempfail") {
+      this.db.query(`UPDATE workflow_work_items
+           SET status='queued', attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
+            gate_deaths=0,
+            workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
+            next_attempt_at=NULL, lease_expires_at=NULL,
+            last_reason='worker exited 75 (tempfail): requeued for retry; attempt refunded (does not count toward redispatch cap)',
+            updated_at=?
+           WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
+      return;
+    }
+    this.db.query(`UPDATE workflow_work_items
+         SET attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
+          gate_deaths=gate_deaths + 1,
+          status=CASE WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING} THEN 'dead_letter' ELSE status END,
+          last_reason=CASE
+            WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING}
+              THEN 'gate-death ceiling reached (' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING} consecutive runs died at worktree prep / triage / planner without reaching the worker): dead-lettered \u2014 the infrastructure fault needs an operator; ''loops routes requeue'' resets and retries'
+            ELSE 'gate death before real work (worktree prep / triage / planner): attempt refunded (does not count toward redispatch cap); consecutive gate deaths: ' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING}'
+          END,
+          updated_at=?
+         WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
+  }
   admitWorkflowWorkItem(id, patch) {
     const now = nowIso();
     const res = this.db.query(`UPDATE workflow_work_items
@@ -4021,6 +4099,8 @@ class Store {
       if (changed) {
         const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
         this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
+        if (itemStatus === "failed")
+          this.demoteNonProductiveWorkItems(workflowRunId, finishedAt);
         this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
       }
       this.db.exec("COMMIT");
@@ -4984,5 +5064,10 @@ export {
   rowToGoalPlanNode,
   rowToGoal,
   persistedRunOutput,
-  Store
+  classifyNonProductiveStepFailure,
+  WORK_ITEM_TEMPFAIL_EXIT_CODE,
+  Store,
+  GATE_STEP_IDS,
+  GATE_DEATH_MAX_DURATION_MS,
+  GATE_DEATH_CEILING
 };

package/dist/mcp/index.js

@@ -1571,6 +1571,7 @@ var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
 var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
 var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
 var SCHEMA_USER_VERSION = 8;
+var BREAKING_SCHEMA_FLOOR = 7;
 var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
 var PRUNE_BATCH_SIZE = 400;
 var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
@@ -1713,6 +1714,7 @@ function rowToWorkflowWorkItem(row) {
     priority: row.priority,
     status: row.status,
     attempts: row.attempts,
+    gateDeaths: row.gate_deaths ?? 0,
     nextAttemptAt: row.next_attempt_at ?? undefined,
     leaseExpiresAt: row.lease_expires_at ?? undefined,
     workflowId: row.workflow_id ?? undefined,
@@ -1862,6 +1864,23 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
   }
   return;
 }
+var WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
+var GATE_STEP_IDS = new Set(["triage", "planner", "plan"]);
+var GATE_DEATH_MAX_DURATION_MS = 60000;
+var GATE_DEATH_CEILING = 20;
+function classifyNonProductiveStepFailure(steps) {
+  const failing = [...steps].reverse().find((step) => step.status === "failed" || step.status === "timed_out");
+  if (!failing)
+    return;
+  if (failing.exitCode === WORK_ITEM_TEMPFAIL_EXIT_CODE)
+    return "tempfail";
+  if (typeof failing.error === "string" && failing.error.includes("worktree preparation failed"))
+    return "gate-death";
+  const fast = failing.durationMs === undefined || failing.durationMs < GATE_DEATH_MAX_DURATION_MS;
+  if (GATE_STEP_IDS.has(failing.stepId) && fast)
+    return "gate-death";
+  return;
+}
 function scrubbedOrNull(value) {
   return value == null ? null : scrubSecrets(value);
 }
@@ -1962,11 +1981,21 @@ class Store {
         id TEXT PRIMARY KEY,
         applied_at TEXT NOT NULL
       );
+      CREATE TABLE IF NOT EXISTS schema_compat (
+        id INTEGER PRIMARY KEY CHECK (id = 1),
+        min_compatible_user_version INTEGER NOT NULL
+      );
     `);
     const versionRow = this.db.query("PRAGMA user_version").get();
     const userVersion = versionRow?.user_version ?? 0;
     if (userVersion > SCHEMA_USER_VERSION) {
-      throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
+      const floorRow = this.db.query("SELECT min_compatible_user_version FROM schema_compat WHERE id = 1").get();
+      if (!floorRow) {
+        throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}) and carries no compatibility floor; upgrade open-loops before opening this database`);
+      }
+      if (SCHEMA_USER_VERSION < floorRow.min_compatible_user_version) {
+        throw new Error(`loops database schema version ${userVersion} requires a binary with schema support >= ${floorRow.min_compatible_user_version} (this binary supports ${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
+      }
     }
     const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
     for (const migration of this.migrations()) {
@@ -1977,8 +2006,10 @@ class Store {
         this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
       }
     }
-    if (userVersion !== SCHEMA_USER_VERSION)
+    if (userVersion < SCHEMA_USER_VERSION)
       this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
+    this.db.query(`INSERT INTO schema_compat (id, min_compatible_user_version) VALUES (1, ?)
+         ON CONFLICT(id) DO UPDATE SET min_compatible_user_version = MAX(min_compatible_user_version, excluded.min_compatible_user_version)`).run(BREAKING_SCHEMA_FLOOR);
   }
   migrations() {
     return [
@@ -2045,6 +2076,12 @@ class Store {
           this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
           this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
         }
+      },
+      {
+        id: "0011_work_item_gate_deaths",
+        apply: () => {
+          this.addColumnIfMissing("workflow_work_items", "gate_deaths", "INTEGER NOT NULL DEFAULT 0");
+        }
       }
     ];
   }
@@ -3175,7 +3212,7 @@ class Store {
   }
   upsertWorkflowWorkItem(input) {
     const now = nowIso();
-    const id = genId();
+    const id = input.id ?? genId();
     const status = input.status ?? "queued";
     this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
           subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
@@ -3303,6 +3340,7 @@ class Store {
     const placeholders = requeueableStatuses.map(() => "?").join(",");
     const res = this.db.query(`UPDATE workflow_work_items
          SET status='queued', workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
+          ${patch.resetAttempts ? "attempts=0, gate_deaths=0," : ""}
           next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
          WHERE id=? AND status IN (${placeholders})`).run(reason, now, id, ...requeueableStatuses);
     const item = this.getWorkflowWorkItem(id);
@@ -3312,6 +3350,46 @@ class Store {
       throw new Error(`workflow work item was not requeued: ${id} status=${item.status}`);
     return item;
   }
+  deadLetterWorkflowWorkItem(id, patch = {}) {
+    const now = nowIso();
+    const reason = patch.reason?.trim() || "redispatch cap reached; dead-lettered";
+    this.db.query(`UPDATE workflow_work_items
+         SET status='dead_letter', next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
+         WHERE id=? AND status IN ('succeeded','failed','cancelled')`).run(reason, now, id);
+    const item = this.getWorkflowWorkItem(id);
+    if (!item)
+      throw new Error(`workflow work item not found after dead-letter: ${id}`);
+    return item;
+  }
+  demoteNonProductiveWorkItems(workflowRunId, finishedAt) {
+    const kind = classifyNonProductiveStepFailure(this.listWorkflowStepRuns(workflowRunId));
+    if (!kind) {
+      this.db.query("UPDATE workflow_work_items SET gate_deaths=0, updated_at=? WHERE workflow_run_id=? AND status='failed' AND gate_deaths > 0").run(finishedAt, workflowRunId);
+      return;
+    }
+    if (kind === "tempfail") {
+      this.db.query(`UPDATE workflow_work_items
+           SET status='queued', attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
+            gate_deaths=0,
+            workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
+            next_attempt_at=NULL, lease_expires_at=NULL,
+            last_reason='worker exited 75 (tempfail): requeued for retry; attempt refunded (does not count toward redispatch cap)',
+            updated_at=?
+           WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
+      return;
+    }
+    this.db.query(`UPDATE workflow_work_items
+         SET attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
+          gate_deaths=gate_deaths + 1,
+          status=CASE WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING} THEN 'dead_letter' ELSE status END,
+          last_reason=CASE
+            WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING}
+              THEN 'gate-death ceiling reached (' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING} consecutive runs died at worktree prep / triage / planner without reaching the worker): dead-lettered \u2014 the infrastructure fault needs an operator; ''loops routes requeue'' resets and retries'
+            ELSE 'gate death before real work (worktree prep / triage / planner): attempt refunded (does not count toward redispatch cap); consecutive gate deaths: ' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING}'
+          END,
+          updated_at=?
+         WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
+  }
   admitWorkflowWorkItem(id, patch) {
     const now = nowIso();
     const res = this.db.query(`UPDATE workflow_work_items
@@ -4023,6 +4101,8 @@ class Store {
       if (changed) {
         const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
         this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
+        if (itemStatus === "failed")
+          this.demoteNonProductiveWorkItems(workflowRunId, finishedAt);
         this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
       }
       this.db.exec("COMMIT");
@@ -4972,7 +5052,7 @@ class Store {
 // package.json
 var package_default = {
   name: "@hasna/loops",
-  version: "0.4.26",
+  version: "0.4.28",
   description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
   type: "module",
   main: "dist/index.js",
@@ -5082,6 +5162,7 @@ var package_default = {
     bun: ">=1.0.0"
   },
   dependencies: {
+    "@hasna/contracts": "^0.5.1",
     "@hasna/events": "^0.1.9",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@openrouter/ai-sdk-provider": "2.9.1",
@@ -5091,8 +5172,7 @@ var package_default = {
     zod: "4.4.3"
   },
   optionalDependencies: {
-    "@hasna/machines": "0.0.49",
-    "@hasna/contracts": "^0.4.2"
+    "@hasna/machines": "0.0.49"
   },
   devDependencies: {
     "@types/bun": "latest",
@@ -6239,11 +6319,30 @@ function remoteWorktreePrepareLines(worktree) {
     '  git -C "$repo" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { echo "worktree repoRoot is not a git repository: $repo" >&2; return 1; }',
     '  mkdir -p "$(dirname "$path")" || return 1',
     "  # Preparation chatter goes to stderr so run stdout stays the agent's.",
-    '  if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
-    '    git -C "$repo" worktree add "$path" "$branch" 1>&2 || return 1',
-    "  else",
-    '    git -C "$repo" worktree add -b "$branch" "$path" HEAD 1>&2 || return 1',
+    "  __openloops_worktree_add() {",
+    '    if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
+    '      git -C "$repo" worktree add "$path" "$branch"',
+    "    else",
+    '      git -C "$repo" worktree add -b "$branch" "$path" HEAD',
+    "    fi",
+    "  }",
+    "  local __ol_add_out",
+    '  if __ol_add_out="$(__openloops_worktree_add 2>&1)"; then',
+    '    if [ -n "$__ol_add_out" ]; then printf "%s\\n" "$__ol_add_out" >&2; fi',
+    "    return 0",
     "  fi",
+    '  printf "%s\\n" "$__ol_add_out" >&2',
+    "  # Self-heal git's own remedy for a stale 'missing but already registered",
+    "  # worktree': prune the dead registration (metadata-only; the directory is",
+    "  # already gone) and retry the add exactly once, then fail honestly.",
+    '  case "$__ol_add_out" in',
+    '    *"missing but already registered worktree"*)',
+    '      git -C "$repo" worktree prune 1>&2 || true',
+    "      __openloops_worktree_add 1>&2 || return 1",
+    "      return 0",
+    "      ;;",
+    "  esac",
+    "  return 1",
     "}"
   ];
 }
@@ -6383,6 +6482,9 @@ async function preflightNativeAuthProfile(spec, env) {
 function spawnDetail(result) {
   return (result.stderr || result.stdout || result.error || "").toString().trim();
 }
+function isStaleWorktreeRegistration(detail) {
+  return typeof detail === "string" && /missing but already registered worktree/i.test(detail);
+}
 function resolvedDirEquals(left, right) {
   try {
     return realpathSync(left) === realpathSync(right);
@@ -6484,7 +6586,12 @@ async function ensureLocalWorktree(worktree, env) {
     return { error: `could not create worktree parent directory: ${error instanceof Error ? error.message : String(error)}` };
   }
   const hasBranch = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
-  const add = hasBranch.status === 0 ? await git(["-C", repoRoot, "worktree", "add", path, branch]) : await git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
+  const runAdd = () => hasBranch.status === 0 ? git(["-C", repoRoot, "worktree", "add", path, branch]) : git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
+  let add = await runAdd();
+  if ((add.error || (add.status ?? 1) !== 0) && isStaleWorktreeRegistration(spawnDetail(add))) {
+    await git(["-C", repoRoot, "worktree", "prune"]);
+    add = await runAdd();
+  }
   if (add.error || (add.status ?? 1) !== 0) {
     const detail = spawnDetail(add);
     return { error: `git worktree add failed for ${path}${detail ? `: ${detail}` : ""}` };

package/dist/runner/index.js

@@ -3,7 +3,7 @@
 // package.json
 var package_default = {
   name: "@hasna/loops",
-  version: "0.4.26",
+  version: "0.4.28",
   description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
   type: "module",
   main: "dist/index.js",
@@ -113,6 +113,7 @@ var package_default = {
     bun: ">=1.0.0"
   },
   dependencies: {
+    "@hasna/contracts": "^0.5.1",
     "@hasna/events": "^0.1.9",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@openrouter/ai-sdk-provider": "2.9.1",
@@ -122,8 +123,7 @@ var package_default = {
     zod: "4.4.3"
   },
   optionalDependencies: {
-    "@hasna/machines": "0.0.49",
-    "@hasna/contracts": "^0.4.2"
+    "@hasna/machines": "0.0.49"
   },
   devDependencies: {
     "@types/bun": "latest",
@@ -1419,11 +1419,30 @@ function remoteWorktreePrepareLines(worktree) {
     '  git -C "$repo" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { echo "worktree repoRoot is not a git repository: $repo" >&2; return 1; }',
     '  mkdir -p "$(dirname "$path")" || return 1',
     "  # Preparation chatter goes to stderr so run stdout stays the agent's.",
-    '  if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
-    '    git -C "$repo" worktree add "$path" "$branch" 1>&2 || return 1',
-    "  else",
-    '    git -C "$repo" worktree add -b "$branch" "$path" HEAD 1>&2 || return 1',
+    "  __openloops_worktree_add() {",
+    '    if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
+    '      git -C "$repo" worktree add "$path" "$branch"',
+    "    else",
+    '      git -C "$repo" worktree add -b "$branch" "$path" HEAD',
+    "    fi",
+    "  }",
+    "  local __ol_add_out",
+    '  if __ol_add_out="$(__openloops_worktree_add 2>&1)"; then',
+    '    if [ -n "$__ol_add_out" ]; then printf "%s\\n" "$__ol_add_out" >&2; fi',
+    "    return 0",
     "  fi",
+    '  printf "%s\\n" "$__ol_add_out" >&2',
+    "  # Self-heal git's own remedy for a stale 'missing but already registered",
+    "  # worktree': prune the dead registration (metadata-only; the directory is",
+    "  # already gone) and retry the add exactly once, then fail honestly.",
+    '  case "$__ol_add_out" in',
+    '    *"missing but already registered worktree"*)',
+    '      git -C "$repo" worktree prune 1>&2 || true',
+    "      __openloops_worktree_add 1>&2 || return 1",
+    "      return 0",
+    "      ;;",
+    "  esac",
+    "  return 1",
     "}"
   ];
 }
@@ -1563,6 +1582,9 @@ async function preflightNativeAuthProfile(spec, env) {
 function spawnDetail(result) {
   return (result.stderr || result.stdout || result.error || "").toString().trim();
 }
+function isStaleWorktreeRegistration(detail) {
+  return typeof detail === "string" && /missing but already registered worktree/i.test(detail);
+}
 function resolvedDirEquals(left, right) {
   try {
     return realpathSync(left) === realpathSync(right);
@@ -1664,7 +1686,12 @@ async function ensureLocalWorktree(worktree, env) {
     return { error: `could not create worktree parent directory: ${error instanceof Error ? error.message : String(error)}` };
   }
   const hasBranch = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
-  const add = hasBranch.status === 0 ? await git(["-C", repoRoot, "worktree", "add", path, branch]) : await git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
+  const runAdd = () => hasBranch.status === 0 ? git(["-C", repoRoot, "worktree", "add", path, branch]) : git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
+  let add = await runAdd();
+  if ((add.error || (add.status ?? 1) !== 0) && isStaleWorktreeRegistration(spawnDetail(add))) {
+    await git(["-C", repoRoot, "worktree", "prune"]);
+    add = await runAdd();
+  }
   if (add.error || (add.status ?? 1) !== 0) {
     const detail = spawnDetail(add);
     return { error: `git worktree add failed for ${path}${detail ? `: ${detail}` : ""}` };