@hasna/loops 0.4.26 → 0.4.28

package/dist/daemon/index.js

@@ -1571,6 +1571,7 @@ var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
 var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
 var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
 var SCHEMA_USER_VERSION = 8;
+var BREAKING_SCHEMA_FLOOR = 7;
 var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
 var PRUNE_BATCH_SIZE = 400;
 var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
@@ -1713,6 +1714,7 @@ function rowToWorkflowWorkItem(row) {
     priority: row.priority,
     status: row.status,
     attempts: row.attempts,
+    gateDeaths: row.gate_deaths ?? 0,
     nextAttemptAt: row.next_attempt_at ?? undefined,
     leaseExpiresAt: row.lease_expires_at ?? undefined,
     workflowId: row.workflow_id ?? undefined,
@@ -1862,6 +1864,23 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
   }
   return;
 }
+var WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
+var GATE_STEP_IDS = new Set(["triage", "planner", "plan"]);
+var GATE_DEATH_MAX_DURATION_MS = 60000;
+var GATE_DEATH_CEILING = 20;
+function classifyNonProductiveStepFailure(steps) {
+  const failing = [...steps].reverse().find((step) => step.status === "failed" || step.status === "timed_out");
+  if (!failing)
+    return;
+  if (failing.exitCode === WORK_ITEM_TEMPFAIL_EXIT_CODE)
+    return "tempfail";
+  if (typeof failing.error === "string" && failing.error.includes("worktree preparation failed"))
+    return "gate-death";
+  const fast = failing.durationMs === undefined || failing.durationMs < GATE_DEATH_MAX_DURATION_MS;
+  if (GATE_STEP_IDS.has(failing.stepId) && fast)
+    return "gate-death";
+  return;
+}
 function scrubbedOrNull(value) {
   return value == null ? null : scrubSecrets(value);
 }
@@ -1962,11 +1981,21 @@ class Store {
         id TEXT PRIMARY KEY,
         applied_at TEXT NOT NULL
       );
+      CREATE TABLE IF NOT EXISTS schema_compat (
+        id INTEGER PRIMARY KEY CHECK (id = 1),
+        min_compatible_user_version INTEGER NOT NULL
+      );
     `);
     const versionRow = this.db.query("PRAGMA user_version").get();
     const userVersion = versionRow?.user_version ?? 0;
     if (userVersion > SCHEMA_USER_VERSION) {
-      throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
+      const floorRow = this.db.query("SELECT min_compatible_user_version FROM schema_compat WHERE id = 1").get();
+      if (!floorRow) {
+        throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}) and carries no compatibility floor; upgrade open-loops before opening this database`);
+      }
+      if (SCHEMA_USER_VERSION < floorRow.min_compatible_user_version) {
+        throw new Error(`loops database schema version ${userVersion} requires a binary with schema support >= ${floorRow.min_compatible_user_version} (this binary supports ${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
+      }
     }
     const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
     for (const migration of this.migrations()) {
@@ -1977,8 +2006,10 @@ class Store {
         this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
       }
     }
-    if (userVersion !== SCHEMA_USER_VERSION)
+    if (userVersion < SCHEMA_USER_VERSION)
       this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
+    this.db.query(`INSERT INTO schema_compat (id, min_compatible_user_version) VALUES (1, ?)
+         ON CONFLICT(id) DO UPDATE SET min_compatible_user_version = MAX(min_compatible_user_version, excluded.min_compatible_user_version)`).run(BREAKING_SCHEMA_FLOOR);
   }
   migrations() {
     return [
@@ -2045,6 +2076,12 @@ class Store {
           this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
           this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
         }
+      },
+      {
+        id: "0011_work_item_gate_deaths",
+        apply: () => {
+          this.addColumnIfMissing("workflow_work_items", "gate_deaths", "INTEGER NOT NULL DEFAULT 0");
+        }
       }
     ];
   }
@@ -3175,7 +3212,7 @@ class Store {
   }
   upsertWorkflowWorkItem(input) {
     const now = nowIso();
-    const id = genId();
+    const id = input.id ?? genId();
     const status = input.status ?? "queued";
     this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
           subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
@@ -3303,6 +3340,7 @@ class Store {
     const placeholders = requeueableStatuses.map(() => "?").join(",");
     const res = this.db.query(`UPDATE workflow_work_items
          SET status='queued', workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
+          ${patch.resetAttempts ? "attempts=0, gate_deaths=0," : ""}
           next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
          WHERE id=? AND status IN (${placeholders})`).run(reason, now, id, ...requeueableStatuses);
     const item = this.getWorkflowWorkItem(id);
@@ -3312,6 +3350,46 @@ class Store {
       throw new Error(`workflow work item was not requeued: ${id} status=${item.status}`);
     return item;
   }
+  deadLetterWorkflowWorkItem(id, patch = {}) {
+    const now = nowIso();
+    const reason = patch.reason?.trim() || "redispatch cap reached; dead-lettered";
+    this.db.query(`UPDATE workflow_work_items
+         SET status='dead_letter', next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
+         WHERE id=? AND status IN ('succeeded','failed','cancelled')`).run(reason, now, id);
+    const item = this.getWorkflowWorkItem(id);
+    if (!item)
+      throw new Error(`workflow work item not found after dead-letter: ${id}`);
+    return item;
+  }
+  demoteNonProductiveWorkItems(workflowRunId, finishedAt) {
+    const kind = classifyNonProductiveStepFailure(this.listWorkflowStepRuns(workflowRunId));
+    if (!kind) {
+      this.db.query("UPDATE workflow_work_items SET gate_deaths=0, updated_at=? WHERE workflow_run_id=? AND status='failed' AND gate_deaths > 0").run(finishedAt, workflowRunId);
+      return;
+    }
+    if (kind === "tempfail") {
+      this.db.query(`UPDATE workflow_work_items
+           SET status='queued', attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
+            gate_deaths=0,
+            workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
+            next_attempt_at=NULL, lease_expires_at=NULL,
+            last_reason='worker exited 75 (tempfail): requeued for retry; attempt refunded (does not count toward redispatch cap)',
+            updated_at=?
+           WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
+      return;
+    }
+    this.db.query(`UPDATE workflow_work_items
+         SET attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
+          gate_deaths=gate_deaths + 1,
+          status=CASE WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING} THEN 'dead_letter' ELSE status END,
+          last_reason=CASE
+            WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING}
+              THEN 'gate-death ceiling reached (' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING} consecutive runs died at worktree prep / triage / planner without reaching the worker): dead-lettered \u2014 the infrastructure fault needs an operator; ''loops routes requeue'' resets and retries'
+            ELSE 'gate death before real work (worktree prep / triage / planner): attempt refunded (does not count toward redispatch cap); consecutive gate deaths: ' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING}'
+          END,
+          updated_at=?
+         WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
+  }
   admitWorkflowWorkItem(id, patch) {
     const now = nowIso();
     const res = this.db.query(`UPDATE workflow_work_items
@@ -4023,6 +4101,8 @@ class Store {
       if (changed) {
         const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
         this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
+        if (itemStatus === "failed")
+          this.demoteNonProductiveWorkItems(workflowRunId, finishedAt);
         this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
       }
       this.db.exec("COMMIT");
@@ -6425,11 +6505,30 @@ function remoteWorktreePrepareLines(worktree) {
     '  git -C "$repo" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { echo "worktree repoRoot is not a git repository: $repo" >&2; return 1; }',
     '  mkdir -p "$(dirname "$path")" || return 1',
     "  # Preparation chatter goes to stderr so run stdout stays the agent's.",
-    '  if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
-    '    git -C "$repo" worktree add "$path" "$branch" 1>&2 || return 1',
-    "  else",
-    '    git -C "$repo" worktree add -b "$branch" "$path" HEAD 1>&2 || return 1',
+    "  __openloops_worktree_add() {",
+    '    if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
+    '      git -C "$repo" worktree add "$path" "$branch"',
+    "    else",
+    '      git -C "$repo" worktree add -b "$branch" "$path" HEAD',
+    "    fi",
+    "  }",
+    "  local __ol_add_out",
+    '  if __ol_add_out="$(__openloops_worktree_add 2>&1)"; then',
+    '    if [ -n "$__ol_add_out" ]; then printf "%s\\n" "$__ol_add_out" >&2; fi',
+    "    return 0",
     "  fi",
+    '  printf "%s\\n" "$__ol_add_out" >&2',
+    "  # Self-heal git's own remedy for a stale 'missing but already registered",
+    "  # worktree': prune the dead registration (metadata-only; the directory is",
+    "  # already gone) and retry the add exactly once, then fail honestly.",
+    '  case "$__ol_add_out" in',
+    '    *"missing but already registered worktree"*)',
+    '      git -C "$repo" worktree prune 1>&2 || true',
+    "      __openloops_worktree_add 1>&2 || return 1",
+    "      return 0",
+    "      ;;",
+    "  esac",
+    "  return 1",
     "}"
   ];
 }
@@ -6569,6 +6668,9 @@ async function preflightNativeAuthProfile(spec, env) {
 function spawnDetail(result) {
   return (result.stderr || result.stdout || result.error || "").toString().trim();
 }
+function isStaleWorktreeRegistration(detail) {
+  return typeof detail === "string" && /missing but already registered worktree/i.test(detail);
+}
 function resolvedDirEquals(left, right) {
   try {
     return realpathSync(left) === realpathSync(right);
@@ -6670,7 +6772,12 @@ async function ensureLocalWorktree(worktree, env) {
     return { error: `could not create worktree parent directory: ${error instanceof Error ? error.message : String(error)}` };
   }
   const hasBranch = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
-  const add = hasBranch.status === 0 ? await git(["-C", repoRoot, "worktree", "add", path, branch]) : await git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
+  const runAdd = () => hasBranch.status === 0 ? git(["-C", repoRoot, "worktree", "add", path, branch]) : git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
+  let add = await runAdd();
+  if ((add.error || (add.status ?? 1) !== 0) && isStaleWorktreeRegistration(spawnDetail(add))) {
+    await git(["-C", repoRoot, "worktree", "prune"]);
+    add = await runAdd();
+  }
   if (add.error || (add.status ?? 1) !== 0) {
     const detail = spawnDetail(add);
     return { error: `git worktree add failed for ${path}${detail ? `: ${detail}` : ""}` };
@@ -8934,7 +9041,7 @@ function enableStartup(result) {
 // package.json
 var package_default = {
   name: "@hasna/loops",
-  version: "0.4.26",
+  version: "0.4.28",
   description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
   type: "module",
   main: "dist/index.js",
@@ -9044,6 +9151,7 @@ var package_default = {
     bun: ">=1.0.0"
   },
   dependencies: {
+    "@hasna/contracts": "^0.5.1",
     "@hasna/events": "^0.1.9",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@openrouter/ai-sdk-provider": "2.9.1",
@@ -9053,8 +9161,7 @@ var package_default = {
     zod: "4.4.3"
   },
   optionalDependencies: {
-    "@hasna/machines": "0.0.49",
-    "@hasna/contracts": "^0.4.2"
+    "@hasna/machines": "0.0.49"
   },
   devDependencies: {
     "@types/bun": "latest",