@hasna/loops 0.4.26 → 0.4.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -56,6 +56,15 @@ interface MachineCommandPlan {
56
56
  }
57
57
  /** Exported for tests: resolves the default idle watchdog for agent targets. */
58
58
  export declare function defaultAgentIdleTimeoutMs(target: AgentTarget, opts: ExecuteOptions): number | undefined;
59
+ /**
60
+ * Detects git's "missing but already registered worktree" error: the target
61
+ * path is recorded in `.git/worktrees/` but its directory was deleted out from
62
+ * under it, so `git worktree add` refuses. git's own prescribed remedy is
63
+ * `git worktree prune` (or `add -f`). Left unhandled this terminal-fails worktree
64
+ * prep on every attempt and the drain dedupes the failed item into a silent
65
+ * wedge — historically the single biggest burner of the redispatch cap.
66
+ */
67
+ export declare function isStaleWorktreeRegistration(detail: string | undefined): boolean;
59
68
  export declare function preflightTarget(target: ExecutableTarget, metadata?: ExecutionMetadata, opts?: ExecuteOptions): PreflightResult;
60
69
  export declare function executeTarget(target: ExecutableTarget, metadata?: ExecutionMetadata, opts?: ExecuteOptions): Promise<ExecutorResult>;
61
70
  export declare function executeLoop(loop: Loop, run: LoopRun, opts?: ExecuteOptions): Promise<ExecutorResult>;
@@ -1,6 +1,7 @@
1
1
  import type { Loop, LoopRun, WorkflowSpec } from "../types.js";
2
2
  import type { Store, StoreMigrationChecks } from "./store.js";
3
3
  export declare const LOOPS_MIGRATION_SCHEMA = "open-loops.migration/v1";
4
+ export declare const LOOPS_SELF_HOSTED_PUSH_MANIFEST_SCHEMA = "open-loops.self-hosted-push-manifest/v1";
4
5
  export type LoopsMigrationResource = "workflow" | "loop" | "run" | "remote";
5
6
  export type LoopsMigrationAction = "insert" | "update" | "skip" | "conflict" | "blocked";
6
7
  export interface LoopsMigrationBundle {
@@ -67,6 +68,7 @@ export interface LoopsMigrationPlan {
67
68
  summary: LoopsMigrationPlanSummary;
68
69
  rows: LoopsMigrationPlanRow[];
69
70
  warnings: string[];
71
+ manifest?: SelfHostedPushManifest;
70
72
  }
71
73
  export interface ApplyLoopsMigrationResult {
72
74
  plan: LoopsMigrationPlan;
@@ -83,6 +85,7 @@ export interface SelfHostedPlanOptions {
83
85
  fetchImpl?: typeof fetch;
84
86
  env?: NodeJS.ProcessEnv;
85
87
  includeRuns?: boolean;
88
+ replace?: boolean;
86
89
  }
87
90
  export interface RunnerRegistrationOptions {
88
91
  apiUrl?: string;
@@ -134,6 +137,69 @@ export interface SelfHostedPushResult {
134
137
  orphanRuns: number;
135
138
  };
136
139
  requests: number;
140
+ manifest: SelfHostedPushManifest;
141
+ }
142
+ export interface SelfHostedPushManifest {
143
+ schema: typeof LOOPS_SELF_HOSTED_PUSH_MANIFEST_SCHEMA;
144
+ generatedAt: string;
145
+ apiUrl?: string;
146
+ dryRun: boolean;
147
+ replace: boolean;
148
+ includeRuns: boolean;
149
+ safety: {
150
+ forcedLoopStatus: "paused";
151
+ clearedLoopRunPointers: true;
152
+ forcedWorkflowStatus: "archived";
153
+ resumesLoops: false;
154
+ };
155
+ counts: {
156
+ source: {
157
+ workflows: number;
158
+ loops: number;
159
+ runs: number;
160
+ };
161
+ remoteBefore: {
162
+ workflows?: number;
163
+ loops?: number;
164
+ runs?: number;
165
+ };
166
+ remoteAfter?: {
167
+ workflows?: number;
168
+ loops?: number;
169
+ runs?: number;
170
+ };
171
+ plan: LoopsMigrationPlanSummary;
172
+ applied?: {
173
+ workflows: number;
174
+ loops: number;
175
+ runs: number;
176
+ };
177
+ skipped?: {
178
+ runningRuns: number;
179
+ orphanRuns: number;
180
+ };
181
+ requests?: number;
182
+ };
183
+ missingIds: {
184
+ workflows: string[];
185
+ loops: string[];
186
+ runs: string[];
187
+ };
188
+ existingIds: {
189
+ workflows: string[];
190
+ loops: string[];
191
+ runs: string[];
192
+ };
193
+ unsafe: {
194
+ blocked: LoopsMigrationPlanRow[];
195
+ conflicts: LoopsMigrationPlanRow[];
196
+ unsupported: string[];
197
+ warnings: string[];
198
+ };
199
+ rollback: {
200
+ notes: string[];
201
+ commands: string[];
202
+ };
137
203
  }
138
204
  /**
139
205
  * Apply a local->self-hosted backfill through the control plane's id-preserving
package/dist/lib/mode.js CHANGED
@@ -2,7 +2,7 @@
2
2
  // package.json
3
3
  var package_default = {
4
4
  name: "@hasna/loops",
5
- version: "0.4.26",
5
+ version: "0.4.28",
6
6
  description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
7
7
  type: "module",
8
8
  main: "dist/index.js",
@@ -112,6 +112,7 @@ var package_default = {
112
112
  bun: ">=1.0.0"
113
113
  },
114
114
  dependencies: {
115
+ "@hasna/contracts": "^0.5.1",
115
116
  "@hasna/events": "^0.1.9",
116
117
  "@modelcontextprotocol/sdk": "^1.29.0",
117
118
  "@openrouter/ai-sdk-provider": "2.9.1",
@@ -121,8 +122,7 @@ var package_default = {
121
122
  zod: "4.4.3"
122
123
  },
123
124
  optionalDependencies: {
124
- "@hasna/machines": "0.0.49",
125
- "@hasna/contracts": "^0.4.2"
125
+ "@hasna/machines": "0.0.49"
126
126
  },
127
127
  devDependencies: {
128
128
  "@types/bun": "latest",
@@ -15,6 +15,14 @@ export declare function firstTruthyField(records: Record<string, unknown>[], key
15
15
  export declare function automationRecords(data: Record<string, unknown>, metadata: Record<string, unknown>): Record<string, unknown>[];
16
16
  export declare function tagsFromValue(value: unknown): string[];
17
17
  export declare function taskEventTags(records: Record<string, unknown>[]): string[];
18
+ /**
19
+ * First route-disallowed tag on a task, if any. Exposed so drains can exclude
20
+ * tasks that can never route (no-auto/blocked/manual/…) from the bounded
21
+ * candidate window *before* slicing, instead of letting them occupy scan slots
22
+ * every tick only to be rejected by eligibility — the "skip re-burns the window
23
+ * forever" half of the merge-lane decay.
24
+ */
25
+ export declare function taskRouteDisallowedTag(tags: string[]): string | undefined;
18
26
  export declare function taskRouteEligibility(data: Record<string, unknown>, metadata: Record<string, unknown>): {
19
27
  eligible: boolean;
20
28
  reason?: string;
@@ -1569,6 +1569,7 @@ var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
1569
1569
  var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
1570
1570
  var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
1571
1571
  var SCHEMA_USER_VERSION = 8;
1572
+ var BREAKING_SCHEMA_FLOOR = 7;
1572
1573
  var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
1573
1574
  var PRUNE_BATCH_SIZE = 400;
1574
1575
  var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
@@ -1711,6 +1712,7 @@ function rowToWorkflowWorkItem(row) {
1711
1712
  priority: row.priority,
1712
1713
  status: row.status,
1713
1714
  attempts: row.attempts,
1715
+ gateDeaths: row.gate_deaths ?? 0,
1714
1716
  nextAttemptAt: row.next_attempt_at ?? undefined,
1715
1717
  leaseExpiresAt: row.lease_expires_at ?? undefined,
1716
1718
  workflowId: row.workflow_id ?? undefined,
@@ -1860,6 +1862,23 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
1860
1862
  }
1861
1863
  return;
1862
1864
  }
1865
+ var WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
1866
+ var GATE_STEP_IDS = new Set(["triage", "planner", "plan"]);
1867
+ var GATE_DEATH_MAX_DURATION_MS = 60000;
1868
+ var GATE_DEATH_CEILING = 20;
1869
+ function classifyNonProductiveStepFailure(steps) {
1870
+ const failing = [...steps].reverse().find((step) => step.status === "failed" || step.status === "timed_out");
1871
+ if (!failing)
1872
+ return;
1873
+ if (failing.exitCode === WORK_ITEM_TEMPFAIL_EXIT_CODE)
1874
+ return "tempfail";
1875
+ if (typeof failing.error === "string" && failing.error.includes("worktree preparation failed"))
1876
+ return "gate-death";
1877
+ const fast = failing.durationMs === undefined || failing.durationMs < GATE_DEATH_MAX_DURATION_MS;
1878
+ if (GATE_STEP_IDS.has(failing.stepId) && fast)
1879
+ return "gate-death";
1880
+ return;
1881
+ }
1863
1882
  function scrubbedOrNull(value) {
1864
1883
  return value == null ? null : scrubSecrets(value);
1865
1884
  }
@@ -1960,11 +1979,21 @@ class Store {
1960
1979
  id TEXT PRIMARY KEY,
1961
1980
  applied_at TEXT NOT NULL
1962
1981
  );
1982
+ CREATE TABLE IF NOT EXISTS schema_compat (
1983
+ id INTEGER PRIMARY KEY CHECK (id = 1),
1984
+ min_compatible_user_version INTEGER NOT NULL
1985
+ );
1963
1986
  `);
1964
1987
  const versionRow = this.db.query("PRAGMA user_version").get();
1965
1988
  const userVersion = versionRow?.user_version ?? 0;
1966
1989
  if (userVersion > SCHEMA_USER_VERSION) {
1967
- throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
1990
+ const floorRow = this.db.query("SELECT min_compatible_user_version FROM schema_compat WHERE id = 1").get();
1991
+ if (!floorRow) {
1992
+ throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}) and carries no compatibility floor; upgrade open-loops before opening this database`);
1993
+ }
1994
+ if (SCHEMA_USER_VERSION < floorRow.min_compatible_user_version) {
1995
+ throw new Error(`loops database schema version ${userVersion} requires a binary with schema support >= ${floorRow.min_compatible_user_version} (this binary supports ${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
1996
+ }
1968
1997
  }
1969
1998
  const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
1970
1999
  for (const migration of this.migrations()) {
@@ -1975,8 +2004,10 @@ class Store {
1975
2004
  this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
1976
2005
  }
1977
2006
  }
1978
- if (userVersion !== SCHEMA_USER_VERSION)
2007
+ if (userVersion < SCHEMA_USER_VERSION)
1979
2008
  this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
2009
+ this.db.query(`INSERT INTO schema_compat (id, min_compatible_user_version) VALUES (1, ?)
2010
+ ON CONFLICT(id) DO UPDATE SET min_compatible_user_version = MAX(min_compatible_user_version, excluded.min_compatible_user_version)`).run(BREAKING_SCHEMA_FLOOR);
1980
2011
  }
1981
2012
  migrations() {
1982
2013
  return [
@@ -2043,6 +2074,12 @@ class Store {
2043
2074
  this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
2044
2075
  this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
2045
2076
  }
2077
+ },
2078
+ {
2079
+ id: "0011_work_item_gate_deaths",
2080
+ apply: () => {
2081
+ this.addColumnIfMissing("workflow_work_items", "gate_deaths", "INTEGER NOT NULL DEFAULT 0");
2082
+ }
2046
2083
  }
2047
2084
  ];
2048
2085
  }
@@ -3173,7 +3210,7 @@ class Store {
3173
3210
  }
3174
3211
  upsertWorkflowWorkItem(input) {
3175
3212
  const now = nowIso();
3176
- const id = genId();
3213
+ const id = input.id ?? genId();
3177
3214
  const status = input.status ?? "queued";
3178
3215
  this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
3179
3216
  subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
@@ -3301,6 +3338,7 @@ class Store {
3301
3338
  const placeholders = requeueableStatuses.map(() => "?").join(",");
3302
3339
  const res = this.db.query(`UPDATE workflow_work_items
3303
3340
  SET status='queued', workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
3341
+ ${patch.resetAttempts ? "attempts=0, gate_deaths=0," : ""}
3304
3342
  next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
3305
3343
  WHERE id=? AND status IN (${placeholders})`).run(reason, now, id, ...requeueableStatuses);
3306
3344
  const item = this.getWorkflowWorkItem(id);
@@ -3310,6 +3348,46 @@ class Store {
3310
3348
  throw new Error(`workflow work item was not requeued: ${id} status=${item.status}`);
3311
3349
  return item;
3312
3350
  }
3351
+ deadLetterWorkflowWorkItem(id, patch = {}) {
3352
+ const now = nowIso();
3353
+ const reason = patch.reason?.trim() || "redispatch cap reached; dead-lettered";
3354
+ this.db.query(`UPDATE workflow_work_items
3355
+ SET status='dead_letter', next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
3356
+ WHERE id=? AND status IN ('succeeded','failed','cancelled')`).run(reason, now, id);
3357
+ const item = this.getWorkflowWorkItem(id);
3358
+ if (!item)
3359
+ throw new Error(`workflow work item not found after dead-letter: ${id}`);
3360
+ return item;
3361
+ }
3362
+ demoteNonProductiveWorkItems(workflowRunId, finishedAt) {
3363
+ const kind = classifyNonProductiveStepFailure(this.listWorkflowStepRuns(workflowRunId));
3364
+ if (!kind) {
3365
+ this.db.query("UPDATE workflow_work_items SET gate_deaths=0, updated_at=? WHERE workflow_run_id=? AND status='failed' AND gate_deaths > 0").run(finishedAt, workflowRunId);
3366
+ return;
3367
+ }
3368
+ if (kind === "tempfail") {
3369
+ this.db.query(`UPDATE workflow_work_items
3370
+ SET status='queued', attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
3371
+ gate_deaths=0,
3372
+ workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
3373
+ next_attempt_at=NULL, lease_expires_at=NULL,
3374
+ last_reason='worker exited 75 (tempfail): requeued for retry; attempt refunded (does not count toward redispatch cap)',
3375
+ updated_at=?
3376
+ WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
3377
+ return;
3378
+ }
3379
+ this.db.query(`UPDATE workflow_work_items
3380
+ SET attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
3381
+ gate_deaths=gate_deaths + 1,
3382
+ status=CASE WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING} THEN 'dead_letter' ELSE status END,
3383
+ last_reason=CASE
3384
+ WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING}
3385
+ THEN 'gate-death ceiling reached (' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING} consecutive runs died at worktree prep / triage / planner without reaching the worker): dead-lettered \u2014 the infrastructure fault needs an operator; ''loops routes requeue'' resets and retries'
3386
+ ELSE 'gate death before real work (worktree prep / triage / planner): attempt refunded (does not count toward redispatch cap); consecutive gate deaths: ' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING}'
3387
+ END,
3388
+ updated_at=?
3389
+ WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
3390
+ }
3313
3391
  admitWorkflowWorkItem(id, patch) {
3314
3392
  const now = nowIso();
3315
3393
  const res = this.db.query(`UPDATE workflow_work_items
@@ -4021,6 +4099,8 @@ class Store {
4021
4099
  if (changed) {
4022
4100
  const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
4023
4101
  this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
4102
+ if (itemStatus === "failed")
4103
+ this.demoteNonProductiveWorkItems(workflowRunId, finishedAt);
4024
4104
  this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
4025
4105
  }
4026
4106
  this.db.exec("COMMIT");
@@ -5321,6 +5401,9 @@ CREATE INDEX IF NOT EXISTS idx_run_receipts_knowledge_ids ON run_receipts USING
5321
5401
  migration("0006_work_item_machine_id", `
5322
5402
  ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS machine_id TEXT;
5323
5403
  CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status);
5404
+ `),
5405
+ migration("0007_work_item_gate_deaths", `
5406
+ ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS gate_deaths INTEGER NOT NULL DEFAULT 0;
5324
5407
  `)
5325
5408
  ]);
5326
5409
 
@@ -6671,11 +6754,118 @@ class PostgresLoopStorage {
6671
6754
  throw new Error(`workflow not found after archive: ${existing.id}`);
6672
6755
  return archived;
6673
6756
  }
6674
- createWorkflowInvocation() {
6675
- throw new NotImplementedError("createWorkflowInvocation");
6757
+ async createWorkflowInvocation(...args) {
6758
+ const [input] = args;
6759
+ const now = nowIso();
6760
+ const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
6761
+ if (sourceDedupeKey) {
6762
+ const existing = await this.client.get("SELECT * FROM workflow_invocations WHERE source_kind = $1 AND source_dedupe_key = $2 LIMIT 1", [input.sourceRef.kind, sourceDedupeKey]);
6763
+ if (existing)
6764
+ return rowToWorkflowInvocation(existing);
6765
+ }
6766
+ const id = input.id ?? genId();
6767
+ await this.client.execute(`INSERT INTO workflow_invocations (id, workflow_id, template_id, source_kind, source_id, source_dedupe_key,
6768
+ source_json, subject_kind, subject_id, subject_path, subject_url, subject_json, intent, scope_json,
6769
+ output_policy_json, created_at, updated_at)
6770
+ VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8,$9,$10,$11,$12::jsonb,$13,$14::jsonb,$15::jsonb,$16,$17)`, [
6771
+ id,
6772
+ input.workflowId ?? null,
6773
+ input.templateId ?? null,
6774
+ input.sourceRef.kind,
6775
+ input.sourceRef.id ?? null,
6776
+ sourceDedupeKey ?? null,
6777
+ JSON.stringify(input.sourceRef),
6778
+ input.subjectRef.kind,
6779
+ input.subjectRef.id ?? null,
6780
+ input.subjectRef.path ?? null,
6781
+ input.subjectRef.url ?? null,
6782
+ JSON.stringify(input.subjectRef),
6783
+ input.intent,
6784
+ input.scope ? JSON.stringify(input.scope) : null,
6785
+ input.outputPolicy ? JSON.stringify(input.outputPolicy) : null,
6786
+ now,
6787
+ now
6788
+ ]);
6789
+ const row = await this.client.get("SELECT * FROM workflow_invocations WHERE id = $1", [id]);
6790
+ if (!row)
6791
+ throw new Error(`workflow invocation not found after create: ${id}`);
6792
+ return rowToWorkflowInvocation(row);
6676
6793
  }
6677
- upsertWorkflowWorkItem() {
6678
- throw new NotImplementedError("upsertWorkflowWorkItem");
6794
+ async upsertWorkflowWorkItem(...args) {
6795
+ const [input] = args;
6796
+ const now = nowIso();
6797
+ const id = input.id ?? genId();
6798
+ const status = input.status ?? "queued";
6799
+ await this.client.execute(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
6800
+ subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at,
6801
+ lease_expires_at, workflow_id, loop_id, workflow_run_id, last_reason, created_at, updated_at)
6802
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,0,$14,NULL,NULL,NULL,NULL,$15,$16,$17)
6803
+ ON CONFLICT(route_key, idempotency_key) DO UPDATE SET
6804
+ invocation_id=excluded.invocation_id,
6805
+ source_type=excluded.source_type,
6806
+ source_ref=excluded.source_ref,
6807
+ subject_ref=excluded.subject_ref,
6808
+ project_key=excluded.project_key,
6809
+ project_group=excluded.project_group,
6810
+ machine_id=CASE
6811
+ WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.machine_id
6812
+ ELSE excluded.machine_id
6813
+ END,
6814
+ route_scope=excluded.route_scope,
6815
+ priority=excluded.priority,
6816
+ status=CASE
6817
+ WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled')
6818
+ THEN workflow_work_items.status
6819
+ ELSE excluded.status
6820
+ END,
6821
+ workflow_id=CASE
6822
+ WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.workflow_id
6823
+ ELSE NULL
6824
+ END,
6825
+ loop_id=CASE
6826
+ WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.loop_id
6827
+ ELSE NULL
6828
+ END,
6829
+ workflow_run_id=CASE
6830
+ WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.workflow_run_id
6831
+ ELSE NULL
6832
+ END,
6833
+ lease_expires_at=CASE
6834
+ WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.lease_expires_at
6835
+ ELSE NULL
6836
+ END,
6837
+ next_attempt_at=excluded.next_attempt_at,
6838
+ last_reason=CASE
6839
+ WHEN workflow_work_items.attempts > 0
6840
+ AND workflow_work_items.status IN ('queued', 'deferred')
6841
+ AND workflow_work_items.last_reason IS NOT NULL
6842
+ AND excluded.last_reason IS NOT NULL
6843
+ THEN workflow_work_items.last_reason || '; ' || excluded.last_reason
6844
+ ELSE COALESCE(excluded.last_reason, workflow_work_items.last_reason)
6845
+ END,
6846
+ updated_at=excluded.updated_at`, [
6847
+ id,
6848
+ input.routeKey,
6849
+ input.idempotencyKey,
6850
+ input.invocationId,
6851
+ input.sourceType,
6852
+ input.sourceRef,
6853
+ input.subjectRef,
6854
+ input.projectKey ?? null,
6855
+ input.projectGroup ?? null,
6856
+ input.machineId ?? null,
6857
+ input.routeScope ?? null,
6858
+ input.priority ?? 0,
6859
+ status,
6860
+ input.nextAttemptAt ?? null,
6861
+ input.lastReason ?? null,
6862
+ now,
6863
+ now
6864
+ ]);
6865
+ const row = await this.client.get("SELECT * FROM workflow_work_items WHERE route_key = $1 AND idempotency_key = $2 LIMIT 1", [input.routeKey, input.idempotencyKey]);
6866
+ if (!row)
6867
+ throw new Error(`workflow work item not found after upsert: ${input.routeKey}/${input.idempotencyKey}`);
6868
+ return rowToWorkflowWorkItem(row);
6679
6869
  }
6680
6870
  admitWorkflowWorkItem() {
6681
6871
  throw new NotImplementedError("admitWorkflowWorkItem");
@@ -101,8 +101,8 @@ export declare class PostgresLoopStorage implements LoopStorageContract {
101
101
  listGoalRuns(...args: M<"listGoalRuns">["args"]): Promise<M<"listGoalRuns">["result"]>;
102
102
  createWorkflow(...args: M<"createWorkflow">["args"]): Promise<M<"createWorkflow">["result"]>;
103
103
  archiveWorkflow(...args: M<"archiveWorkflow">["args"]): Promise<M<"archiveWorkflow">["result"]>;
104
- createWorkflowInvocation(): never;
105
- upsertWorkflowWorkItem(): never;
104
+ createWorkflowInvocation(...args: M<"createWorkflowInvocation">["args"]): Promise<M<"createWorkflowInvocation">["result"]>;
105
+ upsertWorkflowWorkItem(...args: M<"upsertWorkflowWorkItem">["args"]): Promise<M<"upsertWorkflowWorkItem">["result"]>;
106
106
  admitWorkflowWorkItem(): never;
107
107
  createGoal(): never;
108
108
  createGoalPlanNodes(): never;
@@ -352,6 +352,9 @@ CREATE INDEX IF NOT EXISTS idx_run_receipts_knowledge_ids ON run_receipts USING
352
352
  migration("0006_work_item_machine_id", `
353
353
  ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS machine_id TEXT;
354
354
  CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status);
355
+ `),
356
+ migration("0007_work_item_gate_deaths", `
357
+ ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS gate_deaths INTEGER NOT NULL DEFAULT 0;
355
358
  `)
356
359
  ]);
357
360
  export {
@@ -352,6 +352,9 @@ CREATE INDEX IF NOT EXISTS idx_run_receipts_knowledge_ids ON run_receipts USING
352
352
  migration("0006_work_item_machine_id", `
353
353
  ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS machine_id TEXT;
354
354
  CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status);
355
+ `),
356
+ migration("0007_work_item_gate_deaths", `
357
+ ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS gate_deaths INTEGER NOT NULL DEFAULT 0;
355
358
  `)
356
359
  ]);
357
360
 
@@ -1569,6 +1569,7 @@ var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
1569
1569
  var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
1570
1570
  var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
1571
1571
  var SCHEMA_USER_VERSION = 8;
1572
+ var BREAKING_SCHEMA_FLOOR = 7;
1572
1573
  var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
1573
1574
  var PRUNE_BATCH_SIZE = 400;
1574
1575
  var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
@@ -1711,6 +1712,7 @@ function rowToWorkflowWorkItem(row) {
1711
1712
  priority: row.priority,
1712
1713
  status: row.status,
1713
1714
  attempts: row.attempts,
1715
+ gateDeaths: row.gate_deaths ?? 0,
1714
1716
  nextAttemptAt: row.next_attempt_at ?? undefined,
1715
1717
  leaseExpiresAt: row.lease_expires_at ?? undefined,
1716
1718
  workflowId: row.workflow_id ?? undefined,
@@ -1860,6 +1862,23 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
1860
1862
  }
1861
1863
  return;
1862
1864
  }
1865
+ var WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
1866
+ var GATE_STEP_IDS = new Set(["triage", "planner", "plan"]);
1867
+ var GATE_DEATH_MAX_DURATION_MS = 60000;
1868
+ var GATE_DEATH_CEILING = 20;
1869
+ function classifyNonProductiveStepFailure(steps) {
1870
+ const failing = [...steps].reverse().find((step) => step.status === "failed" || step.status === "timed_out");
1871
+ if (!failing)
1872
+ return;
1873
+ if (failing.exitCode === WORK_ITEM_TEMPFAIL_EXIT_CODE)
1874
+ return "tempfail";
1875
+ if (typeof failing.error === "string" && failing.error.includes("worktree preparation failed"))
1876
+ return "gate-death";
1877
+ const fast = failing.durationMs === undefined || failing.durationMs < GATE_DEATH_MAX_DURATION_MS;
1878
+ if (GATE_STEP_IDS.has(failing.stepId) && fast)
1879
+ return "gate-death";
1880
+ return;
1881
+ }
1863
1882
  function scrubbedOrNull(value) {
1864
1883
  return value == null ? null : scrubSecrets(value);
1865
1884
  }
@@ -1960,11 +1979,21 @@ class Store {
1960
1979
  id TEXT PRIMARY KEY,
1961
1980
  applied_at TEXT NOT NULL
1962
1981
  );
1982
+ CREATE TABLE IF NOT EXISTS schema_compat (
1983
+ id INTEGER PRIMARY KEY CHECK (id = 1),
1984
+ min_compatible_user_version INTEGER NOT NULL
1985
+ );
1963
1986
  `);
1964
1987
  const versionRow = this.db.query("PRAGMA user_version").get();
1965
1988
  const userVersion = versionRow?.user_version ?? 0;
1966
1989
  if (userVersion > SCHEMA_USER_VERSION) {
1967
- throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
1990
+ const floorRow = this.db.query("SELECT min_compatible_user_version FROM schema_compat WHERE id = 1").get();
1991
+ if (!floorRow) {
1992
+ throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}) and carries no compatibility floor; upgrade open-loops before opening this database`);
1993
+ }
1994
+ if (SCHEMA_USER_VERSION < floorRow.min_compatible_user_version) {
1995
+ throw new Error(`loops database schema version ${userVersion} requires a binary with schema support >= ${floorRow.min_compatible_user_version} (this binary supports ${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
1996
+ }
1968
1997
  }
1969
1998
  const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
1970
1999
  for (const migration of this.migrations()) {
@@ -1975,8 +2004,10 @@ class Store {
1975
2004
  this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
1976
2005
  }
1977
2006
  }
1978
- if (userVersion !== SCHEMA_USER_VERSION)
2007
+ if (userVersion < SCHEMA_USER_VERSION)
1979
2008
  this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
2009
+ this.db.query(`INSERT INTO schema_compat (id, min_compatible_user_version) VALUES (1, ?)
2010
+ ON CONFLICT(id) DO UPDATE SET min_compatible_user_version = MAX(min_compatible_user_version, excluded.min_compatible_user_version)`).run(BREAKING_SCHEMA_FLOOR);
1980
2011
  }
1981
2012
  migrations() {
1982
2013
  return [
@@ -2043,6 +2074,12 @@ class Store {
2043
2074
  this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
2044
2075
  this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
2045
2076
  }
2077
+ },
2078
+ {
2079
+ id: "0011_work_item_gate_deaths",
2080
+ apply: () => {
2081
+ this.addColumnIfMissing("workflow_work_items", "gate_deaths", "INTEGER NOT NULL DEFAULT 0");
2082
+ }
2046
2083
  }
2047
2084
  ];
2048
2085
  }
@@ -3173,7 +3210,7 @@ class Store {
3173
3210
  }
3174
3211
  upsertWorkflowWorkItem(input) {
3175
3212
  const now = nowIso();
3176
- const id = genId();
3213
+ const id = input.id ?? genId();
3177
3214
  const status = input.status ?? "queued";
3178
3215
  this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
3179
3216
  subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
@@ -3301,6 +3338,7 @@ class Store {
3301
3338
  const placeholders = requeueableStatuses.map(() => "?").join(",");
3302
3339
  const res = this.db.query(`UPDATE workflow_work_items
3303
3340
  SET status='queued', workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
3341
+ ${patch.resetAttempts ? "attempts=0, gate_deaths=0," : ""}
3304
3342
  next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
3305
3343
  WHERE id=? AND status IN (${placeholders})`).run(reason, now, id, ...requeueableStatuses);
3306
3344
  const item = this.getWorkflowWorkItem(id);
@@ -3310,6 +3348,46 @@ class Store {
3310
3348
  throw new Error(`workflow work item was not requeued: ${id} status=${item.status}`);
3311
3349
  return item;
3312
3350
  }
3351
+ deadLetterWorkflowWorkItem(id, patch = {}) {
3352
+ const now = nowIso();
3353
+ const reason = patch.reason?.trim() || "redispatch cap reached; dead-lettered";
3354
+ this.db.query(`UPDATE workflow_work_items
3355
+ SET status='dead_letter', next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
3356
+ WHERE id=? AND status IN ('succeeded','failed','cancelled')`).run(reason, now, id);
3357
+ const item = this.getWorkflowWorkItem(id);
3358
+ if (!item)
3359
+ throw new Error(`workflow work item not found after dead-letter: ${id}`);
3360
+ return item;
3361
+ }
3362
+ demoteNonProductiveWorkItems(workflowRunId, finishedAt) {
3363
+ const kind = classifyNonProductiveStepFailure(this.listWorkflowStepRuns(workflowRunId));
3364
+ if (!kind) {
3365
+ this.db.query("UPDATE workflow_work_items SET gate_deaths=0, updated_at=? WHERE workflow_run_id=? AND status='failed' AND gate_deaths > 0").run(finishedAt, workflowRunId);
3366
+ return;
3367
+ }
3368
+ if (kind === "tempfail") {
3369
+ this.db.query(`UPDATE workflow_work_items
3370
+ SET status='queued', attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
3371
+ gate_deaths=0,
3372
+ workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
3373
+ next_attempt_at=NULL, lease_expires_at=NULL,
3374
+ last_reason='worker exited 75 (tempfail): requeued for retry; attempt refunded (does not count toward redispatch cap)',
3375
+ updated_at=?
3376
+ WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
3377
+ return;
3378
+ }
3379
+ this.db.query(`UPDATE workflow_work_items
3380
+ SET attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
3381
+ gate_deaths=gate_deaths + 1,
3382
+ status=CASE WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING} THEN 'dead_letter' ELSE status END,
3383
+ last_reason=CASE
3384
+ WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING}
3385
+ THEN 'gate-death ceiling reached (' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING} consecutive runs died at worktree prep / triage / planner without reaching the worker): dead-lettered \u2014 the infrastructure fault needs an operator; ''loops routes requeue'' resets and retries'
3386
+ ELSE 'gate death before real work (worktree prep / triage / planner): attempt refunded (does not count toward redispatch cap); consecutive gate deaths: ' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING}'
3387
+ END,
3388
+ updated_at=?
3389
+ WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
3390
+ }
3313
3391
  admitWorkflowWorkItem(id, patch) {
3314
3392
  const now = nowIso();
3315
3393
  const res = this.db.query(`UPDATE workflow_work_items
@@ -4021,6 +4099,8 @@ class Store {
4021
4099
  if (changed) {
4022
4100
  const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
4023
4101
  this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
4102
+ if (itemStatus === "failed")
4103
+ this.demoteNonProductiveWorkItems(workflowRunId, finishedAt);
4024
4104
  this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
4025
4105
  }
4026
4106
  this.db.exec("COMMIT");
@@ -81,6 +81,7 @@ export interface LoopStore {
81
81
  getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
82
82
  requeueWorkflowWorkItem(id: string, patch?: {
83
83
  reason?: string;
84
+ resetAttempts?: boolean;
84
85
  }): Promise<WorkflowWorkItem>;
85
86
  listWorkflowInvocations(opts?: {
86
87
  limit?: number;
@@ -174,6 +175,7 @@ export declare class LocalStore implements LoopStore {
174
175
  getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
175
176
  requeueWorkflowWorkItem(id: string, patch?: {
176
177
  reason?: string;
178
+ resetAttempts?: boolean;
177
179
  }): Promise<WorkflowWorkItem>;
178
180
  listWorkflowInvocations(opts?: Parameters<Store["listWorkflowInvocations"]>[0]): Promise<WorkflowInvocation[]>;
179
181
  getWorkflowInvocation(id: string): Promise<WorkflowInvocation | undefined>;
@@ -242,6 +244,7 @@ export declare class ApiStore implements LoopStore {
242
244
  getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
243
245
  requeueWorkflowWorkItem(_id: string, _patch?: {
244
246
  reason?: string;
247
+ resetAttempts?: boolean;
245
248
  }): Promise<WorkflowWorkItem>;
246
249
  listWorkflowInvocations(opts?: Parameters<LoopStore["listWorkflowInvocations"]>[0]): Promise<WorkflowInvocation[]>;
247
250
  getWorkflowInvocation(id: string): Promise<WorkflowInvocation | undefined>;