@hasna/loops 0.4.27 → 0.4.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +205 -1
  2. package/README.md +232 -49
  3. package/dist/api/index.d.ts +20 -19
  4. package/dist/api/index.js +7770 -1342
  5. package/dist/cli/index.js +2976 -966
  6. package/dist/cli/safe-error-context.d.ts +1 -0
  7. package/dist/daemon/daemon.d.ts +1 -0
  8. package/dist/daemon/index.js +1899 -499
  9. package/dist/generated/storage-kit/index.d.ts +1 -1
  10. package/dist/generated/storage-kit/mode.d.ts +6 -12
  11. package/dist/index.d.ts +3 -3
  12. package/dist/index.js +5319 -1096
  13. package/dist/lib/advancement.d.ts +52 -0
  14. package/dist/lib/agent-adapter.d.ts +20 -2
  15. package/dist/lib/auth/route-policy.d.ts +14 -0
  16. package/dist/lib/auth/tenant-auth.d.ts +38 -0
  17. package/dist/lib/cloud/mode.d.ts +2 -8
  18. package/dist/lib/cloud/storage.d.ts +1 -2
  19. package/dist/lib/cloud/transport.d.ts +4 -18
  20. package/dist/lib/errors.d.ts +43 -1
  21. package/dist/lib/executor.d.ts +9 -0
  22. package/dist/lib/format.d.ts +2 -2
  23. package/dist/lib/goal/runner.d.ts +36 -3
  24. package/dist/lib/health.d.ts +1 -1
  25. package/dist/lib/labels.d.ts +4 -0
  26. package/dist/lib/loop-status.d.ts +4 -0
  27. package/dist/lib/migration.d.ts +70 -17
  28. package/dist/lib/mode.d.ts +2 -5
  29. package/dist/lib/mode.js +28 -37
  30. package/dist/lib/route/fields.d.ts +8 -0
  31. package/dist/lib/route/index.d.ts +2 -2
  32. package/dist/lib/route/throttle.d.ts +7 -0
  33. package/dist/lib/route/types.d.ts +3 -0
  34. package/dist/lib/run-completion.d.ts +18 -0
  35. package/dist/lib/scheduler.d.ts +5 -11
  36. package/dist/lib/storage/contract.d.ts +15 -1
  37. package/dist/lib/storage/index.d.ts +1 -1
  38. package/dist/lib/storage/index.js +4083 -486
  39. package/dist/lib/storage/pg-executor.d.ts +10 -5
  40. package/dist/lib/storage/postgres-loop-storage.d.ts +52 -26
  41. package/dist/lib/storage/postgres-schema.d.ts +8 -0
  42. package/dist/lib/storage/postgres-schema.js +1326 -1
  43. package/dist/lib/storage/postgres.d.ts +3 -1
  44. package/dist/lib/storage/postgres.js +1356 -27
  45. package/dist/lib/storage/provider-credentials.d.ts +84 -0
  46. package/dist/lib/storage/shared-database-transfer.d.ts +136 -0
  47. package/dist/lib/storage/sqlite.d.ts +15 -2
  48. package/dist/lib/storage/sqlite.js +1379 -217
  49. package/dist/lib/storage/tenant-backfill-s3.d.ts +53 -0
  50. package/dist/lib/storage/tenant-backfill.d.ts +40 -0
  51. package/dist/lib/store/index.d.ts +14 -8
  52. package/dist/lib/store.d.ts +129 -7
  53. package/dist/lib/store.js +1352 -218
  54. package/dist/lib/templates.d.ts +11 -0
  55. package/dist/lib/workflow-events.d.ts +6 -0
  56. package/dist/lib/workflow-provenance.d.ts +8 -0
  57. package/dist/lib/workflow-runner.d.ts +52 -4
  58. package/dist/mcp/index.js +2074 -657
  59. package/dist/runner/index.d.ts +20 -2
  60. package/dist/runner/index.js +2146 -183
  61. package/dist/sdk/http.d.ts +364 -4
  62. package/dist/sdk/http.js +379 -1
  63. package/dist/sdk/index.d.ts +7 -2
  64. package/dist/sdk/index.js +2341 -742
  65. package/dist/serve/index.d.ts +14 -0
  66. package/dist/serve/index.js +13269 -1830
  67. package/dist/types.d.ts +75 -3
  68. package/docs/AUTOMATION_RUNTIME_DESIGN.md +32 -32
  69. package/docs/CUTOVER-RUNBOOK.md +158 -31
  70. package/docs/DEPLOYMENT_MODES.md +84 -56
  71. package/docs/RUNTIME_BOUNDARY.md +26 -26
  72. package/docs/SHARED-DATABASE-TRANSFER.md +95 -0
  73. package/docs/SHARED_KIT_EXTRACTION_INVENTORY.md +631 -0
  74. package/docs/TRANSCRIPT_LOOP_PATTERNS.md +3 -3
  75. package/docs/UNIFIED_PRODUCT_CONTRACT.md +365 -0
  76. package/docs/USAGE.md +150 -56
  77. package/docs/workflows/transcript-feedback-to-loops.json +2 -2
  78. package/package.json +8 -4
  79. package/dist/lib/storage/pg-runner-claim.d.ts +0 -40
@@ -0,0 +1,53 @@
1
+ import type { PoolQueryClient } from "../../generated/storage-kit/query.js";
2
+ import { loadTenantBackfillBundle } from "./tenant-backfill.js";
3
+ export declare const TENANT_BACKFILL_MAX_BYTES: number;
4
+ export interface TenantBackfillS3Client {
5
+ list(input: {
6
+ prefix: string;
7
+ maxKeys: number;
8
+ }): Promise<{
9
+ contents?: Array<{
10
+ key: string;
11
+ size?: number;
12
+ }>;
13
+ isTruncated?: boolean;
14
+ }>;
15
+ file(key: string): {
16
+ bytes(): Promise<Uint8Array>;
17
+ };
18
+ delete(key: string): Promise<void>;
19
+ }
20
+ interface TaskRoleCredentials {
21
+ accessKeyId: string;
22
+ secretAccessKey: string;
23
+ sessionToken: string;
24
+ }
25
+ export type TenantBackfillFetch = (input: string, init?: RequestInit) => Promise<Response>;
26
+ interface TenantBackfillS3Options {
27
+ bucket: string;
28
+ region: string;
29
+ credentialsRelativeUri: string;
30
+ }
31
+ interface TenantBackfillS3Dependencies {
32
+ fetch?: TenantBackfillFetch;
33
+ createS3Client?: (options: TaskRoleCredentials & {
34
+ bucket: string;
35
+ region: string;
36
+ endpoint: string;
37
+ }) => TenantBackfillS3Client;
38
+ loadBundle?: typeof loadTenantBackfillBundle;
39
+ }
40
+ export interface TenantBackfillS3Result {
41
+ digest: string;
42
+ counts: {
43
+ tenants: number;
44
+ principals: number;
45
+ memberships: number;
46
+ keyBindings: number;
47
+ rowAssignments: number;
48
+ };
49
+ }
50
+ export declare function fetchEcsTaskRoleCredentials(relativeUri: string, fetchImpl?: TenantBackfillFetch): Promise<TaskRoleCredentials>;
51
+ export declare function loadApprovedTenantBackfillBundle(queryClient: PoolQueryClient, options: TenantBackfillS3Options, dependencies?: TenantBackfillS3Dependencies): Promise<TenantBackfillS3Result>;
52
+ export declare function logTenantBackfillS3Success(result: TenantBackfillS3Result): void;
53
+ export {};
@@ -0,0 +1,40 @@
1
+ import type { PoolQueryClient } from "../../generated/storage-kit/query.js";
2
+ declare const BUNDLE_SCHEMA = "open-loops.tenant-backfill/v1";
3
+ export interface TenantBackfillBundle {
4
+ schema: typeof BUNDLE_SCHEMA;
5
+ tenants: Array<{
6
+ id: string;
7
+ slug: string;
8
+ name: string;
9
+ status: "active" | "suspended";
10
+ }>;
11
+ principals: Array<{
12
+ id: string;
13
+ kind: "human" | "service" | "machine";
14
+ displayName: string;
15
+ status: "active" | "suspended";
16
+ }>;
17
+ memberships: Array<{
18
+ tenantId: string;
19
+ principalId: string;
20
+ status: "active" | "suspended";
21
+ roles: string[];
22
+ }>;
23
+ keyBindings: Array<{
24
+ kid: string;
25
+ tenantId: string;
26
+ principalId: string;
27
+ tokenKind: "api_key" | "service" | "machine";
28
+ }>;
29
+ rowAssignments: Array<{
30
+ table: string;
31
+ rowId: string;
32
+ tenantId: string;
33
+ }>;
34
+ }
35
+ export declare function parseTenantBackfillBundle(value: unknown): TenantBackfillBundle;
36
+ export declare function loadTenantBackfillBundle(client: PoolQueryClient, bundle: TenantBackfillBundle): Promise<{
37
+ digest: string;
38
+ assignments: number;
39
+ }>;
40
+ export {};
@@ -1,4 +1,4 @@
1
- import type { CreateLoopInput, CreateWorkflowInput, Loop, LoopRun, LoopStatus, RunReceipt, RunStatus, WorkflowEvent, WorkflowInvocation, WorkflowRun, WorkflowSpec, WorkflowStepRun, WorkflowWorkItem, WorkflowWorkItemStatus, WriteRunReceiptInput } from "../../types.js";
1
+ import type { CreateLoopInput, CreateWorkflowInput, Loop, LoopRun, LoopStatus, PublicWorkflowEvent, RunReceipt, RunStatus, WorkflowInvocation, WorkflowRun, WorkflowSpec, WorkflowStepRun, WorkflowWorkItem, WorkflowWorkItemStatus, WriteRunReceiptInput } from "../../types.js";
2
2
  import type { Goal, GoalPlanNode, GoalRun, GoalStatus } from "../goal/types.js";
3
3
  import type { HasnaStorageClient } from "../cloud/storage.js";
4
4
  import type { Env } from "../cloud/mode.js";
@@ -16,7 +16,7 @@ export declare class CloudUnsupportedError extends Error {
16
16
  constructor(operation: string);
17
17
  }
18
18
  /**
19
- * The single data interface for the OpenLoops client. Both {@link LocalStore}
19
+ * The single data interface for the Loops client. Both {@link LocalStore}
20
20
  * and {@link ApiStore} implement it; callers hold a `LoopStore` and never know
21
21
  * (or branch on) which one. Every method is async so the two transports share
22
22
  * one surface.
@@ -31,6 +31,7 @@ export interface LoopStore {
31
31
  requireUniqueLoop(idOrName: string): Promise<Loop>;
32
32
  listLoops(opts?: {
33
33
  status?: LoopStatus;
34
+ labels?: string[];
34
35
  limit?: number;
35
36
  offset?: number;
36
37
  archived?: boolean;
@@ -41,7 +42,7 @@ export interface LoopStore {
41
42
  archived?: boolean;
42
43
  includeArchived?: boolean;
43
44
  }): Promise<number>;
44
- updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt">>): Promise<Loop>;
45
+ updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt" | "labels">>): Promise<Loop>;
45
46
  renameLoop(id: string, name: string): Promise<Loop>;
46
47
  archiveLoop(idOrName: string): Promise<Loop>;
47
48
  unarchiveLoop(idOrName: string): Promise<Loop>;
@@ -67,7 +68,7 @@ export interface LoopStore {
67
68
  limit?: number;
68
69
  }): Promise<WorkflowRun[]>;
69
70
  listWorkflowStepRuns(workflowRunId: string): Promise<WorkflowStepRun[]>;
70
- listWorkflowEvents(workflowRunId: string, limit?: number): Promise<WorkflowEvent[]>;
71
+ listWorkflowEvents(workflowRunId: string, limit?: number): Promise<PublicWorkflowEvent[]>;
71
72
  recoverWorkflowRun(workflowRunId: string, reason?: string): Promise<{
72
73
  run: WorkflowRun;
73
74
  recoveredSteps: WorkflowStepRun[];
@@ -81,6 +82,7 @@ export interface LoopStore {
81
82
  getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
82
83
  requeueWorkflowWorkItem(id: string, patch?: {
83
84
  reason?: string;
85
+ resetAttempts?: boolean;
84
86
  }): Promise<WorkflowWorkItem>;
85
87
  listWorkflowInvocations(opts?: {
86
88
  limit?: number;
@@ -102,6 +104,7 @@ export interface LoopStore {
102
104
  listRuns(opts?: {
103
105
  loopId?: string;
104
106
  status?: RunStatus;
107
+ labels?: string[];
105
108
  limit?: number;
106
109
  offset?: number;
107
110
  }): Promise<LoopRun[]>;
@@ -146,7 +149,7 @@ export declare class LocalStore implements LoopStore {
146
149
  archived?: boolean;
147
150
  includeArchived?: boolean;
148
151
  }): Promise<number>;
149
- updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt">>): Promise<Loop>;
152
+ updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt" | "labels">>): Promise<Loop>;
150
153
  renameLoop(id: string, name: string): Promise<Loop>;
151
154
  archiveLoop(idOrName: string): Promise<Loop>;
152
155
  unarchiveLoop(idOrName: string): Promise<Loop>;
@@ -164,7 +167,7 @@ export declare class LocalStore implements LoopStore {
164
167
  requireWorkflowRun(id: string): Promise<WorkflowRun>;
165
168
  listWorkflowRuns(opts?: Parameters<Store["listWorkflowRuns"]>[0]): Promise<WorkflowRun[]>;
166
169
  listWorkflowStepRuns(workflowRunId: string): Promise<WorkflowStepRun[]>;
167
- listWorkflowEvents(workflowRunId: string, limit?: number): Promise<WorkflowEvent[]>;
170
+ listWorkflowEvents(workflowRunId: string, limit?: number): Promise<PublicWorkflowEvent[]>;
168
171
  recoverWorkflowRun(workflowRunId: string, reason?: string): Promise<{
169
172
  run: WorkflowRun;
170
173
  recoveredSteps: WorkflowStepRun[];
@@ -174,6 +177,7 @@ export declare class LocalStore implements LoopStore {
174
177
  getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
175
178
  requeueWorkflowWorkItem(id: string, patch?: {
176
179
  reason?: string;
180
+ resetAttempts?: boolean;
177
181
  }): Promise<WorkflowWorkItem>;
178
182
  listWorkflowInvocations(opts?: Parameters<Store["listWorkflowInvocations"]>[0]): Promise<WorkflowInvocation[]>;
179
183
  getWorkflowInvocation(id: string): Promise<WorkflowInvocation | undefined>;
@@ -214,10 +218,11 @@ export declare class ApiStore implements LoopStore {
214
218
  archived?: boolean;
215
219
  includeArchived?: boolean;
216
220
  }): Promise<number>;
217
- updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt">>): Promise<Loop>;
221
+ updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt" | "labels">>): Promise<Loop>;
218
222
  renameLoop(id: string, name: string): Promise<Loop>;
219
223
  archiveLoop(idOrName: string): Promise<Loop>;
220
224
  unarchiveLoop(idOrName: string): Promise<Loop>;
225
+ private mutateArchiveState;
221
226
  deleteLoop(idOrName: string): Promise<boolean>;
222
227
  createWorkflow(input: CreateWorkflowInput): Promise<WorkflowSpec>;
223
228
  getWorkflow(id: string): Promise<WorkflowSpec | undefined>;
@@ -232,7 +237,7 @@ export declare class ApiStore implements LoopStore {
232
237
  requireWorkflowRun(id: string): Promise<WorkflowRun>;
233
238
  listWorkflowRuns(opts?: Parameters<LoopStore["listWorkflowRuns"]>[0]): Promise<WorkflowRun[]>;
234
239
  listWorkflowStepRuns(workflowRunId: string): Promise<WorkflowStepRun[]>;
235
- listWorkflowEvents(workflowRunId: string, limit?: number): Promise<WorkflowEvent[]>;
240
+ listWorkflowEvents(workflowRunId: string, limit?: number): Promise<PublicWorkflowEvent[]>;
236
241
  recoverWorkflowRun(workflowRunId: string, reason?: string): Promise<{
237
242
  run: WorkflowRun;
238
243
  recoveredSteps: WorkflowStepRun[];
@@ -242,6 +247,7 @@ export declare class ApiStore implements LoopStore {
242
247
  getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
243
248
  requeueWorkflowWorkItem(_id: string, _patch?: {
244
249
  reason?: string;
250
+ resetAttempts?: boolean;
245
251
  }): Promise<WorkflowWorkItem>;
246
252
  listWorkflowInvocations(opts?: Parameters<LoopStore["listWorkflowInvocations"]>[0]): Promise<WorkflowInvocation[]>;
247
253
  getWorkflowInvocation(id: string): Promise<WorkflowInvocation | undefined>;
@@ -1,13 +1,31 @@
1
- import type { CreateLoopInput, CreateWorkflowInvocationInput, CreateWorkflowInput, Goal, GoalAutoExecute, GoalPlanNode, GoalRun, GoalStatus, Loop, LoopRun, LoopStatus, RunReceipt, RunStatus, TimeoutMs, WriteRunReceiptInput, WorkflowEvent, WorkflowInvocation, WorkflowRun, WorkflowRunStatus, WorkflowSpec, WorkflowStepRun, WorkflowWorkItem, WorkflowWorkItemStatus, UpsertWorkflowWorkItemInput } from "../types.js";
1
+ import type { CreateLoopInput, CreateWorkflowInvocationInput, CreateWorkflowInput, Goal, GoalAutoExecute, GoalPlanNode, GoalRun, GoalStatus, Loop, LoopRun, LoopStatus, RecoveredLeaseRunSnapshotEntry, RunReceipt, RunStatus, TimeoutMs, StoredWorkflowEvent, WriteRunReceiptInput, WorkflowInvocation, WorkflowRun, WorkflowRunStatus, WorkflowSpec, WorkflowStepRun, WorkflowWorkItem, WorkflowWorkItemStatus, UpsertWorkflowWorkItemInput } from "../types.js";
2
+ import { type InitialAgentSessionContractEvent } from "./workflow-provenance.js";
2
3
  interface DaemonLeaseFence {
3
4
  daemonLeaseId?: string;
4
5
  now?: Date;
5
6
  claimToken?: string;
7
+ recoveredRun?: RecoveredLeaseRunSnapshotEntry;
6
8
  }
9
+ export interface WorkflowRecoveryContext {
10
+ mode?: "internal" | "operator" | "runner";
11
+ now?: Date;
12
+ loopRunId?: string;
13
+ claimedBy?: string;
14
+ claimToken?: string;
15
+ }
16
+ export type LoopSchedulingState = Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor">;
17
+ export interface CircuitBreakerTransitionResult {
18
+ loop: Loop;
19
+ marker: LoopRun;
20
+ }
21
+ export declare const GENERATED_ROUTE_TEMPLATE_IDS: Set<string>;
22
+ export declare const GENERATED_ROUTE_KEYS: Set<string>;
23
+ export declare function isGeneratedRouteTemplate(routeKey: string, templateId: string): boolean;
7
24
  export interface LoopRow {
8
25
  id: string;
9
26
  name: string;
10
27
  description: string | null;
28
+ labels_json: string | null;
11
29
  status: string;
12
30
  archived_at: string | null;
13
31
  archived_from_status: string | null;
@@ -89,6 +107,7 @@ export interface WorkflowRunRow {
89
107
  work_item_id: string | null;
90
108
  scheduled_for: string | null;
91
109
  idempotency_key: string | null;
110
+ workflow_definition_hash: string | null;
92
111
  manifest_path: string | null;
93
112
  status: string;
94
113
  started_at: string | null;
@@ -133,6 +152,8 @@ export interface WorkflowWorkItemRow {
133
152
  priority: number;
134
153
  status: string;
135
154
  attempts: number;
155
+ /** Nullable for rows read before migration 0011 backfills the column. */
156
+ gate_deaths: number | null;
136
157
  next_attempt_at: string | null;
137
158
  lease_expires_at: string | null;
138
159
  workflow_id: string | null;
@@ -256,12 +277,13 @@ export declare function rowToWorkflowStepRun(row: WorkflowStepRunRow): WorkflowS
256
277
  export declare function rowToGoal(row: GoalRow): Goal;
257
278
  export declare function rowToGoalPlanNode(row: GoalPlanNodeRow): GoalPlanNode;
258
279
  export declare function rowToGoalRun(row: GoalRunRow): GoalRun;
259
- export declare function rowToWorkflowEvent(row: WorkflowEventRow): WorkflowEvent;
280
+ export declare function rowToWorkflowEvent(row: WorkflowEventRow): StoredWorkflowEvent;
281
+ export declare function isLiveStepProcess(pid: number, stepStartedAt: string | null | undefined): boolean;
260
282
  export declare function rowToLease(row: LeaseRow): DaemonLease;
261
283
  export interface ClaimRunResult {
262
284
  run: LoopRun;
263
285
  loop: Loop;
264
- claimToken?: string;
286
+ claimToken: string;
265
287
  }
266
288
  export interface CreateWorkflowRunInput {
267
289
  workflow: WorkflowSpec;
@@ -272,6 +294,8 @@ export interface CreateWorkflowRunInput {
272
294
  invocationId?: string;
273
295
  workItemId?: string;
274
296
  daemonLeaseId?: string;
297
+ /** Internal deterministic fault-injection seam used to verify atomic initial event persistence. */
298
+ beforeInitialWorkflowEventPersist?: (event: InitialAgentSessionContractEvent) => void;
275
299
  }
276
300
  export interface CreateGoalInput {
277
301
  objective: string;
@@ -304,6 +328,11 @@ export interface RecoverExpiredRunLeasesResult {
304
328
  /** Runs whose lease expired while their process (group) is still alive; lease deferred. */
305
329
  deferred: LoopRun[];
306
330
  }
331
+ export interface RecoveredLeaseRunPage {
332
+ runs: LoopRun[];
333
+ snapshot?: RecoveredLeaseRunSnapshotEntry[];
334
+ nextOffset?: number;
335
+ }
307
336
  export interface PruneHistoryOptions {
308
337
  /** Delete terminal runs whose created_at is older than this many days. */
309
338
  maxAgeDays?: number;
@@ -368,9 +397,55 @@ export interface RecordGoalEventInput {
368
397
  rawResponse?: unknown;
369
398
  }
370
399
  export declare function workItemStatusForLoopRun(status: RunStatus, attempt: number, maxAttempts: number | undefined): WorkflowWorkItemStatus | undefined;
400
+ /**
401
+ * `exit(75)` = `EX_TEMPFAIL`: the sysexits.h "temporary failure, retry later"
402
+ * code. A gate/worker step that exits 75 (e.g. an account-quota probe that is
403
+ * still dry) is signalling "not now", not a real failed attempt — so it must
404
+ * not burn the todos-task redispatch cap and must leave the work item
405
+ * requeueable rather than persisting as a terminal, dedupe-forever row.
406
+ */
407
+ export declare const WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
408
+ /**
409
+ * Workflow step ids that run BEFORE the worker does any real work. A failure in
410
+ * one of these (triage/planner gate, or the pre-step worktree preparation) that
411
+ * dies quickly is a "gate death": the run never executed the worker, so it must
412
+ * not count toward the redispatch cap (otherwise a purely infrastructural fault
413
+ * — e.g. a stale worktree registration — silently dead-letters a task that
414
+ * never actually ran).
415
+ */
416
+ export declare const GATE_STEP_IDS: ReadonlySet<string>;
417
+ /** A gate-step failure only counts as a gate death when it dies before any real
418
+ * work could have happened. Worktree-prep deaths are always gate deaths (they
419
+ * fail before the agent is spawned) regardless of this bound. */
420
+ export declare const GATE_DEATH_MAX_DURATION_MS = 60000;
421
+ /**
422
+ * Secondary ceiling for CONSECUTIVE gate deaths. Gate deaths refund their
423
+ * redispatch attempt (the worker never ran), which is correct for transient
424
+ * infrastructure faults — but a deterministic fault (e.g. a permanently broken
425
+ * repo path) would otherwise retry forever at the backoff floor. After this
426
+ * many consecutive gate deaths the work item is dead-lettered (visible in
427
+ * drain reports) instead of spinning; any run that reaches the worker resets
428
+ * the streak, and an operator requeue (attempts reset) re-arms it. At the
429
+ * ~2–4 minute refunded-attempts backoff this bounds a deterministic fault to
430
+ * roughly an hour of auto-retry before it demands an operator.
431
+ */
432
+ export declare const GATE_DEATH_CEILING = 20;
433
+ export type NonProductiveFailureKind = "tempfail" | "gate-death";
434
+ type ClassifiableStepRun = Pick<WorkflowStepRun, "stepId" | "status" | "exitCode" | "durationMs" | "error">;
435
+ /**
436
+ * Classify a just-finalized *failed* workflow run's decisive failing step. A
437
+ * non-productive finish (a tempfail retry-signal, or a gate death before the
438
+ * worker ran) must not count toward the todos-task redispatch cap. Returns the
439
+ * non-productive kind, or `undefined` when the run represents a real worker
440
+ * attempt that legitimately counts toward the cap. Pure/exported for tests.
441
+ */
442
+ export declare function classifyNonProductiveStepFailure(steps: ClassifiableStepRun[]): NonProductiveFailureKind | undefined;
371
443
  export declare function scrubbedOrNull(value: string | undefined | null): string | null;
372
444
  /** Scrub secrets then bound size before persisting run stdout/stderr. */
373
445
  export declare function persistedRunOutput(value: string | undefined | null): string | null;
446
+ /** Scrub structured string leaves before stringify, then scrub the encoded JSON. */
447
+ export declare function persistedJson(value: unknown): string;
448
+ export declare function persistedWorkflowEventPayload(payload: Record<string, unknown> | undefined | null): string | null;
374
449
  export declare class Store {
375
450
  private db;
376
451
  private rootDir;
@@ -398,9 +473,11 @@ export declare class Store {
398
473
  getLoop(id: string): Loop | undefined;
399
474
  findLoopByName(name: string): Loop | undefined;
400
475
  requireUniqueLoop(idOrName: string): Loop;
476
+ private requireArchiveMutationLoop;
401
477
  requireLoop(idOrName: string): Loop;
402
478
  listLoops(opts?: {
403
479
  status?: LoopStatus;
480
+ labels?: string[];
404
481
  limit?: number;
405
482
  offset?: number;
406
483
  archived?: boolean;
@@ -409,7 +486,12 @@ export declare class Store {
409
486
  }): Loop[];
410
487
  private withLatestRunSummaries;
411
488
  dueLoops(now: Date, limit?: number): Loop[];
412
- updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt">>, opts?: DaemonLeaseFence): Loop;
489
+ updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt" | "labels">>, opts?: DaemonLeaseFence): Loop;
490
+ advanceLoopIfCurrent(id: string, expected: LoopSchedulingState, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor">>, opts?: DaemonLeaseFence): Loop | undefined;
491
+ tripCircuitBreakerIfCurrent(id: string, expected: LoopSchedulingState, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor">>, marker: {
492
+ scheduledFor: string;
493
+ reason: string;
494
+ }, opts?: DaemonLeaseFence): CircuitBreakerTransitionResult | undefined;
413
495
  private activeLoopReferenceCount;
414
496
  private archiveWorkflowIfUnreferenced;
415
497
  retargetWorkflowLoop(idOrName: string, workflowId: string, opts?: DaemonLeaseFence & {
@@ -490,9 +572,41 @@ export declare class Store {
490
572
  * step on that account right now — exactly the concurrency to bound.
491
573
  */
492
574
  countRunningWorkflowStepsByAuthProfile(): Record<string, number>;
575
+ /**
576
+ * Requeue a terminal admission work item for the next task/event delivery.
577
+ * By default `attempts` is preserved (used by the bounded stale-terminal
578
+ * re-admission on the route path, which must keep counting toward the cap).
579
+ * Pass `resetAttempts: true` for the operator unwedge (`loops routes requeue`)
580
+ * so a manual requeue is DURABLE rather than one-shot: without the reset a
581
+ * capped item that finishes terminal once more re-caps instantly.
582
+ */
493
583
  requeueWorkflowWorkItem(id: string, patch?: {
494
584
  reason?: string;
585
+ resetAttempts?: boolean;
495
586
  }): WorkflowWorkItem;
587
+ /**
588
+ * Transition a terminal admission work item to `dead_letter`. Used by the
589
+ * route path when a still-actionable todos task has exhausted the redispatch
590
+ * cap: instead of silently deduping the same terminal row forever (the "black
591
+ * hole" — `considered=N created=0` with no signal), the item is moved to a
592
+ * visible `dead_letter` state so drain reports can surface + count it and an
593
+ * operator can `loops routes requeue` it. Idempotent: a no-op on an item that
594
+ * is already dead-lettered.
595
+ */
596
+ deadLetterWorkflowWorkItem(id: string, patch?: {
597
+ reason?: string;
598
+ }): WorkflowWorkItem;
599
+ /**
600
+ * Refund a redispatch attempt for a *failed* run that never did real work.
601
+ * Called from {@link finalizeWorkflowRun} the moment a work item is set
602
+ * `failed`, so the todos-task redispatch cap only ever counts real worker
603
+ * attempts. A tempfail (`exit 75`) is additionally made requeueable (dropped
604
+ * back to `queued`, bindings cleared) so its "retry later" contract fires on
605
+ * the next drain instead of persisting as a terminal, dedupe-forever row. A
606
+ * gate death stays `failed` (the bounded re-admission picks it up after
607
+ * backoff once the underlying infra fault clears). Both floor attempts at 0.
608
+ */
609
+ private demoteNonProductiveWorkItems;
496
610
  admitWorkflowWorkItem(id: string, patch: {
497
611
  workflowId: string;
498
612
  loopId: string;
@@ -553,7 +667,7 @@ export declare class Store {
553
667
  stderr?: string;
554
668
  payload?: Record<string, unknown>;
555
669
  }, opts?: DaemonLeaseFence): WorkflowStepRun;
556
- recoverWorkflowRun(workflowRunId: string, reason?: string): {
670
+ recoverWorkflowRun(workflowRunId: string, reason?: string, _context?: WorkflowRecoveryContext): {
557
671
  run: WorkflowRun;
558
672
  recoveredSteps: WorkflowStepRun[];
559
673
  };
@@ -561,8 +675,8 @@ export declare class Store {
561
675
  skipWorkflowStepRun(workflowRunId: string, stepId: string, reason: string, opts?: DaemonLeaseFence): WorkflowStepRun;
562
676
  finalizeWorkflowRun(workflowRunId: string, status: WorkflowRunStatus, patch?: Partial<Pick<WorkflowRun, "finishedAt" | "durationMs" | "error">>, opts?: DaemonLeaseFence): WorkflowRun;
563
677
  cancelWorkflowRun(workflowRunId: string, reason?: string): WorkflowRun;
564
- appendWorkflowEvent(workflowRunId: string, eventType: string, stepId?: string, payload?: Record<string, unknown>): WorkflowEvent;
565
- listWorkflowEvents(workflowRunId: string, limit?: number): WorkflowEvent[];
678
+ appendWorkflowEvent(workflowRunId: string, eventType: string, stepId?: string, payload?: Record<string, unknown>): StoredWorkflowEvent;
679
+ listWorkflowEvents(workflowRunId: string, limit?: number): StoredWorkflowEvent[];
566
680
  hasRunningRun(loopId: string): boolean;
567
681
  hasRunningRunForSlot(loopId: string, scheduledFor: string): boolean;
568
682
  private hasBlockingRunningRunForOtherSlot;
@@ -588,9 +702,15 @@ export declare class Store {
588
702
  listRuns(opts?: {
589
703
  loopId?: string;
590
704
  status?: RunStatus;
705
+ labels?: string[];
591
706
  limit?: number;
592
707
  offset?: number;
593
708
  }): LoopRun[];
709
+ listRecoveredLeaseRunsPage(opts?: {
710
+ snapshot?: RecoveredLeaseRunSnapshotEntry[];
711
+ offset?: number;
712
+ limit?: number;
713
+ }): RecoveredLeaseRunPage;
594
714
  writeRunReceipt(input: WriteRunReceiptInput, opts?: {
595
715
  now?: Date;
596
716
  }): RunReceipt;
@@ -607,6 +727,7 @@ export declare class Store {
607
727
  recoverExpiredRunLeases(now?: Date, opts?: DaemonLeaseFence & {
608
728
  limit?: number;
609
729
  scanLimit?: number;
730
+ runId?: string;
610
731
  }): LoopRun[];
611
732
  /**
612
733
  * Recover expired run leases and report both outcomes: runs abandoned (no
@@ -617,6 +738,7 @@ export declare class Store {
617
738
  recoverExpiredRunLeasesDetailed(now?: Date, opts?: DaemonLeaseFence & {
618
739
  limit?: number;
619
740
  scanLimit?: number;
741
+ runId?: string;
620
742
  }): RecoverExpiredRunLeasesResult;
621
743
  expireLoops(now?: Date, opts?: DaemonLeaseFence): Loop[];
622
744
  countLoops(status?: LoopStatus, opts?: {