@hasna/loops 0.4.27 → 0.4.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +205 -1
- package/README.md +232 -49
- package/dist/api/index.d.ts +20 -19
- package/dist/api/index.js +7770 -1342
- package/dist/cli/index.js +2976 -966
- package/dist/cli/safe-error-context.d.ts +1 -0
- package/dist/daemon/daemon.d.ts +1 -0
- package/dist/daemon/index.js +1899 -499
- package/dist/generated/storage-kit/index.d.ts +1 -1
- package/dist/generated/storage-kit/mode.d.ts +6 -12
- package/dist/index.d.ts +3 -3
- package/dist/index.js +5319 -1096
- package/dist/lib/advancement.d.ts +52 -0
- package/dist/lib/agent-adapter.d.ts +20 -2
- package/dist/lib/auth/route-policy.d.ts +14 -0
- package/dist/lib/auth/tenant-auth.d.ts +38 -0
- package/dist/lib/cloud/mode.d.ts +2 -8
- package/dist/lib/cloud/storage.d.ts +1 -2
- package/dist/lib/cloud/transport.d.ts +4 -18
- package/dist/lib/errors.d.ts +43 -1
- package/dist/lib/executor.d.ts +9 -0
- package/dist/lib/format.d.ts +2 -2
- package/dist/lib/goal/runner.d.ts +36 -3
- package/dist/lib/health.d.ts +1 -1
- package/dist/lib/labels.d.ts +4 -0
- package/dist/lib/loop-status.d.ts +4 -0
- package/dist/lib/migration.d.ts +70 -17
- package/dist/lib/mode.d.ts +2 -5
- package/dist/lib/mode.js +28 -37
- package/dist/lib/route/fields.d.ts +8 -0
- package/dist/lib/route/index.d.ts +2 -2
- package/dist/lib/route/throttle.d.ts +7 -0
- package/dist/lib/route/types.d.ts +3 -0
- package/dist/lib/run-completion.d.ts +18 -0
- package/dist/lib/scheduler.d.ts +5 -11
- package/dist/lib/storage/contract.d.ts +15 -1
- package/dist/lib/storage/index.d.ts +1 -1
- package/dist/lib/storage/index.js +4083 -486
- package/dist/lib/storage/pg-executor.d.ts +10 -5
- package/dist/lib/storage/postgres-loop-storage.d.ts +52 -26
- package/dist/lib/storage/postgres-schema.d.ts +8 -0
- package/dist/lib/storage/postgres-schema.js +1326 -1
- package/dist/lib/storage/postgres.d.ts +3 -1
- package/dist/lib/storage/postgres.js +1356 -27
- package/dist/lib/storage/provider-credentials.d.ts +84 -0
- package/dist/lib/storage/shared-database-transfer.d.ts +136 -0
- package/dist/lib/storage/sqlite.d.ts +15 -2
- package/dist/lib/storage/sqlite.js +1379 -217
- package/dist/lib/storage/tenant-backfill-s3.d.ts +53 -0
- package/dist/lib/storage/tenant-backfill.d.ts +40 -0
- package/dist/lib/store/index.d.ts +14 -8
- package/dist/lib/store.d.ts +129 -7
- package/dist/lib/store.js +1352 -218
- package/dist/lib/templates.d.ts +11 -0
- package/dist/lib/workflow-events.d.ts +6 -0
- package/dist/lib/workflow-provenance.d.ts +8 -0
- package/dist/lib/workflow-runner.d.ts +52 -4
- package/dist/mcp/index.js +2074 -657
- package/dist/runner/index.d.ts +20 -2
- package/dist/runner/index.js +2146 -183
- package/dist/sdk/http.d.ts +364 -4
- package/dist/sdk/http.js +379 -1
- package/dist/sdk/index.d.ts +7 -2
- package/dist/sdk/index.js +2341 -742
- package/dist/serve/index.d.ts +14 -0
- package/dist/serve/index.js +13269 -1830
- package/dist/types.d.ts +75 -3
- package/docs/AUTOMATION_RUNTIME_DESIGN.md +32 -32
- package/docs/CUTOVER-RUNBOOK.md +158 -31
- package/docs/DEPLOYMENT_MODES.md +84 -56
- package/docs/RUNTIME_BOUNDARY.md +26 -26
- package/docs/SHARED-DATABASE-TRANSFER.md +95 -0
- package/docs/SHARED_KIT_EXTRACTION_INVENTORY.md +631 -0
- package/docs/TRANSCRIPT_LOOP_PATTERNS.md +3 -3
- package/docs/UNIFIED_PRODUCT_CONTRACT.md +365 -0
- package/docs/USAGE.md +150 -56
- package/docs/workflows/transcript-feedback-to-loops.json +2 -2
- package/package.json +8 -4
- package/dist/lib/storage/pg-runner-claim.d.ts +0 -40
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { PoolQueryClient } from "../../generated/storage-kit/query.js";
|
|
2
|
+
import { loadTenantBackfillBundle } from "./tenant-backfill.js";
|
|
3
|
+
export declare const TENANT_BACKFILL_MAX_BYTES: number;
|
|
4
|
+
export interface TenantBackfillS3Client {
|
|
5
|
+
list(input: {
|
|
6
|
+
prefix: string;
|
|
7
|
+
maxKeys: number;
|
|
8
|
+
}): Promise<{
|
|
9
|
+
contents?: Array<{
|
|
10
|
+
key: string;
|
|
11
|
+
size?: number;
|
|
12
|
+
}>;
|
|
13
|
+
isTruncated?: boolean;
|
|
14
|
+
}>;
|
|
15
|
+
file(key: string): {
|
|
16
|
+
bytes(): Promise<Uint8Array>;
|
|
17
|
+
};
|
|
18
|
+
delete(key: string): Promise<void>;
|
|
19
|
+
}
|
|
20
|
+
interface TaskRoleCredentials {
|
|
21
|
+
accessKeyId: string;
|
|
22
|
+
secretAccessKey: string;
|
|
23
|
+
sessionToken: string;
|
|
24
|
+
}
|
|
25
|
+
export type TenantBackfillFetch = (input: string, init?: RequestInit) => Promise<Response>;
|
|
26
|
+
interface TenantBackfillS3Options {
|
|
27
|
+
bucket: string;
|
|
28
|
+
region: string;
|
|
29
|
+
credentialsRelativeUri: string;
|
|
30
|
+
}
|
|
31
|
+
interface TenantBackfillS3Dependencies {
|
|
32
|
+
fetch?: TenantBackfillFetch;
|
|
33
|
+
createS3Client?: (options: TaskRoleCredentials & {
|
|
34
|
+
bucket: string;
|
|
35
|
+
region: string;
|
|
36
|
+
endpoint: string;
|
|
37
|
+
}) => TenantBackfillS3Client;
|
|
38
|
+
loadBundle?: typeof loadTenantBackfillBundle;
|
|
39
|
+
}
|
|
40
|
+
export interface TenantBackfillS3Result {
|
|
41
|
+
digest: string;
|
|
42
|
+
counts: {
|
|
43
|
+
tenants: number;
|
|
44
|
+
principals: number;
|
|
45
|
+
memberships: number;
|
|
46
|
+
keyBindings: number;
|
|
47
|
+
rowAssignments: number;
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
export declare function fetchEcsTaskRoleCredentials(relativeUri: string, fetchImpl?: TenantBackfillFetch): Promise<TaskRoleCredentials>;
|
|
51
|
+
export declare function loadApprovedTenantBackfillBundle(queryClient: PoolQueryClient, options: TenantBackfillS3Options, dependencies?: TenantBackfillS3Dependencies): Promise<TenantBackfillS3Result>;
|
|
52
|
+
export declare function logTenantBackfillS3Success(result: TenantBackfillS3Result): void;
|
|
53
|
+
export {};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { PoolQueryClient } from "../../generated/storage-kit/query.js";
|
|
2
|
+
declare const BUNDLE_SCHEMA = "open-loops.tenant-backfill/v1";
|
|
3
|
+
export interface TenantBackfillBundle {
|
|
4
|
+
schema: typeof BUNDLE_SCHEMA;
|
|
5
|
+
tenants: Array<{
|
|
6
|
+
id: string;
|
|
7
|
+
slug: string;
|
|
8
|
+
name: string;
|
|
9
|
+
status: "active" | "suspended";
|
|
10
|
+
}>;
|
|
11
|
+
principals: Array<{
|
|
12
|
+
id: string;
|
|
13
|
+
kind: "human" | "service" | "machine";
|
|
14
|
+
displayName: string;
|
|
15
|
+
status: "active" | "suspended";
|
|
16
|
+
}>;
|
|
17
|
+
memberships: Array<{
|
|
18
|
+
tenantId: string;
|
|
19
|
+
principalId: string;
|
|
20
|
+
status: "active" | "suspended";
|
|
21
|
+
roles: string[];
|
|
22
|
+
}>;
|
|
23
|
+
keyBindings: Array<{
|
|
24
|
+
kid: string;
|
|
25
|
+
tenantId: string;
|
|
26
|
+
principalId: string;
|
|
27
|
+
tokenKind: "api_key" | "service" | "machine";
|
|
28
|
+
}>;
|
|
29
|
+
rowAssignments: Array<{
|
|
30
|
+
table: string;
|
|
31
|
+
rowId: string;
|
|
32
|
+
tenantId: string;
|
|
33
|
+
}>;
|
|
34
|
+
}
|
|
35
|
+
export declare function parseTenantBackfillBundle(value: unknown): TenantBackfillBundle;
|
|
36
|
+
export declare function loadTenantBackfillBundle(client: PoolQueryClient, bundle: TenantBackfillBundle): Promise<{
|
|
37
|
+
digest: string;
|
|
38
|
+
assignments: number;
|
|
39
|
+
}>;
|
|
40
|
+
export {};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CreateLoopInput, CreateWorkflowInput, Loop, LoopRun, LoopStatus, RunReceipt, RunStatus,
|
|
1
|
+
import type { CreateLoopInput, CreateWorkflowInput, Loop, LoopRun, LoopStatus, PublicWorkflowEvent, RunReceipt, RunStatus, WorkflowInvocation, WorkflowRun, WorkflowSpec, WorkflowStepRun, WorkflowWorkItem, WorkflowWorkItemStatus, WriteRunReceiptInput } from "../../types.js";
|
|
2
2
|
import type { Goal, GoalPlanNode, GoalRun, GoalStatus } from "../goal/types.js";
|
|
3
3
|
import type { HasnaStorageClient } from "../cloud/storage.js";
|
|
4
4
|
import type { Env } from "../cloud/mode.js";
|
|
@@ -16,7 +16,7 @@ export declare class CloudUnsupportedError extends Error {
|
|
|
16
16
|
constructor(operation: string);
|
|
17
17
|
}
|
|
18
18
|
/**
|
|
19
|
-
* The single data interface for the
|
|
19
|
+
* The single data interface for the Loops client. Both {@link LocalStore}
|
|
20
20
|
* and {@link ApiStore} implement it; callers hold a `LoopStore` and never know
|
|
21
21
|
* (or branch on) which one. Every method is async so the two transports share
|
|
22
22
|
* one surface.
|
|
@@ -31,6 +31,7 @@ export interface LoopStore {
|
|
|
31
31
|
requireUniqueLoop(idOrName: string): Promise<Loop>;
|
|
32
32
|
listLoops(opts?: {
|
|
33
33
|
status?: LoopStatus;
|
|
34
|
+
labels?: string[];
|
|
34
35
|
limit?: number;
|
|
35
36
|
offset?: number;
|
|
36
37
|
archived?: boolean;
|
|
@@ -41,7 +42,7 @@ export interface LoopStore {
|
|
|
41
42
|
archived?: boolean;
|
|
42
43
|
includeArchived?: boolean;
|
|
43
44
|
}): Promise<number>;
|
|
44
|
-
updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt">>): Promise<Loop>;
|
|
45
|
+
updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt" | "labels">>): Promise<Loop>;
|
|
45
46
|
renameLoop(id: string, name: string): Promise<Loop>;
|
|
46
47
|
archiveLoop(idOrName: string): Promise<Loop>;
|
|
47
48
|
unarchiveLoop(idOrName: string): Promise<Loop>;
|
|
@@ -67,7 +68,7 @@ export interface LoopStore {
|
|
|
67
68
|
limit?: number;
|
|
68
69
|
}): Promise<WorkflowRun[]>;
|
|
69
70
|
listWorkflowStepRuns(workflowRunId: string): Promise<WorkflowStepRun[]>;
|
|
70
|
-
listWorkflowEvents(workflowRunId: string, limit?: number): Promise<
|
|
71
|
+
listWorkflowEvents(workflowRunId: string, limit?: number): Promise<PublicWorkflowEvent[]>;
|
|
71
72
|
recoverWorkflowRun(workflowRunId: string, reason?: string): Promise<{
|
|
72
73
|
run: WorkflowRun;
|
|
73
74
|
recoveredSteps: WorkflowStepRun[];
|
|
@@ -81,6 +82,7 @@ export interface LoopStore {
|
|
|
81
82
|
getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
|
|
82
83
|
requeueWorkflowWorkItem(id: string, patch?: {
|
|
83
84
|
reason?: string;
|
|
85
|
+
resetAttempts?: boolean;
|
|
84
86
|
}): Promise<WorkflowWorkItem>;
|
|
85
87
|
listWorkflowInvocations(opts?: {
|
|
86
88
|
limit?: number;
|
|
@@ -102,6 +104,7 @@ export interface LoopStore {
|
|
|
102
104
|
listRuns(opts?: {
|
|
103
105
|
loopId?: string;
|
|
104
106
|
status?: RunStatus;
|
|
107
|
+
labels?: string[];
|
|
105
108
|
limit?: number;
|
|
106
109
|
offset?: number;
|
|
107
110
|
}): Promise<LoopRun[]>;
|
|
@@ -146,7 +149,7 @@ export declare class LocalStore implements LoopStore {
|
|
|
146
149
|
archived?: boolean;
|
|
147
150
|
includeArchived?: boolean;
|
|
148
151
|
}): Promise<number>;
|
|
149
|
-
updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt">>): Promise<Loop>;
|
|
152
|
+
updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt" | "labels">>): Promise<Loop>;
|
|
150
153
|
renameLoop(id: string, name: string): Promise<Loop>;
|
|
151
154
|
archiveLoop(idOrName: string): Promise<Loop>;
|
|
152
155
|
unarchiveLoop(idOrName: string): Promise<Loop>;
|
|
@@ -164,7 +167,7 @@ export declare class LocalStore implements LoopStore {
|
|
|
164
167
|
requireWorkflowRun(id: string): Promise<WorkflowRun>;
|
|
165
168
|
listWorkflowRuns(opts?: Parameters<Store["listWorkflowRuns"]>[0]): Promise<WorkflowRun[]>;
|
|
166
169
|
listWorkflowStepRuns(workflowRunId: string): Promise<WorkflowStepRun[]>;
|
|
167
|
-
listWorkflowEvents(workflowRunId: string, limit?: number): Promise<
|
|
170
|
+
listWorkflowEvents(workflowRunId: string, limit?: number): Promise<PublicWorkflowEvent[]>;
|
|
168
171
|
recoverWorkflowRun(workflowRunId: string, reason?: string): Promise<{
|
|
169
172
|
run: WorkflowRun;
|
|
170
173
|
recoveredSteps: WorkflowStepRun[];
|
|
@@ -174,6 +177,7 @@ export declare class LocalStore implements LoopStore {
|
|
|
174
177
|
getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
|
|
175
178
|
requeueWorkflowWorkItem(id: string, patch?: {
|
|
176
179
|
reason?: string;
|
|
180
|
+
resetAttempts?: boolean;
|
|
177
181
|
}): Promise<WorkflowWorkItem>;
|
|
178
182
|
listWorkflowInvocations(opts?: Parameters<Store["listWorkflowInvocations"]>[0]): Promise<WorkflowInvocation[]>;
|
|
179
183
|
getWorkflowInvocation(id: string): Promise<WorkflowInvocation | undefined>;
|
|
@@ -214,10 +218,11 @@ export declare class ApiStore implements LoopStore {
|
|
|
214
218
|
archived?: boolean;
|
|
215
219
|
includeArchived?: boolean;
|
|
216
220
|
}): Promise<number>;
|
|
217
|
-
updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt">>): Promise<Loop>;
|
|
221
|
+
updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt" | "labels">>): Promise<Loop>;
|
|
218
222
|
renameLoop(id: string, name: string): Promise<Loop>;
|
|
219
223
|
archiveLoop(idOrName: string): Promise<Loop>;
|
|
220
224
|
unarchiveLoop(idOrName: string): Promise<Loop>;
|
|
225
|
+
private mutateArchiveState;
|
|
221
226
|
deleteLoop(idOrName: string): Promise<boolean>;
|
|
222
227
|
createWorkflow(input: CreateWorkflowInput): Promise<WorkflowSpec>;
|
|
223
228
|
getWorkflow(id: string): Promise<WorkflowSpec | undefined>;
|
|
@@ -232,7 +237,7 @@ export declare class ApiStore implements LoopStore {
|
|
|
232
237
|
requireWorkflowRun(id: string): Promise<WorkflowRun>;
|
|
233
238
|
listWorkflowRuns(opts?: Parameters<LoopStore["listWorkflowRuns"]>[0]): Promise<WorkflowRun[]>;
|
|
234
239
|
listWorkflowStepRuns(workflowRunId: string): Promise<WorkflowStepRun[]>;
|
|
235
|
-
listWorkflowEvents(workflowRunId: string, limit?: number): Promise<
|
|
240
|
+
listWorkflowEvents(workflowRunId: string, limit?: number): Promise<PublicWorkflowEvent[]>;
|
|
236
241
|
recoverWorkflowRun(workflowRunId: string, reason?: string): Promise<{
|
|
237
242
|
run: WorkflowRun;
|
|
238
243
|
recoveredSteps: WorkflowStepRun[];
|
|
@@ -242,6 +247,7 @@ export declare class ApiStore implements LoopStore {
|
|
|
242
247
|
getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
|
|
243
248
|
requeueWorkflowWorkItem(_id: string, _patch?: {
|
|
244
249
|
reason?: string;
|
|
250
|
+
resetAttempts?: boolean;
|
|
245
251
|
}): Promise<WorkflowWorkItem>;
|
|
246
252
|
listWorkflowInvocations(opts?: Parameters<LoopStore["listWorkflowInvocations"]>[0]): Promise<WorkflowInvocation[]>;
|
|
247
253
|
getWorkflowInvocation(id: string): Promise<WorkflowInvocation | undefined>;
|
package/dist/lib/store.d.ts
CHANGED
|
@@ -1,13 +1,31 @@
|
|
|
1
|
-
import type { CreateLoopInput, CreateWorkflowInvocationInput, CreateWorkflowInput, Goal, GoalAutoExecute, GoalPlanNode, GoalRun, GoalStatus, Loop, LoopRun, LoopStatus, RunReceipt, RunStatus, TimeoutMs,
|
|
1
|
+
import type { CreateLoopInput, CreateWorkflowInvocationInput, CreateWorkflowInput, Goal, GoalAutoExecute, GoalPlanNode, GoalRun, GoalStatus, Loop, LoopRun, LoopStatus, RecoveredLeaseRunSnapshotEntry, RunReceipt, RunStatus, TimeoutMs, StoredWorkflowEvent, WriteRunReceiptInput, WorkflowInvocation, WorkflowRun, WorkflowRunStatus, WorkflowSpec, WorkflowStepRun, WorkflowWorkItem, WorkflowWorkItemStatus, UpsertWorkflowWorkItemInput } from "../types.js";
|
|
2
|
+
import { type InitialAgentSessionContractEvent } from "./workflow-provenance.js";
|
|
2
3
|
interface DaemonLeaseFence {
|
|
3
4
|
daemonLeaseId?: string;
|
|
4
5
|
now?: Date;
|
|
5
6
|
claimToken?: string;
|
|
7
|
+
recoveredRun?: RecoveredLeaseRunSnapshotEntry;
|
|
6
8
|
}
|
|
9
|
+
export interface WorkflowRecoveryContext {
|
|
10
|
+
mode?: "internal" | "operator" | "runner";
|
|
11
|
+
now?: Date;
|
|
12
|
+
loopRunId?: string;
|
|
13
|
+
claimedBy?: string;
|
|
14
|
+
claimToken?: string;
|
|
15
|
+
}
|
|
16
|
+
export type LoopSchedulingState = Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor">;
|
|
17
|
+
export interface CircuitBreakerTransitionResult {
|
|
18
|
+
loop: Loop;
|
|
19
|
+
marker: LoopRun;
|
|
20
|
+
}
|
|
21
|
+
export declare const GENERATED_ROUTE_TEMPLATE_IDS: Set<string>;
|
|
22
|
+
export declare const GENERATED_ROUTE_KEYS: Set<string>;
|
|
23
|
+
export declare function isGeneratedRouteTemplate(routeKey: string, templateId: string): boolean;
|
|
7
24
|
export interface LoopRow {
|
|
8
25
|
id: string;
|
|
9
26
|
name: string;
|
|
10
27
|
description: string | null;
|
|
28
|
+
labels_json: string | null;
|
|
11
29
|
status: string;
|
|
12
30
|
archived_at: string | null;
|
|
13
31
|
archived_from_status: string | null;
|
|
@@ -89,6 +107,7 @@ export interface WorkflowRunRow {
|
|
|
89
107
|
work_item_id: string | null;
|
|
90
108
|
scheduled_for: string | null;
|
|
91
109
|
idempotency_key: string | null;
|
|
110
|
+
workflow_definition_hash: string | null;
|
|
92
111
|
manifest_path: string | null;
|
|
93
112
|
status: string;
|
|
94
113
|
started_at: string | null;
|
|
@@ -133,6 +152,8 @@ export interface WorkflowWorkItemRow {
|
|
|
133
152
|
priority: number;
|
|
134
153
|
status: string;
|
|
135
154
|
attempts: number;
|
|
155
|
+
/** Nullable for rows read before migration 0011 backfills the column. */
|
|
156
|
+
gate_deaths: number | null;
|
|
136
157
|
next_attempt_at: string | null;
|
|
137
158
|
lease_expires_at: string | null;
|
|
138
159
|
workflow_id: string | null;
|
|
@@ -256,12 +277,13 @@ export declare function rowToWorkflowStepRun(row: WorkflowStepRunRow): WorkflowS
|
|
|
256
277
|
export declare function rowToGoal(row: GoalRow): Goal;
|
|
257
278
|
export declare function rowToGoalPlanNode(row: GoalPlanNodeRow): GoalPlanNode;
|
|
258
279
|
export declare function rowToGoalRun(row: GoalRunRow): GoalRun;
|
|
259
|
-
export declare function rowToWorkflowEvent(row: WorkflowEventRow):
|
|
280
|
+
export declare function rowToWorkflowEvent(row: WorkflowEventRow): StoredWorkflowEvent;
|
|
281
|
+
export declare function isLiveStepProcess(pid: number, stepStartedAt: string | null | undefined): boolean;
|
|
260
282
|
export declare function rowToLease(row: LeaseRow): DaemonLease;
|
|
261
283
|
export interface ClaimRunResult {
|
|
262
284
|
run: LoopRun;
|
|
263
285
|
loop: Loop;
|
|
264
|
-
claimToken
|
|
286
|
+
claimToken: string;
|
|
265
287
|
}
|
|
266
288
|
export interface CreateWorkflowRunInput {
|
|
267
289
|
workflow: WorkflowSpec;
|
|
@@ -272,6 +294,8 @@ export interface CreateWorkflowRunInput {
|
|
|
272
294
|
invocationId?: string;
|
|
273
295
|
workItemId?: string;
|
|
274
296
|
daemonLeaseId?: string;
|
|
297
|
+
/** Internal deterministic fault-injection seam used to verify atomic initial event persistence. */
|
|
298
|
+
beforeInitialWorkflowEventPersist?: (event: InitialAgentSessionContractEvent) => void;
|
|
275
299
|
}
|
|
276
300
|
export interface CreateGoalInput {
|
|
277
301
|
objective: string;
|
|
@@ -304,6 +328,11 @@ export interface RecoverExpiredRunLeasesResult {
|
|
|
304
328
|
/** Runs whose lease expired while their process (group) is still alive; lease deferred. */
|
|
305
329
|
deferred: LoopRun[];
|
|
306
330
|
}
|
|
331
|
+
export interface RecoveredLeaseRunPage {
|
|
332
|
+
runs: LoopRun[];
|
|
333
|
+
snapshot?: RecoveredLeaseRunSnapshotEntry[];
|
|
334
|
+
nextOffset?: number;
|
|
335
|
+
}
|
|
307
336
|
export interface PruneHistoryOptions {
|
|
308
337
|
/** Delete terminal runs whose created_at is older than this many days. */
|
|
309
338
|
maxAgeDays?: number;
|
|
@@ -368,9 +397,55 @@ export interface RecordGoalEventInput {
|
|
|
368
397
|
rawResponse?: unknown;
|
|
369
398
|
}
|
|
370
399
|
export declare function workItemStatusForLoopRun(status: RunStatus, attempt: number, maxAttempts: number | undefined): WorkflowWorkItemStatus | undefined;
|
|
400
|
+
/**
|
|
401
|
+
* `exit(75)` = `EX_TEMPFAIL`: the sysexits.h "temporary failure, retry later"
|
|
402
|
+
* code. A gate/worker step that exits 75 (e.g. an account-quota probe that is
|
|
403
|
+
* still dry) is signalling "not now", not a real failed attempt — so it must
|
|
404
|
+
* not burn the todos-task redispatch cap and must leave the work item
|
|
405
|
+
* requeueable rather than persisting as a terminal, dedupe-forever row.
|
|
406
|
+
*/
|
|
407
|
+
export declare const WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
|
|
408
|
+
/**
|
|
409
|
+
* Workflow step ids that run BEFORE the worker does any real work. A failure in
|
|
410
|
+
* one of these (triage/planner gate, or the pre-step worktree preparation) that
|
|
411
|
+
* dies quickly is a "gate death": the run never executed the worker, so it must
|
|
412
|
+
* not count toward the redispatch cap (otherwise a purely infrastructural fault
|
|
413
|
+
* — e.g. a stale worktree registration — silently dead-letters a task that
|
|
414
|
+
* never actually ran).
|
|
415
|
+
*/
|
|
416
|
+
export declare const GATE_STEP_IDS: ReadonlySet<string>;
|
|
417
|
+
/** A gate-step failure only counts as a gate death when it dies before any real
|
|
418
|
+
* work could have happened. Worktree-prep deaths are always gate deaths (they
|
|
419
|
+
* fail before the agent is spawned) regardless of this bound. */
|
|
420
|
+
export declare const GATE_DEATH_MAX_DURATION_MS = 60000;
|
|
421
|
+
/**
|
|
422
|
+
* Secondary ceiling for CONSECUTIVE gate deaths. Gate deaths refund their
|
|
423
|
+
* redispatch attempt (the worker never ran), which is correct for transient
|
|
424
|
+
* infrastructure faults — but a deterministic fault (e.g. a permanently broken
|
|
425
|
+
* repo path) would otherwise retry forever at the backoff floor. After this
|
|
426
|
+
* many consecutive gate deaths the work item is dead-lettered (visible in
|
|
427
|
+
* drain reports) instead of spinning; any run that reaches the worker resets
|
|
428
|
+
* the streak, and an operator requeue (attempts reset) re-arms it. At the
|
|
429
|
+
* ~2–4 minute refunded-attempts backoff this bounds a deterministic fault to
|
|
430
|
+
* roughly an hour of auto-retry before it demands an operator.
|
|
431
|
+
*/
|
|
432
|
+
export declare const GATE_DEATH_CEILING = 20;
|
|
433
|
+
export type NonProductiveFailureKind = "tempfail" | "gate-death";
|
|
434
|
+
type ClassifiableStepRun = Pick<WorkflowStepRun, "stepId" | "status" | "exitCode" | "durationMs" | "error">;
|
|
435
|
+
/**
|
|
436
|
+
* Classify a just-finalized *failed* workflow run's decisive failing step. A
|
|
437
|
+
* non-productive finish (a tempfail retry-signal, or a gate death before the
|
|
438
|
+
* worker ran) must not count toward the todos-task redispatch cap. Returns the
|
|
439
|
+
* non-productive kind, or `undefined` when the run represents a real worker
|
|
440
|
+
* attempt that legitimately counts toward the cap. Pure/exported for tests.
|
|
441
|
+
*/
|
|
442
|
+
export declare function classifyNonProductiveStepFailure(steps: ClassifiableStepRun[]): NonProductiveFailureKind | undefined;
|
|
371
443
|
export declare function scrubbedOrNull(value: string | undefined | null): string | null;
|
|
372
444
|
/** Scrub secrets then bound size before persisting run stdout/stderr. */
|
|
373
445
|
export declare function persistedRunOutput(value: string | undefined | null): string | null;
|
|
446
|
+
/** Scrub structured string leaves before stringify, then scrub the encoded JSON. */
|
|
447
|
+
export declare function persistedJson(value: unknown): string;
|
|
448
|
+
export declare function persistedWorkflowEventPayload(payload: Record<string, unknown> | undefined | null): string | null;
|
|
374
449
|
export declare class Store {
|
|
375
450
|
private db;
|
|
376
451
|
private rootDir;
|
|
@@ -398,9 +473,11 @@ export declare class Store {
|
|
|
398
473
|
getLoop(id: string): Loop | undefined;
|
|
399
474
|
findLoopByName(name: string): Loop | undefined;
|
|
400
475
|
requireUniqueLoop(idOrName: string): Loop;
|
|
476
|
+
private requireArchiveMutationLoop;
|
|
401
477
|
requireLoop(idOrName: string): Loop;
|
|
402
478
|
listLoops(opts?: {
|
|
403
479
|
status?: LoopStatus;
|
|
480
|
+
labels?: string[];
|
|
404
481
|
limit?: number;
|
|
405
482
|
offset?: number;
|
|
406
483
|
archived?: boolean;
|
|
@@ -409,7 +486,12 @@ export declare class Store {
|
|
|
409
486
|
}): Loop[];
|
|
410
487
|
private withLatestRunSummaries;
|
|
411
488
|
dueLoops(now: Date, limit?: number): Loop[];
|
|
412
|
-
updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt">>, opts?: DaemonLeaseFence): Loop;
|
|
489
|
+
updateLoop(id: string, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor" | "expiresAt" | "labels">>, opts?: DaemonLeaseFence): Loop;
|
|
490
|
+
advanceLoopIfCurrent(id: string, expected: LoopSchedulingState, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor">>, opts?: DaemonLeaseFence): Loop | undefined;
|
|
491
|
+
tripCircuitBreakerIfCurrent(id: string, expected: LoopSchedulingState, patch: Partial<Pick<Loop, "status" | "nextRunAt" | "retryScheduledFor">>, marker: {
|
|
492
|
+
scheduledFor: string;
|
|
493
|
+
reason: string;
|
|
494
|
+
}, opts?: DaemonLeaseFence): CircuitBreakerTransitionResult | undefined;
|
|
413
495
|
private activeLoopReferenceCount;
|
|
414
496
|
private archiveWorkflowIfUnreferenced;
|
|
415
497
|
retargetWorkflowLoop(idOrName: string, workflowId: string, opts?: DaemonLeaseFence & {
|
|
@@ -490,9 +572,41 @@ export declare class Store {
|
|
|
490
572
|
* step on that account right now — exactly the concurrency to bound.
|
|
491
573
|
*/
|
|
492
574
|
countRunningWorkflowStepsByAuthProfile(): Record<string, number>;
|
|
575
|
+
/**
|
|
576
|
+
* Requeue a terminal admission work item for the next task/event delivery.
|
|
577
|
+
* By default `attempts` is preserved (used by the bounded stale-terminal
|
|
578
|
+
* re-admission on the route path, which must keep counting toward the cap).
|
|
579
|
+
* Pass `resetAttempts: true` for the operator unwedge (`loops routes requeue`)
|
|
580
|
+
* so a manual requeue is DURABLE rather than one-shot: without the reset a
|
|
581
|
+
* capped item that finishes terminal once more re-caps instantly.
|
|
582
|
+
*/
|
|
493
583
|
requeueWorkflowWorkItem(id: string, patch?: {
|
|
494
584
|
reason?: string;
|
|
585
|
+
resetAttempts?: boolean;
|
|
495
586
|
}): WorkflowWorkItem;
|
|
587
|
+
/**
|
|
588
|
+
* Transition a terminal admission work item to `dead_letter`. Used by the
|
|
589
|
+
* route path when a still-actionable todos task has exhausted the redispatch
|
|
590
|
+
* cap: instead of silently deduping the same terminal row forever (the "black
|
|
591
|
+
* hole" — `considered=N created=0` with no signal), the item is moved to a
|
|
592
|
+
* visible `dead_letter` state so drain reports can surface + count it and an
|
|
593
|
+
* operator can `loops routes requeue` it. Idempotent: a no-op on an item that
|
|
594
|
+
* is already dead-lettered.
|
|
595
|
+
*/
|
|
596
|
+
deadLetterWorkflowWorkItem(id: string, patch?: {
|
|
597
|
+
reason?: string;
|
|
598
|
+
}): WorkflowWorkItem;
|
|
599
|
+
/**
|
|
600
|
+
* Refund a redispatch attempt for a *failed* run that never did real work.
|
|
601
|
+
* Called from {@link finalizeWorkflowRun} the moment a work item is set
|
|
602
|
+
* `failed`, so the todos-task redispatch cap only ever counts real worker
|
|
603
|
+
* attempts. A tempfail (`exit 75`) is additionally made requeueable (dropped
|
|
604
|
+
* back to `queued`, bindings cleared) so its "retry later" contract fires on
|
|
605
|
+
* the next drain instead of persisting as a terminal, dedupe-forever row. A
|
|
606
|
+
* gate death stays `failed` (the bounded re-admission picks it up after
|
|
607
|
+
* backoff once the underlying infra fault clears). Both floor attempts at 0.
|
|
608
|
+
*/
|
|
609
|
+
private demoteNonProductiveWorkItems;
|
|
496
610
|
admitWorkflowWorkItem(id: string, patch: {
|
|
497
611
|
workflowId: string;
|
|
498
612
|
loopId: string;
|
|
@@ -553,7 +667,7 @@ export declare class Store {
|
|
|
553
667
|
stderr?: string;
|
|
554
668
|
payload?: Record<string, unknown>;
|
|
555
669
|
}, opts?: DaemonLeaseFence): WorkflowStepRun;
|
|
556
|
-
recoverWorkflowRun(workflowRunId: string, reason?: string): {
|
|
670
|
+
recoverWorkflowRun(workflowRunId: string, reason?: string, _context?: WorkflowRecoveryContext): {
|
|
557
671
|
run: WorkflowRun;
|
|
558
672
|
recoveredSteps: WorkflowStepRun[];
|
|
559
673
|
};
|
|
@@ -561,8 +675,8 @@ export declare class Store {
|
|
|
561
675
|
skipWorkflowStepRun(workflowRunId: string, stepId: string, reason: string, opts?: DaemonLeaseFence): WorkflowStepRun;
|
|
562
676
|
finalizeWorkflowRun(workflowRunId: string, status: WorkflowRunStatus, patch?: Partial<Pick<WorkflowRun, "finishedAt" | "durationMs" | "error">>, opts?: DaemonLeaseFence): WorkflowRun;
|
|
563
677
|
cancelWorkflowRun(workflowRunId: string, reason?: string): WorkflowRun;
|
|
564
|
-
appendWorkflowEvent(workflowRunId: string, eventType: string, stepId?: string, payload?: Record<string, unknown>):
|
|
565
|
-
listWorkflowEvents(workflowRunId: string, limit?: number):
|
|
678
|
+
appendWorkflowEvent(workflowRunId: string, eventType: string, stepId?: string, payload?: Record<string, unknown>): StoredWorkflowEvent;
|
|
679
|
+
listWorkflowEvents(workflowRunId: string, limit?: number): StoredWorkflowEvent[];
|
|
566
680
|
hasRunningRun(loopId: string): boolean;
|
|
567
681
|
hasRunningRunForSlot(loopId: string, scheduledFor: string): boolean;
|
|
568
682
|
private hasBlockingRunningRunForOtherSlot;
|
|
@@ -588,9 +702,15 @@ export declare class Store {
|
|
|
588
702
|
listRuns(opts?: {
|
|
589
703
|
loopId?: string;
|
|
590
704
|
status?: RunStatus;
|
|
705
|
+
labels?: string[];
|
|
591
706
|
limit?: number;
|
|
592
707
|
offset?: number;
|
|
593
708
|
}): LoopRun[];
|
|
709
|
+
listRecoveredLeaseRunsPage(opts?: {
|
|
710
|
+
snapshot?: RecoveredLeaseRunSnapshotEntry[];
|
|
711
|
+
offset?: number;
|
|
712
|
+
limit?: number;
|
|
713
|
+
}): RecoveredLeaseRunPage;
|
|
594
714
|
writeRunReceipt(input: WriteRunReceiptInput, opts?: {
|
|
595
715
|
now?: Date;
|
|
596
716
|
}): RunReceipt;
|
|
@@ -607,6 +727,7 @@ export declare class Store {
|
|
|
607
727
|
recoverExpiredRunLeases(now?: Date, opts?: DaemonLeaseFence & {
|
|
608
728
|
limit?: number;
|
|
609
729
|
scanLimit?: number;
|
|
730
|
+
runId?: string;
|
|
610
731
|
}): LoopRun[];
|
|
611
732
|
/**
|
|
612
733
|
* Recover expired run leases and report both outcomes: runs abandoned (no
|
|
@@ -617,6 +738,7 @@ export declare class Store {
|
|
|
617
738
|
recoverExpiredRunLeasesDetailed(now?: Date, opts?: DaemonLeaseFence & {
|
|
618
739
|
limit?: number;
|
|
619
740
|
scanLimit?: number;
|
|
741
|
+
runId?: string;
|
|
620
742
|
}): RecoverExpiredRunLeasesResult;
|
|
621
743
|
expireLoops(now?: Date, opts?: DaemonLeaseFence): Loop[];
|
|
622
744
|
countLoops(status?: LoopStatus, opts?: {
|