@hasna/loops 0.4.27 → 0.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +121 -0
- package/README.md +6 -0
- package/dist/api/index.js +689 -20
- package/dist/cli/index.js +509 -85
- package/dist/daemon/index.js +118 -11
- package/dist/index.js +525 -64
- package/dist/lib/executor.d.ts +9 -0
- package/dist/lib/migration.d.ts +66 -0
- package/dist/lib/mode.js +3 -3
- package/dist/lib/route/fields.d.ts +8 -0
- package/dist/lib/storage/index.js +197 -7
- package/dist/lib/storage/postgres-loop-storage.d.ts +2 -2
- package/dist/lib/storage/postgres-schema.js +3 -0
- package/dist/lib/storage/postgres.js +3 -0
- package/dist/lib/storage/sqlite.js +83 -3
- package/dist/lib/store/index.d.ts +3 -0
- package/dist/lib/store.d.ts +77 -0
- package/dist/lib/store.js +89 -4
- package/dist/mcp/index.js +118 -11
- package/dist/runner/index.js +35 -8
- package/dist/sdk/http.d.ts +153 -1
- package/dist/sdk/http.js +78 -1
- package/dist/sdk/index.js +411 -60
- package/dist/serve/index.js +919 -60
- package/dist/types.d.ts +11 -0
- package/docs/DEPLOYMENT_MODES.md +6 -0
- package/docs/USAGE.md +11 -8
- package/package.json +3 -3
package/dist/lib/executor.d.ts
CHANGED
|
@@ -56,6 +56,15 @@ interface MachineCommandPlan {
|
|
|
56
56
|
}
|
|
57
57
|
/** Exported for tests: resolves the default idle watchdog for agent targets. */
|
|
58
58
|
export declare function defaultAgentIdleTimeoutMs(target: AgentTarget, opts: ExecuteOptions): number | undefined;
|
|
59
|
+
/**
|
|
60
|
+
* Detects git's "missing but already registered worktree" error: the target
|
|
61
|
+
* path is recorded in `.git/worktrees/` but its directory was deleted out from
|
|
62
|
+
* under it, so `git worktree add` refuses. git's own prescribed remedy is
|
|
63
|
+
* `git worktree prune` (or `add -f`). Left unhandled this terminal-fails worktree
|
|
64
|
+
* prep on every attempt and the drain dedupes the failed item into a silent
|
|
65
|
+
* wedge — historically the single biggest burner of the redispatch cap.
|
|
66
|
+
*/
|
|
67
|
+
export declare function isStaleWorktreeRegistration(detail: string | undefined): boolean;
|
|
59
68
|
export declare function preflightTarget(target: ExecutableTarget, metadata?: ExecutionMetadata, opts?: ExecuteOptions): PreflightResult;
|
|
60
69
|
export declare function executeTarget(target: ExecutableTarget, metadata?: ExecutionMetadata, opts?: ExecuteOptions): Promise<ExecutorResult>;
|
|
61
70
|
export declare function executeLoop(loop: Loop, run: LoopRun, opts?: ExecuteOptions): Promise<ExecutorResult>;
|
package/dist/lib/migration.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Loop, LoopRun, WorkflowSpec } from "../types.js";
|
|
2
2
|
import type { Store, StoreMigrationChecks } from "./store.js";
|
|
3
3
|
export declare const LOOPS_MIGRATION_SCHEMA = "open-loops.migration/v1";
|
|
4
|
+
export declare const LOOPS_SELF_HOSTED_PUSH_MANIFEST_SCHEMA = "open-loops.self-hosted-push-manifest/v1";
|
|
4
5
|
export type LoopsMigrationResource = "workflow" | "loop" | "run" | "remote";
|
|
5
6
|
export type LoopsMigrationAction = "insert" | "update" | "skip" | "conflict" | "blocked";
|
|
6
7
|
export interface LoopsMigrationBundle {
|
|
@@ -67,6 +68,7 @@ export interface LoopsMigrationPlan {
|
|
|
67
68
|
summary: LoopsMigrationPlanSummary;
|
|
68
69
|
rows: LoopsMigrationPlanRow[];
|
|
69
70
|
warnings: string[];
|
|
71
|
+
manifest?: SelfHostedPushManifest;
|
|
70
72
|
}
|
|
71
73
|
export interface ApplyLoopsMigrationResult {
|
|
72
74
|
plan: LoopsMigrationPlan;
|
|
@@ -83,6 +85,7 @@ export interface SelfHostedPlanOptions {
|
|
|
83
85
|
fetchImpl?: typeof fetch;
|
|
84
86
|
env?: NodeJS.ProcessEnv;
|
|
85
87
|
includeRuns?: boolean;
|
|
88
|
+
replace?: boolean;
|
|
86
89
|
}
|
|
87
90
|
export interface RunnerRegistrationOptions {
|
|
88
91
|
apiUrl?: string;
|
|
@@ -134,6 +137,69 @@ export interface SelfHostedPushResult {
|
|
|
134
137
|
orphanRuns: number;
|
|
135
138
|
};
|
|
136
139
|
requests: number;
|
|
140
|
+
manifest: SelfHostedPushManifest;
|
|
141
|
+
}
|
|
142
|
+
export interface SelfHostedPushManifest {
|
|
143
|
+
schema: typeof LOOPS_SELF_HOSTED_PUSH_MANIFEST_SCHEMA;
|
|
144
|
+
generatedAt: string;
|
|
145
|
+
apiUrl?: string;
|
|
146
|
+
dryRun: boolean;
|
|
147
|
+
replace: boolean;
|
|
148
|
+
includeRuns: boolean;
|
|
149
|
+
safety: {
|
|
150
|
+
forcedLoopStatus: "paused";
|
|
151
|
+
clearedLoopRunPointers: true;
|
|
152
|
+
forcedWorkflowStatus: "archived";
|
|
153
|
+
resumesLoops: false;
|
|
154
|
+
};
|
|
155
|
+
counts: {
|
|
156
|
+
source: {
|
|
157
|
+
workflows: number;
|
|
158
|
+
loops: number;
|
|
159
|
+
runs: number;
|
|
160
|
+
};
|
|
161
|
+
remoteBefore: {
|
|
162
|
+
workflows?: number;
|
|
163
|
+
loops?: number;
|
|
164
|
+
runs?: number;
|
|
165
|
+
};
|
|
166
|
+
remoteAfter?: {
|
|
167
|
+
workflows?: number;
|
|
168
|
+
loops?: number;
|
|
169
|
+
runs?: number;
|
|
170
|
+
};
|
|
171
|
+
plan: LoopsMigrationPlanSummary;
|
|
172
|
+
applied?: {
|
|
173
|
+
workflows: number;
|
|
174
|
+
loops: number;
|
|
175
|
+
runs: number;
|
|
176
|
+
};
|
|
177
|
+
skipped?: {
|
|
178
|
+
runningRuns: number;
|
|
179
|
+
orphanRuns: number;
|
|
180
|
+
};
|
|
181
|
+
requests?: number;
|
|
182
|
+
};
|
|
183
|
+
missingIds: {
|
|
184
|
+
workflows: string[];
|
|
185
|
+
loops: string[];
|
|
186
|
+
runs: string[];
|
|
187
|
+
};
|
|
188
|
+
existingIds: {
|
|
189
|
+
workflows: string[];
|
|
190
|
+
loops: string[];
|
|
191
|
+
runs: string[];
|
|
192
|
+
};
|
|
193
|
+
unsafe: {
|
|
194
|
+
blocked: LoopsMigrationPlanRow[];
|
|
195
|
+
conflicts: LoopsMigrationPlanRow[];
|
|
196
|
+
unsupported: string[];
|
|
197
|
+
warnings: string[];
|
|
198
|
+
};
|
|
199
|
+
rollback: {
|
|
200
|
+
notes: string[];
|
|
201
|
+
commands: string[];
|
|
202
|
+
};
|
|
137
203
|
}
|
|
138
204
|
/**
|
|
139
205
|
* Apply a local->self-hosted backfill through the control plane's id-preserving
|
package/dist/lib/mode.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// package.json
|
|
3
3
|
var package_default = {
|
|
4
4
|
name: "@hasna/loops",
|
|
5
|
-
version: "0.4.
|
|
5
|
+
version: "0.4.28",
|
|
6
6
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
7
7
|
type: "module",
|
|
8
8
|
main: "dist/index.js",
|
|
@@ -112,6 +112,7 @@ var package_default = {
|
|
|
112
112
|
bun: ">=1.0.0"
|
|
113
113
|
},
|
|
114
114
|
dependencies: {
|
|
115
|
+
"@hasna/contracts": "^0.5.1",
|
|
115
116
|
"@hasna/events": "^0.1.9",
|
|
116
117
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
117
118
|
"@openrouter/ai-sdk-provider": "2.9.1",
|
|
@@ -121,8 +122,7 @@ var package_default = {
|
|
|
121
122
|
zod: "4.4.3"
|
|
122
123
|
},
|
|
123
124
|
optionalDependencies: {
|
|
124
|
-
"@hasna/machines": "0.0.49"
|
|
125
|
-
"@hasna/contracts": "^0.4.2"
|
|
125
|
+
"@hasna/machines": "0.0.49"
|
|
126
126
|
},
|
|
127
127
|
devDependencies: {
|
|
128
128
|
"@types/bun": "latest",
|
|
@@ -15,6 +15,14 @@ export declare function firstTruthyField(records: Record<string, unknown>[], key
|
|
|
15
15
|
export declare function automationRecords(data: Record<string, unknown>, metadata: Record<string, unknown>): Record<string, unknown>[];
|
|
16
16
|
export declare function tagsFromValue(value: unknown): string[];
|
|
17
17
|
export declare function taskEventTags(records: Record<string, unknown>[]): string[];
|
|
18
|
+
/**
|
|
19
|
+
* First route-disallowed tag on a task, if any. Exposed so drains can exclude
|
|
20
|
+
* tasks that can never route (no-auto/blocked/manual/…) from the bounded
|
|
21
|
+
* candidate window *before* slicing, instead of letting them occupy scan slots
|
|
22
|
+
* every tick only to be rejected by eligibility — the "skip re-burns the window
|
|
23
|
+
* forever" half of the merge-lane decay.
|
|
24
|
+
*/
|
|
25
|
+
export declare function taskRouteDisallowedTag(tags: string[]): string | undefined;
|
|
18
26
|
export declare function taskRouteEligibility(data: Record<string, unknown>, metadata: Record<string, unknown>): {
|
|
19
27
|
eligible: boolean;
|
|
20
28
|
reason?: string;
|
|
@@ -1569,6 +1569,7 @@ var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
|
1569
1569
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
1570
1570
|
var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
|
|
1571
1571
|
var SCHEMA_USER_VERSION = 8;
|
|
1572
|
+
var BREAKING_SCHEMA_FLOOR = 7;
|
|
1572
1573
|
var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
|
|
1573
1574
|
var PRUNE_BATCH_SIZE = 400;
|
|
1574
1575
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
@@ -1711,6 +1712,7 @@ function rowToWorkflowWorkItem(row) {
|
|
|
1711
1712
|
priority: row.priority,
|
|
1712
1713
|
status: row.status,
|
|
1713
1714
|
attempts: row.attempts,
|
|
1715
|
+
gateDeaths: row.gate_deaths ?? 0,
|
|
1714
1716
|
nextAttemptAt: row.next_attempt_at ?? undefined,
|
|
1715
1717
|
leaseExpiresAt: row.lease_expires_at ?? undefined,
|
|
1716
1718
|
workflowId: row.workflow_id ?? undefined,
|
|
@@ -1860,6 +1862,23 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
|
|
|
1860
1862
|
}
|
|
1861
1863
|
return;
|
|
1862
1864
|
}
|
|
1865
|
+
var WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
|
|
1866
|
+
var GATE_STEP_IDS = new Set(["triage", "planner", "plan"]);
|
|
1867
|
+
var GATE_DEATH_MAX_DURATION_MS = 60000;
|
|
1868
|
+
var GATE_DEATH_CEILING = 20;
|
|
1869
|
+
function classifyNonProductiveStepFailure(steps) {
|
|
1870
|
+
const failing = [...steps].reverse().find((step) => step.status === "failed" || step.status === "timed_out");
|
|
1871
|
+
if (!failing)
|
|
1872
|
+
return;
|
|
1873
|
+
if (failing.exitCode === WORK_ITEM_TEMPFAIL_EXIT_CODE)
|
|
1874
|
+
return "tempfail";
|
|
1875
|
+
if (typeof failing.error === "string" && failing.error.includes("worktree preparation failed"))
|
|
1876
|
+
return "gate-death";
|
|
1877
|
+
const fast = failing.durationMs === undefined || failing.durationMs < GATE_DEATH_MAX_DURATION_MS;
|
|
1878
|
+
if (GATE_STEP_IDS.has(failing.stepId) && fast)
|
|
1879
|
+
return "gate-death";
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1863
1882
|
function scrubbedOrNull(value) {
|
|
1864
1883
|
return value == null ? null : scrubSecrets(value);
|
|
1865
1884
|
}
|
|
@@ -1960,11 +1979,21 @@ class Store {
|
|
|
1960
1979
|
id TEXT PRIMARY KEY,
|
|
1961
1980
|
applied_at TEXT NOT NULL
|
|
1962
1981
|
);
|
|
1982
|
+
CREATE TABLE IF NOT EXISTS schema_compat (
|
|
1983
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
1984
|
+
min_compatible_user_version INTEGER NOT NULL
|
|
1985
|
+
);
|
|
1963
1986
|
`);
|
|
1964
1987
|
const versionRow = this.db.query("PRAGMA user_version").get();
|
|
1965
1988
|
const userVersion = versionRow?.user_version ?? 0;
|
|
1966
1989
|
if (userVersion > SCHEMA_USER_VERSION) {
|
|
1967
|
-
|
|
1990
|
+
const floorRow = this.db.query("SELECT min_compatible_user_version FROM schema_compat WHERE id = 1").get();
|
|
1991
|
+
if (!floorRow) {
|
|
1992
|
+
throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}) and carries no compatibility floor; upgrade open-loops before opening this database`);
|
|
1993
|
+
}
|
|
1994
|
+
if (SCHEMA_USER_VERSION < floorRow.min_compatible_user_version) {
|
|
1995
|
+
throw new Error(`loops database schema version ${userVersion} requires a binary with schema support >= ${floorRow.min_compatible_user_version} (this binary supports ${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
|
|
1996
|
+
}
|
|
1968
1997
|
}
|
|
1969
1998
|
const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
|
|
1970
1999
|
for (const migration of this.migrations()) {
|
|
@@ -1975,8 +2004,10 @@ class Store {
|
|
|
1975
2004
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
|
|
1976
2005
|
}
|
|
1977
2006
|
}
|
|
1978
|
-
if (userVersion
|
|
2007
|
+
if (userVersion < SCHEMA_USER_VERSION)
|
|
1979
2008
|
this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
|
|
2009
|
+
this.db.query(`INSERT INTO schema_compat (id, min_compatible_user_version) VALUES (1, ?)
|
|
2010
|
+
ON CONFLICT(id) DO UPDATE SET min_compatible_user_version = MAX(min_compatible_user_version, excluded.min_compatible_user_version)`).run(BREAKING_SCHEMA_FLOOR);
|
|
1980
2011
|
}
|
|
1981
2012
|
migrations() {
|
|
1982
2013
|
return [
|
|
@@ -2043,6 +2074,12 @@ class Store {
|
|
|
2043
2074
|
this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
|
|
2044
2075
|
this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
|
|
2045
2076
|
}
|
|
2077
|
+
},
|
|
2078
|
+
{
|
|
2079
|
+
id: "0011_work_item_gate_deaths",
|
|
2080
|
+
apply: () => {
|
|
2081
|
+
this.addColumnIfMissing("workflow_work_items", "gate_deaths", "INTEGER NOT NULL DEFAULT 0");
|
|
2082
|
+
}
|
|
2046
2083
|
}
|
|
2047
2084
|
];
|
|
2048
2085
|
}
|
|
@@ -3173,7 +3210,7 @@ class Store {
|
|
|
3173
3210
|
}
|
|
3174
3211
|
upsertWorkflowWorkItem(input) {
|
|
3175
3212
|
const now = nowIso();
|
|
3176
|
-
const id = genId();
|
|
3213
|
+
const id = input.id ?? genId();
|
|
3177
3214
|
const status = input.status ?? "queued";
|
|
3178
3215
|
this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
|
|
3179
3216
|
subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
|
|
@@ -3301,6 +3338,7 @@ class Store {
|
|
|
3301
3338
|
const placeholders = requeueableStatuses.map(() => "?").join(",");
|
|
3302
3339
|
const res = this.db.query(`UPDATE workflow_work_items
|
|
3303
3340
|
SET status='queued', workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
|
|
3341
|
+
${patch.resetAttempts ? "attempts=0, gate_deaths=0," : ""}
|
|
3304
3342
|
next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
|
|
3305
3343
|
WHERE id=? AND status IN (${placeholders})`).run(reason, now, id, ...requeueableStatuses);
|
|
3306
3344
|
const item = this.getWorkflowWorkItem(id);
|
|
@@ -3310,6 +3348,46 @@ class Store {
|
|
|
3310
3348
|
throw new Error(`workflow work item was not requeued: ${id} status=${item.status}`);
|
|
3311
3349
|
return item;
|
|
3312
3350
|
}
|
|
3351
|
+
deadLetterWorkflowWorkItem(id, patch = {}) {
|
|
3352
|
+
const now = nowIso();
|
|
3353
|
+
const reason = patch.reason?.trim() || "redispatch cap reached; dead-lettered";
|
|
3354
|
+
this.db.query(`UPDATE workflow_work_items
|
|
3355
|
+
SET status='dead_letter', next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
|
|
3356
|
+
WHERE id=? AND status IN ('succeeded','failed','cancelled')`).run(reason, now, id);
|
|
3357
|
+
const item = this.getWorkflowWorkItem(id);
|
|
3358
|
+
if (!item)
|
|
3359
|
+
throw new Error(`workflow work item not found after dead-letter: ${id}`);
|
|
3360
|
+
return item;
|
|
3361
|
+
}
|
|
3362
|
+
demoteNonProductiveWorkItems(workflowRunId, finishedAt) {
|
|
3363
|
+
const kind = classifyNonProductiveStepFailure(this.listWorkflowStepRuns(workflowRunId));
|
|
3364
|
+
if (!kind) {
|
|
3365
|
+
this.db.query("UPDATE workflow_work_items SET gate_deaths=0, updated_at=? WHERE workflow_run_id=? AND status='failed' AND gate_deaths > 0").run(finishedAt, workflowRunId);
|
|
3366
|
+
return;
|
|
3367
|
+
}
|
|
3368
|
+
if (kind === "tempfail") {
|
|
3369
|
+
this.db.query(`UPDATE workflow_work_items
|
|
3370
|
+
SET status='queued', attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
|
|
3371
|
+
gate_deaths=0,
|
|
3372
|
+
workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
|
|
3373
|
+
next_attempt_at=NULL, lease_expires_at=NULL,
|
|
3374
|
+
last_reason='worker exited 75 (tempfail): requeued for retry; attempt refunded (does not count toward redispatch cap)',
|
|
3375
|
+
updated_at=?
|
|
3376
|
+
WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
|
|
3377
|
+
return;
|
|
3378
|
+
}
|
|
3379
|
+
this.db.query(`UPDATE workflow_work_items
|
|
3380
|
+
SET attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
|
|
3381
|
+
gate_deaths=gate_deaths + 1,
|
|
3382
|
+
status=CASE WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING} THEN 'dead_letter' ELSE status END,
|
|
3383
|
+
last_reason=CASE
|
|
3384
|
+
WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING}
|
|
3385
|
+
THEN 'gate-death ceiling reached (' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING} consecutive runs died at worktree prep / triage / planner without reaching the worker): dead-lettered \u2014 the infrastructure fault needs an operator; ''loops routes requeue'' resets and retries'
|
|
3386
|
+
ELSE 'gate death before real work (worktree prep / triage / planner): attempt refunded (does not count toward redispatch cap); consecutive gate deaths: ' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING}'
|
|
3387
|
+
END,
|
|
3388
|
+
updated_at=?
|
|
3389
|
+
WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
|
|
3390
|
+
}
|
|
3313
3391
|
admitWorkflowWorkItem(id, patch) {
|
|
3314
3392
|
const now = nowIso();
|
|
3315
3393
|
const res = this.db.query(`UPDATE workflow_work_items
|
|
@@ -4021,6 +4099,8 @@ class Store {
|
|
|
4021
4099
|
if (changed) {
|
|
4022
4100
|
const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
|
|
4023
4101
|
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
|
|
4102
|
+
if (itemStatus === "failed")
|
|
4103
|
+
this.demoteNonProductiveWorkItems(workflowRunId, finishedAt);
|
|
4024
4104
|
this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
|
|
4025
4105
|
}
|
|
4026
4106
|
this.db.exec("COMMIT");
|
|
@@ -5321,6 +5401,9 @@ CREATE INDEX IF NOT EXISTS idx_run_receipts_knowledge_ids ON run_receipts USING
|
|
|
5321
5401
|
migration("0006_work_item_machine_id", `
|
|
5322
5402
|
ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS machine_id TEXT;
|
|
5323
5403
|
CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status);
|
|
5404
|
+
`),
|
|
5405
|
+
migration("0007_work_item_gate_deaths", `
|
|
5406
|
+
ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS gate_deaths INTEGER NOT NULL DEFAULT 0;
|
|
5324
5407
|
`)
|
|
5325
5408
|
]);
|
|
5326
5409
|
|
|
@@ -6671,11 +6754,118 @@ class PostgresLoopStorage {
|
|
|
6671
6754
|
throw new Error(`workflow not found after archive: ${existing.id}`);
|
|
6672
6755
|
return archived;
|
|
6673
6756
|
}
|
|
6674
|
-
createWorkflowInvocation() {
|
|
6675
|
-
|
|
6757
|
+
async createWorkflowInvocation(...args) {
|
|
6758
|
+
const [input] = args;
|
|
6759
|
+
const now = nowIso();
|
|
6760
|
+
const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
|
|
6761
|
+
if (sourceDedupeKey) {
|
|
6762
|
+
const existing = await this.client.get("SELECT * FROM workflow_invocations WHERE source_kind = $1 AND source_dedupe_key = $2 LIMIT 1", [input.sourceRef.kind, sourceDedupeKey]);
|
|
6763
|
+
if (existing)
|
|
6764
|
+
return rowToWorkflowInvocation(existing);
|
|
6765
|
+
}
|
|
6766
|
+
const id = input.id ?? genId();
|
|
6767
|
+
await this.client.execute(`INSERT INTO workflow_invocations (id, workflow_id, template_id, source_kind, source_id, source_dedupe_key,
|
|
6768
|
+
source_json, subject_kind, subject_id, subject_path, subject_url, subject_json, intent, scope_json,
|
|
6769
|
+
output_policy_json, created_at, updated_at)
|
|
6770
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8,$9,$10,$11,$12::jsonb,$13,$14::jsonb,$15::jsonb,$16,$17)`, [
|
|
6771
|
+
id,
|
|
6772
|
+
input.workflowId ?? null,
|
|
6773
|
+
input.templateId ?? null,
|
|
6774
|
+
input.sourceRef.kind,
|
|
6775
|
+
input.sourceRef.id ?? null,
|
|
6776
|
+
sourceDedupeKey ?? null,
|
|
6777
|
+
JSON.stringify(input.sourceRef),
|
|
6778
|
+
input.subjectRef.kind,
|
|
6779
|
+
input.subjectRef.id ?? null,
|
|
6780
|
+
input.subjectRef.path ?? null,
|
|
6781
|
+
input.subjectRef.url ?? null,
|
|
6782
|
+
JSON.stringify(input.subjectRef),
|
|
6783
|
+
input.intent,
|
|
6784
|
+
input.scope ? JSON.stringify(input.scope) : null,
|
|
6785
|
+
input.outputPolicy ? JSON.stringify(input.outputPolicy) : null,
|
|
6786
|
+
now,
|
|
6787
|
+
now
|
|
6788
|
+
]);
|
|
6789
|
+
const row = await this.client.get("SELECT * FROM workflow_invocations WHERE id = $1", [id]);
|
|
6790
|
+
if (!row)
|
|
6791
|
+
throw new Error(`workflow invocation not found after create: ${id}`);
|
|
6792
|
+
return rowToWorkflowInvocation(row);
|
|
6676
6793
|
}
|
|
6677
|
-
upsertWorkflowWorkItem() {
|
|
6678
|
-
|
|
6794
|
+
async upsertWorkflowWorkItem(...args) {
|
|
6795
|
+
const [input] = args;
|
|
6796
|
+
const now = nowIso();
|
|
6797
|
+
const id = input.id ?? genId();
|
|
6798
|
+
const status = input.status ?? "queued";
|
|
6799
|
+
await this.client.execute(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
|
|
6800
|
+
subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at,
|
|
6801
|
+
lease_expires_at, workflow_id, loop_id, workflow_run_id, last_reason, created_at, updated_at)
|
|
6802
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,0,$14,NULL,NULL,NULL,NULL,$15,$16,$17)
|
|
6803
|
+
ON CONFLICT(route_key, idempotency_key) DO UPDATE SET
|
|
6804
|
+
invocation_id=excluded.invocation_id,
|
|
6805
|
+
source_type=excluded.source_type,
|
|
6806
|
+
source_ref=excluded.source_ref,
|
|
6807
|
+
subject_ref=excluded.subject_ref,
|
|
6808
|
+
project_key=excluded.project_key,
|
|
6809
|
+
project_group=excluded.project_group,
|
|
6810
|
+
machine_id=CASE
|
|
6811
|
+
WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.machine_id
|
|
6812
|
+
ELSE excluded.machine_id
|
|
6813
|
+
END,
|
|
6814
|
+
route_scope=excluded.route_scope,
|
|
6815
|
+
priority=excluded.priority,
|
|
6816
|
+
status=CASE
|
|
6817
|
+
WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled')
|
|
6818
|
+
THEN workflow_work_items.status
|
|
6819
|
+
ELSE excluded.status
|
|
6820
|
+
END,
|
|
6821
|
+
workflow_id=CASE
|
|
6822
|
+
WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.workflow_id
|
|
6823
|
+
ELSE NULL
|
|
6824
|
+
END,
|
|
6825
|
+
loop_id=CASE
|
|
6826
|
+
WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.loop_id
|
|
6827
|
+
ELSE NULL
|
|
6828
|
+
END,
|
|
6829
|
+
workflow_run_id=CASE
|
|
6830
|
+
WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.workflow_run_id
|
|
6831
|
+
ELSE NULL
|
|
6832
|
+
END,
|
|
6833
|
+
lease_expires_at=CASE
|
|
6834
|
+
WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.lease_expires_at
|
|
6835
|
+
ELSE NULL
|
|
6836
|
+
END,
|
|
6837
|
+
next_attempt_at=excluded.next_attempt_at,
|
|
6838
|
+
last_reason=CASE
|
|
6839
|
+
WHEN workflow_work_items.attempts > 0
|
|
6840
|
+
AND workflow_work_items.status IN ('queued', 'deferred')
|
|
6841
|
+
AND workflow_work_items.last_reason IS NOT NULL
|
|
6842
|
+
AND excluded.last_reason IS NOT NULL
|
|
6843
|
+
THEN workflow_work_items.last_reason || '; ' || excluded.last_reason
|
|
6844
|
+
ELSE COALESCE(excluded.last_reason, workflow_work_items.last_reason)
|
|
6845
|
+
END,
|
|
6846
|
+
updated_at=excluded.updated_at`, [
|
|
6847
|
+
id,
|
|
6848
|
+
input.routeKey,
|
|
6849
|
+
input.idempotencyKey,
|
|
6850
|
+
input.invocationId,
|
|
6851
|
+
input.sourceType,
|
|
6852
|
+
input.sourceRef,
|
|
6853
|
+
input.subjectRef,
|
|
6854
|
+
input.projectKey ?? null,
|
|
6855
|
+
input.projectGroup ?? null,
|
|
6856
|
+
input.machineId ?? null,
|
|
6857
|
+
input.routeScope ?? null,
|
|
6858
|
+
input.priority ?? 0,
|
|
6859
|
+
status,
|
|
6860
|
+
input.nextAttemptAt ?? null,
|
|
6861
|
+
input.lastReason ?? null,
|
|
6862
|
+
now,
|
|
6863
|
+
now
|
|
6864
|
+
]);
|
|
6865
|
+
const row = await this.client.get("SELECT * FROM workflow_work_items WHERE route_key = $1 AND idempotency_key = $2 LIMIT 1", [input.routeKey, input.idempotencyKey]);
|
|
6866
|
+
if (!row)
|
|
6867
|
+
throw new Error(`workflow work item not found after upsert: ${input.routeKey}/${input.idempotencyKey}`);
|
|
6868
|
+
return rowToWorkflowWorkItem(row);
|
|
6679
6869
|
}
|
|
6680
6870
|
admitWorkflowWorkItem() {
|
|
6681
6871
|
throw new NotImplementedError("admitWorkflowWorkItem");
|
|
@@ -101,8 +101,8 @@ export declare class PostgresLoopStorage implements LoopStorageContract {
|
|
|
101
101
|
listGoalRuns(...args: M<"listGoalRuns">["args"]): Promise<M<"listGoalRuns">["result"]>;
|
|
102
102
|
createWorkflow(...args: M<"createWorkflow">["args"]): Promise<M<"createWorkflow">["result"]>;
|
|
103
103
|
archiveWorkflow(...args: M<"archiveWorkflow">["args"]): Promise<M<"archiveWorkflow">["result"]>;
|
|
104
|
-
createWorkflowInvocation():
|
|
105
|
-
upsertWorkflowWorkItem():
|
|
104
|
+
createWorkflowInvocation(...args: M<"createWorkflowInvocation">["args"]): Promise<M<"createWorkflowInvocation">["result"]>;
|
|
105
|
+
upsertWorkflowWorkItem(...args: M<"upsertWorkflowWorkItem">["args"]): Promise<M<"upsertWorkflowWorkItem">["result"]>;
|
|
106
106
|
admitWorkflowWorkItem(): never;
|
|
107
107
|
createGoal(): never;
|
|
108
108
|
createGoalPlanNodes(): never;
|
|
@@ -352,6 +352,9 @@ CREATE INDEX IF NOT EXISTS idx_run_receipts_knowledge_ids ON run_receipts USING
|
|
|
352
352
|
migration("0006_work_item_machine_id", `
|
|
353
353
|
ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS machine_id TEXT;
|
|
354
354
|
CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status);
|
|
355
|
+
`),
|
|
356
|
+
migration("0007_work_item_gate_deaths", `
|
|
357
|
+
ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS gate_deaths INTEGER NOT NULL DEFAULT 0;
|
|
355
358
|
`)
|
|
356
359
|
]);
|
|
357
360
|
export {
|
|
@@ -352,6 +352,9 @@ CREATE INDEX IF NOT EXISTS idx_run_receipts_knowledge_ids ON run_receipts USING
|
|
|
352
352
|
migration("0006_work_item_machine_id", `
|
|
353
353
|
ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS machine_id TEXT;
|
|
354
354
|
CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status);
|
|
355
|
+
`),
|
|
356
|
+
migration("0007_work_item_gate_deaths", `
|
|
357
|
+
ALTER TABLE workflow_work_items ADD COLUMN IF NOT EXISTS gate_deaths INTEGER NOT NULL DEFAULT 0;
|
|
355
358
|
`)
|
|
356
359
|
]);
|
|
357
360
|
|
|
@@ -1569,6 +1569,7 @@ var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
|
1569
1569
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
1570
1570
|
var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
|
|
1571
1571
|
var SCHEMA_USER_VERSION = 8;
|
|
1572
|
+
var BREAKING_SCHEMA_FLOOR = 7;
|
|
1572
1573
|
var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
|
|
1573
1574
|
var PRUNE_BATCH_SIZE = 400;
|
|
1574
1575
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
@@ -1711,6 +1712,7 @@ function rowToWorkflowWorkItem(row) {
|
|
|
1711
1712
|
priority: row.priority,
|
|
1712
1713
|
status: row.status,
|
|
1713
1714
|
attempts: row.attempts,
|
|
1715
|
+
gateDeaths: row.gate_deaths ?? 0,
|
|
1714
1716
|
nextAttemptAt: row.next_attempt_at ?? undefined,
|
|
1715
1717
|
leaseExpiresAt: row.lease_expires_at ?? undefined,
|
|
1716
1718
|
workflowId: row.workflow_id ?? undefined,
|
|
@@ -1860,6 +1862,23 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
|
|
|
1860
1862
|
}
|
|
1861
1863
|
return;
|
|
1862
1864
|
}
|
|
1865
|
+
var WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
|
|
1866
|
+
var GATE_STEP_IDS = new Set(["triage", "planner", "plan"]);
|
|
1867
|
+
var GATE_DEATH_MAX_DURATION_MS = 60000;
|
|
1868
|
+
var GATE_DEATH_CEILING = 20;
|
|
1869
|
+
function classifyNonProductiveStepFailure(steps) {
|
|
1870
|
+
const failing = [...steps].reverse().find((step) => step.status === "failed" || step.status === "timed_out");
|
|
1871
|
+
if (!failing)
|
|
1872
|
+
return;
|
|
1873
|
+
if (failing.exitCode === WORK_ITEM_TEMPFAIL_EXIT_CODE)
|
|
1874
|
+
return "tempfail";
|
|
1875
|
+
if (typeof failing.error === "string" && failing.error.includes("worktree preparation failed"))
|
|
1876
|
+
return "gate-death";
|
|
1877
|
+
const fast = failing.durationMs === undefined || failing.durationMs < GATE_DEATH_MAX_DURATION_MS;
|
|
1878
|
+
if (GATE_STEP_IDS.has(failing.stepId) && fast)
|
|
1879
|
+
return "gate-death";
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1863
1882
|
function scrubbedOrNull(value) {
|
|
1864
1883
|
return value == null ? null : scrubSecrets(value);
|
|
1865
1884
|
}
|
|
@@ -1960,11 +1979,21 @@ class Store {
|
|
|
1960
1979
|
id TEXT PRIMARY KEY,
|
|
1961
1980
|
applied_at TEXT NOT NULL
|
|
1962
1981
|
);
|
|
1982
|
+
CREATE TABLE IF NOT EXISTS schema_compat (
|
|
1983
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
1984
|
+
min_compatible_user_version INTEGER NOT NULL
|
|
1985
|
+
);
|
|
1963
1986
|
`);
|
|
1964
1987
|
const versionRow = this.db.query("PRAGMA user_version").get();
|
|
1965
1988
|
const userVersion = versionRow?.user_version ?? 0;
|
|
1966
1989
|
if (userVersion > SCHEMA_USER_VERSION) {
|
|
1967
|
-
|
|
1990
|
+
const floorRow = this.db.query("SELECT min_compatible_user_version FROM schema_compat WHERE id = 1").get();
|
|
1991
|
+
if (!floorRow) {
|
|
1992
|
+
throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}) and carries no compatibility floor; upgrade open-loops before opening this database`);
|
|
1993
|
+
}
|
|
1994
|
+
if (SCHEMA_USER_VERSION < floorRow.min_compatible_user_version) {
|
|
1995
|
+
throw new Error(`loops database schema version ${userVersion} requires a binary with schema support >= ${floorRow.min_compatible_user_version} (this binary supports ${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
|
|
1996
|
+
}
|
|
1968
1997
|
}
|
|
1969
1998
|
const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
|
|
1970
1999
|
for (const migration of this.migrations()) {
|
|
@@ -1975,8 +2004,10 @@ class Store {
|
|
|
1975
2004
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
|
|
1976
2005
|
}
|
|
1977
2006
|
}
|
|
1978
|
-
if (userVersion
|
|
2007
|
+
if (userVersion < SCHEMA_USER_VERSION)
|
|
1979
2008
|
this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
|
|
2009
|
+
this.db.query(`INSERT INTO schema_compat (id, min_compatible_user_version) VALUES (1, ?)
|
|
2010
|
+
ON CONFLICT(id) DO UPDATE SET min_compatible_user_version = MAX(min_compatible_user_version, excluded.min_compatible_user_version)`).run(BREAKING_SCHEMA_FLOOR);
|
|
1980
2011
|
}
|
|
1981
2012
|
migrations() {
|
|
1982
2013
|
return [
|
|
@@ -2043,6 +2074,12 @@ class Store {
|
|
|
2043
2074
|
this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
|
|
2044
2075
|
this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
|
|
2045
2076
|
}
|
|
2077
|
+
},
|
|
2078
|
+
{
|
|
2079
|
+
id: "0011_work_item_gate_deaths",
|
|
2080
|
+
apply: () => {
|
|
2081
|
+
this.addColumnIfMissing("workflow_work_items", "gate_deaths", "INTEGER NOT NULL DEFAULT 0");
|
|
2082
|
+
}
|
|
2046
2083
|
}
|
|
2047
2084
|
];
|
|
2048
2085
|
}
|
|
@@ -3173,7 +3210,7 @@ class Store {
|
|
|
3173
3210
|
}
|
|
3174
3211
|
upsertWorkflowWorkItem(input) {
|
|
3175
3212
|
const now = nowIso();
|
|
3176
|
-
const id = genId();
|
|
3213
|
+
const id = input.id ?? genId();
|
|
3177
3214
|
const status = input.status ?? "queued";
|
|
3178
3215
|
this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
|
|
3179
3216
|
subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
|
|
@@ -3301,6 +3338,7 @@ class Store {
|
|
|
3301
3338
|
const placeholders = requeueableStatuses.map(() => "?").join(",");
|
|
3302
3339
|
const res = this.db.query(`UPDATE workflow_work_items
|
|
3303
3340
|
SET status='queued', workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
|
|
3341
|
+
${patch.resetAttempts ? "attempts=0, gate_deaths=0," : ""}
|
|
3304
3342
|
next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
|
|
3305
3343
|
WHERE id=? AND status IN (${placeholders})`).run(reason, now, id, ...requeueableStatuses);
|
|
3306
3344
|
const item = this.getWorkflowWorkItem(id);
|
|
@@ -3310,6 +3348,46 @@ class Store {
|
|
|
3310
3348
|
throw new Error(`workflow work item was not requeued: ${id} status=${item.status}`);
|
|
3311
3349
|
return item;
|
|
3312
3350
|
}
|
|
3351
|
+
deadLetterWorkflowWorkItem(id, patch = {}) {
|
|
3352
|
+
const now = nowIso();
|
|
3353
|
+
const reason = patch.reason?.trim() || "redispatch cap reached; dead-lettered";
|
|
3354
|
+
this.db.query(`UPDATE workflow_work_items
|
|
3355
|
+
SET status='dead_letter', next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
|
|
3356
|
+
WHERE id=? AND status IN ('succeeded','failed','cancelled')`).run(reason, now, id);
|
|
3357
|
+
const item = this.getWorkflowWorkItem(id);
|
|
3358
|
+
if (!item)
|
|
3359
|
+
throw new Error(`workflow work item not found after dead-letter: ${id}`);
|
|
3360
|
+
return item;
|
|
3361
|
+
}
|
|
3362
|
+
demoteNonProductiveWorkItems(workflowRunId, finishedAt) {
|
|
3363
|
+
const kind = classifyNonProductiveStepFailure(this.listWorkflowStepRuns(workflowRunId));
|
|
3364
|
+
if (!kind) {
|
|
3365
|
+
this.db.query("UPDATE workflow_work_items SET gate_deaths=0, updated_at=? WHERE workflow_run_id=? AND status='failed' AND gate_deaths > 0").run(finishedAt, workflowRunId);
|
|
3366
|
+
return;
|
|
3367
|
+
}
|
|
3368
|
+
if (kind === "tempfail") {
|
|
3369
|
+
this.db.query(`UPDATE workflow_work_items
|
|
3370
|
+
SET status='queued', attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
|
|
3371
|
+
gate_deaths=0,
|
|
3372
|
+
workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
|
|
3373
|
+
next_attempt_at=NULL, lease_expires_at=NULL,
|
|
3374
|
+
last_reason='worker exited 75 (tempfail): requeued for retry; attempt refunded (does not count toward redispatch cap)',
|
|
3375
|
+
updated_at=?
|
|
3376
|
+
WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
|
|
3377
|
+
return;
|
|
3378
|
+
}
|
|
3379
|
+
this.db.query(`UPDATE workflow_work_items
|
|
3380
|
+
SET attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
|
|
3381
|
+
gate_deaths=gate_deaths + 1,
|
|
3382
|
+
status=CASE WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING} THEN 'dead_letter' ELSE status END,
|
|
3383
|
+
last_reason=CASE
|
|
3384
|
+
WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING}
|
|
3385
|
+
THEN 'gate-death ceiling reached (' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING} consecutive runs died at worktree prep / triage / planner without reaching the worker): dead-lettered \u2014 the infrastructure fault needs an operator; ''loops routes requeue'' resets and retries'
|
|
3386
|
+
ELSE 'gate death before real work (worktree prep / triage / planner): attempt refunded (does not count toward redispatch cap); consecutive gate deaths: ' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING}'
|
|
3387
|
+
END,
|
|
3388
|
+
updated_at=?
|
|
3389
|
+
WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
|
|
3390
|
+
}
|
|
3313
3391
|
admitWorkflowWorkItem(id, patch) {
|
|
3314
3392
|
const now = nowIso();
|
|
3315
3393
|
const res = this.db.query(`UPDATE workflow_work_items
|
|
@@ -4021,6 +4099,8 @@ class Store {
|
|
|
4021
4099
|
if (changed) {
|
|
4022
4100
|
const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
|
|
4023
4101
|
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
|
|
4102
|
+
if (itemStatus === "failed")
|
|
4103
|
+
this.demoteNonProductiveWorkItems(workflowRunId, finishedAt);
|
|
4024
4104
|
this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
|
|
4025
4105
|
}
|
|
4026
4106
|
this.db.exec("COMMIT");
|
|
@@ -81,6 +81,7 @@ export interface LoopStore {
|
|
|
81
81
|
getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
|
|
82
82
|
requeueWorkflowWorkItem(id: string, patch?: {
|
|
83
83
|
reason?: string;
|
|
84
|
+
resetAttempts?: boolean;
|
|
84
85
|
}): Promise<WorkflowWorkItem>;
|
|
85
86
|
listWorkflowInvocations(opts?: {
|
|
86
87
|
limit?: number;
|
|
@@ -174,6 +175,7 @@ export declare class LocalStore implements LoopStore {
|
|
|
174
175
|
getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
|
|
175
176
|
requeueWorkflowWorkItem(id: string, patch?: {
|
|
176
177
|
reason?: string;
|
|
178
|
+
resetAttempts?: boolean;
|
|
177
179
|
}): Promise<WorkflowWorkItem>;
|
|
178
180
|
listWorkflowInvocations(opts?: Parameters<Store["listWorkflowInvocations"]>[0]): Promise<WorkflowInvocation[]>;
|
|
179
181
|
getWorkflowInvocation(id: string): Promise<WorkflowInvocation | undefined>;
|
|
@@ -242,6 +244,7 @@ export declare class ApiStore implements LoopStore {
|
|
|
242
244
|
getWorkflowWorkItem(id: string): Promise<WorkflowWorkItem | undefined>;
|
|
243
245
|
requeueWorkflowWorkItem(_id: string, _patch?: {
|
|
244
246
|
reason?: string;
|
|
247
|
+
resetAttempts?: boolean;
|
|
245
248
|
}): Promise<WorkflowWorkItem>;
|
|
246
249
|
listWorkflowInvocations(opts?: Parameters<LoopStore["listWorkflowInvocations"]>[0]): Promise<WorkflowInvocation[]>;
|
|
247
250
|
getWorkflowInvocation(id: string): Promise<WorkflowInvocation | undefined>;
|