@mingchuno/agent-workflows 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -6
- package/dist/src/adapters/agents.js +6 -3
- package/dist/src/adapters/hosting.js +16 -10
- package/dist/src/adapters/sdk-protocol.d.ts +3 -3
- package/dist/src/adapters/sdk-protocol.js +9 -7
- package/dist/src/cli.d.ts +1 -1
- package/dist/src/cli.js +40 -19
- package/dist/src/config.d.ts +22 -22
- package/dist/src/config.js +31 -26
- package/dist/src/defaults.d.ts +2 -0
- package/dist/src/defaults.js +2 -0
- package/dist/src/domain.d.ts +24 -4
- package/dist/src/domain.js +10 -3
- package/dist/src/evidence.d.ts +54 -0
- package/dist/src/evidence.js +214 -0
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/invocation.d.ts +24 -0
- package/dist/src/invocation.js +163 -0
- package/dist/src/operations.d.ts +7 -2
- package/dist/src/operations.js +76 -134
- package/dist/src/prompts.d.ts +28 -0
- package/dist/src/prompts.js +63 -0
- package/dist/src/recovery.d.ts +19 -0
- package/dist/src/recovery.js +99 -0
- package/dist/src/runner.d.ts +5 -0
- package/dist/src/runner.js +145 -18
- package/dist/src/runtime/process.d.ts +2 -0
- package/dist/src/runtime/process.js +41 -12
- package/dist/src/store.d.ts +21 -2
- package/dist/src/store.js +122 -1
- package/dist/src/tui/actions.d.ts +16 -0
- package/dist/src/tui/actions.js +23 -0
- package/dist/src/tui/constants.d.ts +6 -0
- package/dist/src/tui/constants.js +3 -0
- package/dist/src/{tui-data.d.ts → tui/data.d.ts} +8 -6
- package/dist/src/tui/data.js +141 -0
- package/dist/src/tui/dialogs.d.ts +17 -0
- package/dist/src/tui/dialogs.js +149 -0
- package/dist/src/tui/format.d.ts +7 -0
- package/dist/src/tui/format.js +62 -0
- package/dist/src/tui/index.d.ts +2 -0
- package/dist/src/tui/index.js +1 -0
- package/dist/src/tui/layout.d.ts +25 -0
- package/dist/src/tui/layout.js +36 -0
- package/dist/src/tui/log-file.d.ts +26 -0
- package/dist/src/tui/log-file.js +156 -0
- package/dist/src/tui/log.d.ts +11 -0
- package/dist/src/tui/log.js +90 -0
- package/dist/src/tui/monitor.d.ts +8 -0
- package/dist/src/tui/monitor.js +222 -0
- package/dist/src/tui/text.d.ts +3 -0
- package/dist/src/tui/text.js +10 -0
- package/dist/src/tui/use-log-controller.d.ts +27 -0
- package/dist/src/tui/use-log-controller.js +192 -0
- package/dist/src/tui/views.d.ts +17 -0
- package/dist/src/tui/views.js +97 -0
- package/docs/api.md +119 -6
- package/docs/architecture.md +21 -4
- package/docs/configuration.md +137 -5
- package/docs/database.md +7 -0
- package/docs/operations.md +117 -2
- package/docs/providers.md +58 -2
- package/docs/releases.md +34 -79
- package/examples/config.ts +2 -2
- package/package.json +4 -2
- package/dist/src/tui-data.js +0 -89
- package/dist/src/tui.d.ts +0 -5
- package/dist/src/tui.js +0 -69
- package/docs/acceptance.md +0 -35
package/dist/src/runner.js
CHANGED
|
@@ -4,12 +4,17 @@ import { resolve } from "node:path";
|
|
|
4
4
|
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
5
5
|
import { configSchema } from "./config.js";
|
|
6
6
|
import { BlockedError, isBlockedError, } from "./domain.js";
|
|
7
|
+
import { assertEvidenceDirectory } from "./evidence.js";
|
|
7
8
|
import { defaultWorkflow, Operations } from "./operations.js";
|
|
9
|
+
import { projectPrompts } from "./prompts.js";
|
|
10
|
+
import { executionFingerprint, verifyPublicationRecovery } from "./recovery.js";
|
|
8
11
|
import { createQueuedRun } from "./run-record.js";
|
|
9
12
|
import { assertProcessesStopped, CheckoutOwnership, } from "./runtime/ownership.js";
|
|
10
13
|
import { createRedactor, runtimeLogger } from "./runtime/redaction.js";
|
|
11
14
|
import { Store } from "./store.js";
|
|
12
15
|
import { ExistingCheckout } from "./workspace.js";
|
|
16
|
+
const runnerPollIntervalMs = 100;
|
|
17
|
+
const shutdownPollIntervalMs = 20;
|
|
13
18
|
export class Runner {
|
|
14
19
|
options;
|
|
15
20
|
store;
|
|
@@ -28,6 +33,8 @@ export class Runner {
|
|
|
28
33
|
constructor(options) {
|
|
29
34
|
this.options = options;
|
|
30
35
|
this.config = configSchema.parse(options.config);
|
|
36
|
+
for (const project of this.config.projects)
|
|
37
|
+
projectPrompts(project, options.promptBaseDirectory);
|
|
31
38
|
this.store = new Store(options.databaseUrl, this.config.id, this.redact);
|
|
32
39
|
}
|
|
33
40
|
queue(id) {
|
|
@@ -42,6 +49,7 @@ export class Runner {
|
|
|
42
49
|
const ids = new Set();
|
|
43
50
|
for (const project of this.config.projects) {
|
|
44
51
|
project.checkout = await realpath(project.checkout);
|
|
52
|
+
await assertEvidenceDirectory(project.checkout, this.config.stateDirectory);
|
|
45
53
|
if (canonical.has(project.checkout) || ids.has(project.id))
|
|
46
54
|
throw new Error("Duplicate project identity or canonical checkout");
|
|
47
55
|
canonical.add(project.checkout);
|
|
@@ -69,7 +77,7 @@ export class Runner {
|
|
|
69
77
|
DBOS.setConfig({
|
|
70
78
|
name: `agent-workflows-${this.config.id}`,
|
|
71
79
|
systemDatabaseUrl: this.options.databaseUrl,
|
|
72
|
-
applicationVersion: `${this.config.id}-${this.options.workflowVersion ?? "phase1-
|
|
80
|
+
applicationVersion: `${this.config.id}-${this.options.workflowVersion ?? "phase1-v2"}`,
|
|
73
81
|
executorID: this.config.id,
|
|
74
82
|
listenQueues: this.config.projects.map((p) => this.queue(p.id)),
|
|
75
83
|
logger: runtimeLogger(this.redact),
|
|
@@ -80,13 +88,13 @@ export class Runner {
|
|
|
80
88
|
await DBOS.registerQueue(this.queue(project.id), {
|
|
81
89
|
globalConcurrency: 1,
|
|
82
90
|
workerConcurrency: 1,
|
|
83
|
-
minPollingIntervalMs:
|
|
91
|
+
minPollingIntervalMs: runnerPollIntervalMs,
|
|
84
92
|
});
|
|
85
93
|
this.timer = setInterval(() => {
|
|
86
94
|
void this.tick().catch((error) => this.store.emit(null, "runner-error", {
|
|
87
95
|
error: this.redact(String(error)),
|
|
88
96
|
}));
|
|
89
|
-
},
|
|
97
|
+
}, runnerPollIntervalMs);
|
|
90
98
|
await this.tick();
|
|
91
99
|
}
|
|
92
100
|
redact = (text) => createRedactor([
|
|
@@ -162,6 +170,8 @@ export class Runner {
|
|
|
162
170
|
await this.stop(request.target);
|
|
163
171
|
else if (request.kind === "retry")
|
|
164
172
|
await this.retry(request.target, request.id);
|
|
173
|
+
else if (request.kind === "recover")
|
|
174
|
+
await this.recover(request.target, request.id);
|
|
165
175
|
else
|
|
166
176
|
throw new Error("Unknown command");
|
|
167
177
|
await this.store.finishCommand(request.id);
|
|
@@ -197,10 +207,7 @@ export class Runner {
|
|
|
197
207
|
(r.outcome === "queued" || r.outcome === "running"));
|
|
198
208
|
if (!run || this.active.has(run.id))
|
|
199
209
|
continue;
|
|
200
|
-
const handle = await
|
|
201
|
-
workflowID: run.id,
|
|
202
|
-
queueName: this.queue(project.id),
|
|
203
|
-
})(run.id);
|
|
210
|
+
const handle = await this.dispatch(run, project);
|
|
204
211
|
const result = handle
|
|
205
212
|
.getResult()
|
|
206
213
|
.catch(async (error) => {
|
|
@@ -210,18 +217,40 @@ export class Runner {
|
|
|
210
217
|
this.active.set(run.id, result);
|
|
211
218
|
}
|
|
212
219
|
}
|
|
220
|
+
async dispatch(run, project) {
|
|
221
|
+
const execution = run.executions?.at(-1);
|
|
222
|
+
if (execution?.recoveryOf) {
|
|
223
|
+
// Intent is committed first. On restart, adopt an existing fork instead of
|
|
224
|
+
// creating it twice after an uncertain DBOS response.
|
|
225
|
+
if (await DBOS.getWorkflowStatus(execution.id))
|
|
226
|
+
return DBOS.retrieveWorkflow(execution.id);
|
|
227
|
+
return DBOS.forkWorkflow(execution.recoveryOf, execution.startStep, {
|
|
228
|
+
newWorkflowID: execution.id,
|
|
229
|
+
queueName: this.queue(project.id),
|
|
230
|
+
applicationVersion: `${this.config.id}-${this.options.workflowVersion ?? "phase1-v2"}`,
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
return DBOS.startWorkflow(this.workflow, {
|
|
234
|
+
workflowID: run.id,
|
|
235
|
+
queueName: this.queue(project.id),
|
|
236
|
+
})(run.id);
|
|
237
|
+
}
|
|
213
238
|
async recordWorkflowFailure(run, error) {
|
|
214
239
|
const message = this.redact(String(error));
|
|
215
|
-
|
|
240
|
+
const executionId = run.executions?.at(-1)?.id ?? run.id;
|
|
241
|
+
await this.store.emit(run.id, "workflow-error", {
|
|
242
|
+
executionId,
|
|
243
|
+
error: message,
|
|
244
|
+
});
|
|
216
245
|
const current = await this.store.run(run.id);
|
|
246
|
+
if ((current.executions?.at(-1)?.id ?? current.id) !== executionId)
|
|
247
|
+
return;
|
|
217
248
|
if (["queued", "running"].includes(current.outcome)) {
|
|
218
249
|
await this.store.patchRun(run.id, {
|
|
219
250
|
outcome: "blocked",
|
|
220
251
|
error: message,
|
|
221
252
|
});
|
|
222
|
-
await this.store.
|
|
223
|
-
blocked: `DBOS execution failed for ${run.id}; inspect recovery evidence`,
|
|
224
|
-
});
|
|
253
|
+
await this.store.blockProject(run.projectId, `DBOS execution failed for ${run.id}; inspect recovery evidence`);
|
|
225
254
|
}
|
|
226
255
|
}
|
|
227
256
|
async execute(runId) {
|
|
@@ -230,6 +259,8 @@ export class Runner {
|
|
|
230
259
|
controller.abort();
|
|
231
260
|
this.controllers.set(runId, controller);
|
|
232
261
|
try {
|
|
262
|
+
if (DBOS.workflowID !== runId && this.options.workflow)
|
|
263
|
+
throw new BlockedError("Publication recovery supports the default workflow only");
|
|
233
264
|
await this.executeOwned(runId, controller);
|
|
234
265
|
}
|
|
235
266
|
finally {
|
|
@@ -237,14 +268,16 @@ export class Runner {
|
|
|
237
268
|
}
|
|
238
269
|
}
|
|
239
270
|
async executeOwned(runId, controller) {
|
|
240
|
-
const run = await DBOS.runStep(() => this.
|
|
271
|
+
const run = await DBOS.runStep(() => this.initializeExecution(runId), {
|
|
241
272
|
name: "load-run",
|
|
242
273
|
});
|
|
243
274
|
const project = this.config.projects.find((p) => p.id === run.projectId);
|
|
244
275
|
if (!project)
|
|
245
276
|
throw new Error("Project removed from configuration");
|
|
246
277
|
const workspace = this.options.workspace ?? new ExistingCheckout();
|
|
278
|
+
let recoveryChecked = false;
|
|
247
279
|
const operations = new Operations(runId, {
|
|
280
|
+
promptBaseDirectory: this.options.promptBaseDirectory,
|
|
248
281
|
store: this.store,
|
|
249
282
|
project,
|
|
250
283
|
workspace,
|
|
@@ -253,6 +286,36 @@ export class Runner {
|
|
|
253
286
|
artifacts: resolve(this.config.stateDirectory),
|
|
254
287
|
signal: controller.signal,
|
|
255
288
|
redact: this.redact,
|
|
289
|
+
executionFingerprint: () => executionFingerprint(project, this.options.workflowVersion ?? "phase1-v2", controller.signal),
|
|
290
|
+
beforeStep: async () => {
|
|
291
|
+
if (recoveryChecked)
|
|
292
|
+
return;
|
|
293
|
+
const current = await this.store.run(runId);
|
|
294
|
+
const execution = current.executions?.at(-1);
|
|
295
|
+
if (!execution?.recoveryOf) {
|
|
296
|
+
await this.store.patchRun(runId, { outcome: "running" });
|
|
297
|
+
recoveryChecked = true;
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
if (execution.id !== DBOS.workflowID)
|
|
301
|
+
throw new BlockedError("Execution has been superseded");
|
|
302
|
+
if (DBOS.stepID < execution.startStep)
|
|
303
|
+
throw new BlockedError("A reused checkpoint is missing; recovery refused");
|
|
304
|
+
// This runs inside the first non-replayed step, so copied start-gate
|
|
305
|
+
// checkpoints cannot bypass today's pause or checkout checks.
|
|
306
|
+
while (true) {
|
|
307
|
+
controller.signal.throwIfAborted();
|
|
308
|
+
const state = await this.store.project(project.id);
|
|
309
|
+
if (state.blocked)
|
|
310
|
+
throw new BlockedError(state.blocked);
|
|
311
|
+
if (!state.paused)
|
|
312
|
+
break;
|
|
313
|
+
await new Promise((resolve) => setTimeout(resolve, runnerPollIntervalMs));
|
|
314
|
+
}
|
|
315
|
+
await this.store.patchRun(runId, { outcome: "running" });
|
|
316
|
+
await this.checkRecoveryState(current, project, controller.signal);
|
|
317
|
+
recoveryChecked = true;
|
|
318
|
+
},
|
|
256
319
|
});
|
|
257
320
|
try {
|
|
258
321
|
while (true) {
|
|
@@ -293,9 +356,7 @@ export class Runner {
|
|
|
293
356
|
error: this.redact(String(error)),
|
|
294
357
|
});
|
|
295
358
|
if (unsafe || isBlockedError(error))
|
|
296
|
-
await this.store.
|
|
297
|
-
blocked: `Run ${runId} requires recovery: ${this.redact(String(error))}`,
|
|
298
|
-
});
|
|
359
|
+
await this.store.blockProject(project.id, `Run ${runId} requires recovery: ${this.redact(String(error))}`);
|
|
299
360
|
}, { name: "record-failure", retriesAllowed: false });
|
|
300
361
|
}
|
|
301
362
|
finally {
|
|
@@ -306,6 +367,27 @@ export class Runner {
|
|
|
306
367
|
await this.store.project(projectId);
|
|
307
368
|
await this.store.setProject(projectId, { paused: true });
|
|
308
369
|
}
|
|
370
|
+
async initializeExecution(runId) {
|
|
371
|
+
const run = await this.store.run(runId);
|
|
372
|
+
if (run.executions?.length)
|
|
373
|
+
return run;
|
|
374
|
+
const project = this.config.projects.find((item) => item.id === run.projectId);
|
|
375
|
+
if (!project)
|
|
376
|
+
throw new Error("Project removed from configuration");
|
|
377
|
+
return this.store.patchRun(runId, {
|
|
378
|
+
executions: [
|
|
379
|
+
{
|
|
380
|
+
id: run.id,
|
|
381
|
+
// Capture inputs after prepare fetches and checks out the actual base.
|
|
382
|
+
fingerprint: "",
|
|
383
|
+
recoverySupported: !this.options.workflow,
|
|
384
|
+
createdAt: run.createdAt,
|
|
385
|
+
outcome: run.outcome,
|
|
386
|
+
phase: run.phase,
|
|
387
|
+
},
|
|
388
|
+
],
|
|
389
|
+
});
|
|
390
|
+
}
|
|
309
391
|
async resume(projectId) {
|
|
310
392
|
const state = await this.store.project(projectId);
|
|
311
393
|
if (state.blocked)
|
|
@@ -318,11 +400,11 @@ export class Runner {
|
|
|
318
400
|
if (controller) {
|
|
319
401
|
controller.abort();
|
|
320
402
|
while (this.controllers.has(runId))
|
|
321
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
403
|
+
await new Promise((resolve) => setTimeout(resolve, shutdownPollIntervalMs));
|
|
322
404
|
return;
|
|
323
405
|
}
|
|
324
406
|
if (run.outcome === "queued") {
|
|
325
|
-
await DBOS.cancelWorkflow(runId);
|
|
407
|
+
await DBOS.cancelWorkflow(run.executions?.at(-1)?.id ?? runId);
|
|
326
408
|
await this.store.patchRun(runId, { outcome: "cancelled" });
|
|
327
409
|
return;
|
|
328
410
|
}
|
|
@@ -347,6 +429,51 @@ export class Runner {
|
|
|
347
429
|
},
|
|
348
430
|
});
|
|
349
431
|
}
|
|
432
|
+
async recover(runId, commandId) {
|
|
433
|
+
return this.store.admitRecovery(runId, {
|
|
434
|
+
commandId,
|
|
435
|
+
checkSafety: async (run) => {
|
|
436
|
+
if (!this.ownsRuntime || this.stopping)
|
|
437
|
+
throw new Error("Recovery requires an active runner");
|
|
438
|
+
if (this.options.workflow)
|
|
439
|
+
throw new Error("Publication recovery currently supports the default workflow only");
|
|
440
|
+
if (this.controllers.has(runId))
|
|
441
|
+
throw new Error("Work has not stopped");
|
|
442
|
+
const project = this.config.projects.find((item) => item.id === run.projectId);
|
|
443
|
+
if (!project)
|
|
444
|
+
throw new Error("Project removed from configuration");
|
|
445
|
+
const execution = run.executions.at(-1);
|
|
446
|
+
const status = await DBOS.getWorkflowStatus(execution.id);
|
|
447
|
+
if (!status || !["SUCCESS", "ERROR"].includes(status.status))
|
|
448
|
+
throw new Error("Source execution has not finished");
|
|
449
|
+
if (status.applicationVersion !==
|
|
450
|
+
`${this.config.id}-${this.options.workflowVersion ?? "phase1-v2"}`)
|
|
451
|
+
throw new Error("Workflow version changed; use retry");
|
|
452
|
+
const steps = await DBOS.listWorkflowSteps(execution.id);
|
|
453
|
+
const failed = steps?.find((step) => step.functionID === execution.failedStep);
|
|
454
|
+
if (!failed?.error || failed.name !== run.phase)
|
|
455
|
+
throw new Error("Failed publication checkpoint is unavailable");
|
|
456
|
+
const prefix = steps.filter((step) => step.functionID < failed.functionID);
|
|
457
|
+
if (prefix.length !== failed.functionID ||
|
|
458
|
+
prefix.some((step, index) => step.error || step.functionID !== index) ||
|
|
459
|
+
!prefix.some((step) => step.name === "commit"))
|
|
460
|
+
throw new Error("Completed publication checkpoints are unavailable");
|
|
461
|
+
await this.checkRecoveryState(run, project);
|
|
462
|
+
return prefix.map((step) => step.name);
|
|
463
|
+
},
|
|
464
|
+
});
|
|
465
|
+
}
|
|
466
|
+
async checkRecoveryState(run, project, signal) {
|
|
467
|
+
await verifyPublicationRecovery(run, {
|
|
468
|
+
project,
|
|
469
|
+
signal,
|
|
470
|
+
store: this.store,
|
|
471
|
+
workspace: this.options.workspace ?? new ExistingCheckout(),
|
|
472
|
+
hosting: this.hosting.get(project.id),
|
|
473
|
+
stateDirectory: this.config.stateDirectory,
|
|
474
|
+
workflowVersion: this.options.workflowVersion ?? "phase1-v2",
|
|
475
|
+
});
|
|
476
|
+
}
|
|
350
477
|
async shutdown() {
|
|
351
478
|
this.stopping = true;
|
|
352
479
|
if (this.timer)
|
|
@@ -354,7 +481,7 @@ export class Runner {
|
|
|
354
481
|
for (const controller of this.controllers.values())
|
|
355
482
|
controller.abort();
|
|
356
483
|
while (this.controllers.size || this.tickBusy)
|
|
357
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
484
|
+
await new Promise((resolve) => setTimeout(resolve, shutdownPollIntervalMs));
|
|
358
485
|
await Promise.all(this.active.values());
|
|
359
486
|
await Promise.allSettled(this.polling.values());
|
|
360
487
|
if (this.ownsRuntime) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export declare const maxCapturedOutputBytes: number;
|
|
1
2
|
export interface CommandOptions {
|
|
2
3
|
cwd: string;
|
|
3
4
|
processFile?: string;
|
|
@@ -7,6 +8,7 @@ export interface CommandOptions {
|
|
|
7
8
|
env?: NodeJS.ProcessEnv;
|
|
8
9
|
allowFailure?: boolean;
|
|
9
10
|
captureOutput?: boolean;
|
|
11
|
+
strictUtf8?: boolean;
|
|
10
12
|
onOutput?: (chunk: string) => void;
|
|
11
13
|
}
|
|
12
14
|
export interface CommandResult {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
export const maxCapturedOutputBytes = 32 * 1024 * 1024;
|
|
3
4
|
/** Resolves only after the child closes. Cancellation terminates its process group. */
|
|
4
5
|
export function command(executable, args, options) {
|
|
5
6
|
options.signal?.throwIfAborted();
|
|
@@ -23,10 +24,14 @@ export function command(executable, args, options) {
|
|
|
23
24
|
}
|
|
24
25
|
catch (error) {
|
|
25
26
|
if (error.code !== "ESRCH")
|
|
26
|
-
failure =
|
|
27
|
+
failure = failure
|
|
28
|
+
? new AggregateError([failure, error], `${String(failure)}; process group termination failed: ${String(error)}`)
|
|
29
|
+
: error;
|
|
27
30
|
}
|
|
28
31
|
};
|
|
29
32
|
const stop = () => {
|
|
33
|
+
if (cancelled)
|
|
34
|
+
return;
|
|
30
35
|
cancelled = true;
|
|
31
36
|
kill("SIGTERM");
|
|
32
37
|
killTimer ??= setTimeout(() => kill("SIGKILL"), options.killGraceMs ?? 2000);
|
|
@@ -43,27 +48,51 @@ export function command(executable, args, options) {
|
|
|
43
48
|
}
|
|
44
49
|
const timeout = setTimeout(stop, options.timeoutMs ?? 300_000);
|
|
45
50
|
options.signal?.addEventListener("abort", stop, { once: true });
|
|
46
|
-
const
|
|
47
|
-
|
|
51
|
+
const decoders = {
|
|
52
|
+
stdout: new TextDecoder("utf-8", {
|
|
53
|
+
fatal: options.strictUtf8,
|
|
54
|
+
ignoreBOM: true,
|
|
55
|
+
}),
|
|
56
|
+
stderr: new TextDecoder("utf-8", {
|
|
57
|
+
fatal: options.strictUtf8,
|
|
58
|
+
ignoreBOM: true,
|
|
59
|
+
}),
|
|
60
|
+
};
|
|
61
|
+
let capturedBytes = 0;
|
|
62
|
+
const decodeOutput = (target, chunk) => {
|
|
48
63
|
try {
|
|
49
|
-
|
|
64
|
+
const value = decoders[target].decode(chunk, {
|
|
65
|
+
stream: chunk !== undefined,
|
|
66
|
+
});
|
|
67
|
+
if (value)
|
|
68
|
+
options.onOutput?.(value);
|
|
69
|
+
if (options.captureOutput !== false) {
|
|
70
|
+
if (target === "stdout")
|
|
71
|
+
stdout += value;
|
|
72
|
+
else
|
|
73
|
+
stderr += value;
|
|
74
|
+
}
|
|
50
75
|
}
|
|
51
76
|
catch (error) {
|
|
52
|
-
failure
|
|
77
|
+
failure ??= error;
|
|
53
78
|
stop();
|
|
54
79
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
if (stdout.length + stderr.length > 32 * 1024 * 1024)
|
|
80
|
+
};
|
|
81
|
+
const collect = (target, chunk) => {
|
|
82
|
+
if (options.captureOutput !== false) {
|
|
83
|
+
capturedBytes += chunk.length;
|
|
84
|
+
if (capturedBytes > maxCapturedOutputBytes) {
|
|
85
|
+
failure ??= new Error(`Command output size ${capturedBytes} bytes exceeds capture limit ${maxCapturedOutputBytes} bytes`);
|
|
62
86
|
stop();
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
63
89
|
}
|
|
90
|
+
decodeOutput(target, chunk);
|
|
64
91
|
};
|
|
65
92
|
child.stdout.on("data", (chunk) => collect("stdout", chunk));
|
|
66
93
|
child.stderr.on("data", (chunk) => collect("stderr", chunk));
|
|
94
|
+
child.stdout.on("end", () => decodeOutput("stdout"));
|
|
95
|
+
child.stderr.on("end", () => decodeOutput("stderr"));
|
|
67
96
|
const cleanup = () => {
|
|
68
97
|
clearTimeout(timeout);
|
|
69
98
|
if (killTimer)
|
package/dist/src/store.d.ts
CHANGED
|
@@ -25,7 +25,12 @@ export interface InvocationRecord {
|
|
|
25
25
|
requested: unknown;
|
|
26
26
|
effective: unknown;
|
|
27
27
|
prompt: string;
|
|
28
|
-
skills
|
|
28
|
+
/** Historical records only. New stages use runtime-managed skills. */
|
|
29
|
+
skills?: unknown;
|
|
30
|
+
taskPrompt?: import("./prompts.js").ResolvedPrompt;
|
|
31
|
+
outputContract?: string;
|
|
32
|
+
evidence?: import("./evidence.js").ChangeEvidence;
|
|
33
|
+
validationError?: string;
|
|
29
34
|
outcome: string;
|
|
30
35
|
startedAt: string;
|
|
31
36
|
finishedAt?: string;
|
|
@@ -38,6 +43,10 @@ interface RetryAdmission {
|
|
|
38
43
|
branchTemplate: string;
|
|
39
44
|
}>;
|
|
40
45
|
}
|
|
46
|
+
interface RecoveryAdmission {
|
|
47
|
+
commandId?: string;
|
|
48
|
+
checkSafety: (run: RunRecord) => Promise<string[]>;
|
|
49
|
+
}
|
|
41
50
|
/** Public persisted query surface. All queries work without a running executor. */
|
|
42
51
|
export declare class Store {
|
|
43
52
|
readonly scope: string;
|
|
@@ -58,11 +67,21 @@ export declare class Store {
|
|
|
58
67
|
blocked?: string | null;
|
|
59
68
|
}): Promise<void>;
|
|
60
69
|
insertRun(run: RunRecord): Promise<boolean>;
|
|
70
|
+
blockProject(id: string, reason: string): Promise<void>;
|
|
61
71
|
/**
|
|
62
72
|
* Admit a retry and its events atomically. Safety checks run under the project
|
|
63
73
|
* lock only for new admissions; they must not write through this Store.
|
|
64
74
|
*/
|
|
65
75
|
admitRetry(runId: string, admission: RetryAdmission): Promise<string>;
|
|
76
|
+
/** Persist recovery intent before dispatch; the command ID is its durable DBOS ID. */
|
|
77
|
+
admitRecovery(runId: string, admission: RecoveryAdmission): Promise<string>;
|
|
78
|
+
recoveryPlan(runId: string): Promise<{
|
|
79
|
+
eligible: boolean;
|
|
80
|
+
reason: string | null;
|
|
81
|
+
fromStep: string;
|
|
82
|
+
reuses: string;
|
|
83
|
+
checks: string;
|
|
84
|
+
}>;
|
|
66
85
|
run(id: string): Promise<RunRecord>;
|
|
67
86
|
runs(): Promise<RunRecord[]>;
|
|
68
87
|
patchRun(id: string, patch: Partial<RunRecord>): Promise<RunRecord>;
|
|
@@ -74,7 +93,7 @@ export declare class Store {
|
|
|
74
93
|
after?: number;
|
|
75
94
|
intervalMs?: number;
|
|
76
95
|
}): () => void;
|
|
77
|
-
request(kind: "pause" | "resume" | "stop" | "retry", target: string): Promise<string>;
|
|
96
|
+
request(kind: "pause" | "resume" | "stop" | "retry" | "recover", target: string): Promise<string>;
|
|
78
97
|
commands(): Promise<Array<{
|
|
79
98
|
id: string;
|
|
80
99
|
kind: string;
|
package/dist/src/store.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { and, eq, gt, Param, SQL } from "drizzle-orm";
|
|
2
|
+
import { and, eq, gt, isNull, Param, SQL } from "drizzle-orm";
|
|
3
3
|
import { drizzle } from "drizzle-orm/node-postgres";
|
|
4
4
|
import { Pool } from "pg";
|
|
5
5
|
import { tryLock, unlockAll } from "./db/locks.js";
|
|
6
6
|
import { migrateDatabase } from "./db/migrations.js";
|
|
7
7
|
import * as tables from "./db/schema.js";
|
|
8
|
+
import { recoveryUnavailable } from "./recovery.js";
|
|
8
9
|
import { createQueuedRun } from "./run-record.js";
|
|
9
10
|
import { redactValue } from "./runtime/redaction.js";
|
|
10
11
|
/** Public persisted query surface. All queries work without a running executor. */
|
|
@@ -113,6 +114,16 @@ export class Store {
|
|
|
113
114
|
await this.emit(run.id, "run", run);
|
|
114
115
|
return inserted.length > 0;
|
|
115
116
|
}
|
|
117
|
+
async blockProject(id, reason) {
|
|
118
|
+
const { projects } = tables;
|
|
119
|
+
const updated = await this.db
|
|
120
|
+
.update(projects)
|
|
121
|
+
.set({ blocked: this.redact(reason) })
|
|
122
|
+
.where(and(eq(projects.scope, this.scope), eq(projects.id, id), isNull(projects.blocked)))
|
|
123
|
+
.returning({ id: projects.id });
|
|
124
|
+
if (updated.length)
|
|
125
|
+
await this.emit(null, "project", { id, blocked: reason });
|
|
126
|
+
}
|
|
116
127
|
/**
|
|
117
128
|
* Admit a retry and its events atomically. Safety checks run under the project
|
|
118
129
|
* lock only for new admissions; they must not write through this Store.
|
|
@@ -192,6 +203,104 @@ export class Store {
|
|
|
192
203
|
return id;
|
|
193
204
|
});
|
|
194
205
|
}
|
|
206
|
+
/** Persist recovery intent before dispatch; the command ID is its durable DBOS ID. */
|
|
207
|
+
async admitRecovery(runId, admission) {
|
|
208
|
+
const original = await this.run(runId);
|
|
209
|
+
const { projects, runs, events } = tables;
|
|
210
|
+
const projectPredicate = and(eq(projects.scope, this.scope), eq(projects.id, original.projectId));
|
|
211
|
+
return this.db.transaction(async (tx) => {
|
|
212
|
+
const [project] = await tx
|
|
213
|
+
.select()
|
|
214
|
+
.from(projects)
|
|
215
|
+
.where(projectPredicate)
|
|
216
|
+
.for("update");
|
|
217
|
+
if (!project)
|
|
218
|
+
throw new Error(`Unknown project: ${original.projectId}`);
|
|
219
|
+
const history = await tx
|
|
220
|
+
.select({ record: runs.record })
|
|
221
|
+
.from(runs)
|
|
222
|
+
.where(and(eq(runs.scope, this.scope), eq(runs.taskKey, original.taskKey)))
|
|
223
|
+
.orderBy(runs.id)
|
|
224
|
+
.for("update");
|
|
225
|
+
const previous = history.find(({ record }) => record.id === runId)?.record;
|
|
226
|
+
if (!previous)
|
|
227
|
+
throw new Error(`Unknown run: ${runId}`);
|
|
228
|
+
if (admission.commandId) {
|
|
229
|
+
const records = await tx
|
|
230
|
+
.select({ record: runs.record })
|
|
231
|
+
.from(runs)
|
|
232
|
+
.where(eq(runs.scope, this.scope));
|
|
233
|
+
const owner = records.find(({ record }) => record.id === admission.commandId ||
|
|
234
|
+
record.executions?.some((item) => item.id === admission.commandId))?.record;
|
|
235
|
+
if (owner) {
|
|
236
|
+
if (owner.id !== runId ||
|
|
237
|
+
!owner.executions?.some((item) => item.id === admission.commandId && item.recoveryOf))
|
|
238
|
+
throw new Error("Recovery command identity belongs to a different execution");
|
|
239
|
+
return admission.commandId;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
const unavailable = recoveryUnavailable(previous);
|
|
243
|
+
if (unavailable)
|
|
244
|
+
throw new Error(unavailable);
|
|
245
|
+
if (history.some(({ record }) => record.attempt > previous.attempt ||
|
|
246
|
+
(record.id !== runId &&
|
|
247
|
+
["queued", "running"].includes(record.outcome))))
|
|
248
|
+
throw new Error("A newer attempt has superseded this run");
|
|
249
|
+
// Recovery must never remove a block imposed by a different operation.
|
|
250
|
+
if (project.blocked)
|
|
251
|
+
throw new Error(project.blocked);
|
|
252
|
+
const reusedSteps = await admission.checkSafety(previous);
|
|
253
|
+
const source = previous.executions.at(-1);
|
|
254
|
+
const id = admission.commandId ?? randomUUID();
|
|
255
|
+
const now = new Date().toISOString();
|
|
256
|
+
const execution = {
|
|
257
|
+
id,
|
|
258
|
+
recoveryOf: source.id,
|
|
259
|
+
startStep: source.failedStep,
|
|
260
|
+
reusedSteps,
|
|
261
|
+
fingerprint: source.fingerprint,
|
|
262
|
+
createdAt: now,
|
|
263
|
+
outcome: "queued",
|
|
264
|
+
phase: previous.phase,
|
|
265
|
+
recoverySupported: source.recoverySupported,
|
|
266
|
+
};
|
|
267
|
+
const recovered = {
|
|
268
|
+
...previous,
|
|
269
|
+
outcome: "queued",
|
|
270
|
+
updatedAt: now,
|
|
271
|
+
executions: [...previous.executions, execution],
|
|
272
|
+
};
|
|
273
|
+
delete recovered.error;
|
|
274
|
+
delete recovered.failedStep;
|
|
275
|
+
const persisted = redactValue(recovered, this.redact);
|
|
276
|
+
await tx
|
|
277
|
+
.update(runs)
|
|
278
|
+
.set({ record: persisted })
|
|
279
|
+
.where(and(eq(runs.scope, this.scope), eq(runs.id, runId)));
|
|
280
|
+
await tx.insert(events).values({
|
|
281
|
+
scope: this.scope,
|
|
282
|
+
runId,
|
|
283
|
+
kind: "recovery",
|
|
284
|
+
payload: redactValue(execution, this.redact),
|
|
285
|
+
});
|
|
286
|
+
return id;
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
async recoveryPlan(runId) {
|
|
290
|
+
const run = await this.run(runId);
|
|
291
|
+
const reason = recoveryUnavailable(run) ??
|
|
292
|
+
((await this.runs()).some((item) => item.taskKey === run.taskKey && item.attempt > run.attempt)
|
|
293
|
+
? "A newer attempt has superseded this run"
|
|
294
|
+
: undefined) ??
|
|
295
|
+
(await this.project(run.projectId)).blocked;
|
|
296
|
+
return {
|
|
297
|
+
eligible: !reason,
|
|
298
|
+
reason: reason ?? null,
|
|
299
|
+
fromStep: run.phase,
|
|
300
|
+
reuses: "Completed steps before the failed publication step, including implementation, validation and commit",
|
|
301
|
+
checks: "Runner verifies checkpoints, configuration, artifacts and checkout before recovery",
|
|
302
|
+
};
|
|
303
|
+
}
|
|
195
304
|
async run(id) {
|
|
196
305
|
const { runs } = tables;
|
|
197
306
|
const [row] = await this.db
|
|
@@ -232,6 +341,18 @@ export class Store {
|
|
|
232
341
|
// Match JSON serialization: undefined patch fields leave stored fields intact.
|
|
233
342
|
const persistedChange = JSON.parse(JSON.stringify(redactValue(change, this.redact)));
|
|
234
343
|
const merged = { ...row.record, ...persistedChange };
|
|
344
|
+
const execution = merged.executions?.at(-1);
|
|
345
|
+
if (execution) {
|
|
346
|
+
execution.outcome = merged.outcome;
|
|
347
|
+
if (merged.outcome === "running" && !execution.startedAt)
|
|
348
|
+
execution.startedAt = change.updatedAt;
|
|
349
|
+
execution.phase = merged.phase;
|
|
350
|
+
execution.error = merged.error;
|
|
351
|
+
execution.failedStep = merged.failedStep;
|
|
352
|
+
if (!execution.finishedAt &&
|
|
353
|
+
!["queued", "running"].includes(merged.outcome))
|
|
354
|
+
execution.finishedAt = change.updatedAt;
|
|
355
|
+
}
|
|
235
356
|
await tx.update(runs).set({ record: merged }).where(predicate);
|
|
236
357
|
return merged;
|
|
237
358
|
});
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { RunRecord } from "../domain.js";
|
|
2
|
+
import type { ProjectState } from "../store.js";
|
|
3
|
+
/** Display eligibility only; the runner still validates command admission. */
|
|
4
|
+
export declare function actionAvailability({ run, project, projectRuns, pending, }: {
|
|
5
|
+
run?: RunRecord;
|
|
6
|
+
project?: ProjectState;
|
|
7
|
+
projectRuns: RunRecord[];
|
|
8
|
+
pending: boolean;
|
|
9
|
+
}): {
|
|
10
|
+
recoveryReason: string | undefined;
|
|
11
|
+
available: {
|
|
12
|
+
stop: boolean;
|
|
13
|
+
retry: boolean;
|
|
14
|
+
recover: boolean;
|
|
15
|
+
};
|
|
16
|
+
};
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { recoveryUnavailable } from "../recovery.js";
|
|
2
|
+
/** Display eligibility only; the runner still validates command admission. */
|
|
3
|
+
export function actionAvailability({ run, project, projectRuns, pending, }) {
|
|
4
|
+
const recoveryReason = run
|
|
5
|
+
? (recoveryUnavailable(run) ??
|
|
6
|
+
project?.blocked ??
|
|
7
|
+
(projectRuns.some((item) => item.taskKey === run.taskKey && item.attempt > run.attempt)
|
|
8
|
+
? "A newer attempt has superseded this run"
|
|
9
|
+
: undefined))
|
|
10
|
+
: undefined;
|
|
11
|
+
return {
|
|
12
|
+
recoveryReason,
|
|
13
|
+
available: {
|
|
14
|
+
stop: Boolean(run && !pending && ["queued", "running"].includes(run.outcome)),
|
|
15
|
+
retry: Boolean(run &&
|
|
16
|
+
!pending &&
|
|
17
|
+
["failed", "blocked", "cancelled"].includes(run.outcome) &&
|
|
18
|
+
!projectRuns.some((item) => item.taskKey === run.taskKey &&
|
|
19
|
+
["queued", "running"].includes(item.outcome))),
|
|
20
|
+
recover: Boolean(run && !pending && !recoveryReason),
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|