@mingchuno/agent-workflows 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -6
- package/dist/src/adapters/agents.js +6 -3
- package/dist/src/adapters/hosting.js +16 -10
- package/dist/src/adapters/sdk-protocol.d.ts +3 -3
- package/dist/src/adapters/sdk-protocol.js +9 -7
- package/dist/src/attribution.d.ts +9 -0
- package/dist/src/attribution.js +57 -0
- package/dist/src/cli.d.ts +1 -1
- package/dist/src/cli.js +80 -25
- package/dist/src/config.d.ts +24 -30
- package/dist/src/config.js +32 -27
- package/dist/src/defaults.d.ts +2 -0
- package/dist/src/defaults.js +2 -0
- package/dist/src/domain.d.ts +31 -4
- package/dist/src/domain.js +10 -3
- package/dist/src/evidence.d.ts +54 -0
- package/dist/src/evidence.js +214 -0
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/invocation.d.ts +25 -0
- package/dist/src/invocation.js +166 -0
- package/dist/src/operations.d.ts +8 -2
- package/dist/src/operations.js +100 -136
- package/dist/src/prompts.d.ts +28 -0
- package/dist/src/prompts.js +63 -0
- package/dist/src/recovery.d.ts +19 -0
- package/dist/src/recovery.js +99 -0
- package/dist/src/runner.d.ts +7 -0
- package/dist/src/runner.js +170 -18
- package/dist/src/runtime/process.d.ts +2 -0
- package/dist/src/runtime/process.js +41 -12
- package/dist/src/store.d.ts +21 -2
- package/dist/src/store.js +122 -1
- package/dist/src/tui/actions.d.ts +16 -0
- package/dist/src/tui/actions.js +23 -0
- package/dist/src/tui/constants.d.ts +6 -0
- package/dist/src/tui/constants.js +3 -0
- package/dist/src/{tui-data.d.ts → tui/data.d.ts} +10 -7
- package/dist/src/tui/data.js +146 -0
- package/dist/src/tui/dialogs.d.ts +17 -0
- package/dist/src/tui/dialogs.js +149 -0
- package/dist/src/tui/format.d.ts +7 -0
- package/dist/src/tui/format.js +62 -0
- package/dist/src/tui/index.d.ts +3 -0
- package/dist/src/tui/index.js +2 -0
- package/dist/src/tui/layout.d.ts +25 -0
- package/dist/src/tui/layout.js +36 -0
- package/dist/src/tui/log-file.d.ts +26 -0
- package/dist/src/tui/log-file.js +156 -0
- package/dist/src/tui/log.d.ts +11 -0
- package/dist/src/tui/log.js +90 -0
- package/dist/src/tui/monitor.d.ts +10 -0
- package/dist/src/tui/monitor.js +284 -0
- package/dist/src/tui/notifications.d.ts +23 -0
- package/dist/src/tui/notifications.js +104 -0
- package/dist/src/tui/text.d.ts +3 -0
- package/dist/src/tui/text.js +10 -0
- package/dist/src/tui/use-log-controller.d.ts +27 -0
- package/dist/src/tui/use-log-controller.js +192 -0
- package/dist/src/tui/views.d.ts +25 -0
- package/dist/src/tui/views.js +327 -0
- package/dist/src/workspace.js +21 -8
- package/docs/api.md +132 -8
- package/docs/architecture.md +21 -4
- package/docs/configuration.md +181 -8
- package/docs/database.md +7 -0
- package/docs/operations.md +160 -5
- package/docs/providers.md +58 -2
- package/docs/releases.md +34 -79
- package/examples/config.ts +6 -6
- package/examples/run.ts +4 -1
- package/package.json +4 -2
- package/dist/src/tui-data.js +0 -89
- package/dist/src/tui.d.ts +0 -5
- package/dist/src/tui.js +0 -69
- package/docs/acceptance.md +0 -35
package/dist/src/runner.js
CHANGED
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { realpathSync, statSync } from "node:fs";
|
|
2
3
|
import { mkdir, realpath } from "node:fs/promises";
|
|
3
4
|
import { resolve } from "node:path";
|
|
4
5
|
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
5
6
|
import { configSchema } from "./config.js";
|
|
6
7
|
import { BlockedError, isBlockedError, } from "./domain.js";
|
|
8
|
+
import { assertEvidenceDirectory } from "./evidence.js";
|
|
7
9
|
import { defaultWorkflow, Operations } from "./operations.js";
|
|
10
|
+
import { projectPrompts } from "./prompts.js";
|
|
11
|
+
import { executionFingerprint, verifyPublicationRecovery } from "./recovery.js";
|
|
8
12
|
import { createQueuedRun } from "./run-record.js";
|
|
9
13
|
import { assertProcessesStopped, CheckoutOwnership, } from "./runtime/ownership.js";
|
|
10
14
|
import { createRedactor, runtimeLogger } from "./runtime/redaction.js";
|
|
11
15
|
import { Store } from "./store.js";
|
|
12
16
|
import { ExistingCheckout } from "./workspace.js";
|
|
17
|
+
const runnerPollIntervalMs = 100;
|
|
18
|
+
const shutdownPollIntervalMs = 20;
|
|
13
19
|
export class Runner {
|
|
14
20
|
options;
|
|
15
21
|
store;
|
|
@@ -17,6 +23,7 @@ export class Runner {
|
|
|
17
23
|
controllers = new Map();
|
|
18
24
|
active = new Map();
|
|
19
25
|
hosting = new Map();
|
|
26
|
+
promptBaseDirectory;
|
|
20
27
|
workflow;
|
|
21
28
|
ownership = new CheckoutOwnership();
|
|
22
29
|
stopping = false;
|
|
@@ -28,6 +35,16 @@ export class Runner {
|
|
|
28
35
|
constructor(options) {
|
|
29
36
|
this.options = options;
|
|
30
37
|
this.config = configSchema.parse(options.config);
|
|
38
|
+
const pathBaseDirectory = canonicalDirectory(options.pathBaseDirectory ?? process.cwd(), "Configuration path base");
|
|
39
|
+
this.config.stateDirectory = resolve(pathBaseDirectory, this.config.stateDirectory);
|
|
40
|
+
for (const project of this.config.projects)
|
|
41
|
+
project.checkout = resolve(pathBaseDirectory, project.checkout);
|
|
42
|
+
const promptBaseDirectory = options.promptBaseDirectory
|
|
43
|
+
? resolve(options.promptBaseDirectory)
|
|
44
|
+
: pathBaseDirectory;
|
|
45
|
+
this.promptBaseDirectory = promptBaseDirectory;
|
|
46
|
+
for (const project of this.config.projects)
|
|
47
|
+
projectPrompts(project, promptBaseDirectory);
|
|
31
48
|
this.store = new Store(options.databaseUrl, this.config.id, this.redact);
|
|
32
49
|
}
|
|
33
50
|
queue(id) {
|
|
@@ -42,6 +59,7 @@ export class Runner {
|
|
|
42
59
|
const ids = new Set();
|
|
43
60
|
for (const project of this.config.projects) {
|
|
44
61
|
project.checkout = await realpath(project.checkout);
|
|
62
|
+
this.config.stateDirectory = await assertEvidenceDirectory(project.checkout, this.config.stateDirectory);
|
|
45
63
|
if (canonical.has(project.checkout) || ids.has(project.id))
|
|
46
64
|
throw new Error("Duplicate project identity or canonical checkout");
|
|
47
65
|
canonical.add(project.checkout);
|
|
@@ -69,7 +87,7 @@ export class Runner {
|
|
|
69
87
|
DBOS.setConfig({
|
|
70
88
|
name: `agent-workflows-${this.config.id}`,
|
|
71
89
|
systemDatabaseUrl: this.options.databaseUrl,
|
|
72
|
-
applicationVersion: `${this.config.id}-${this.options.workflowVersion ?? "phase1-
|
|
90
|
+
applicationVersion: `${this.config.id}-${this.options.workflowVersion ?? "phase1-v2"}`,
|
|
73
91
|
executorID: this.config.id,
|
|
74
92
|
listenQueues: this.config.projects.map((p) => this.queue(p.id)),
|
|
75
93
|
logger: runtimeLogger(this.redact),
|
|
@@ -80,13 +98,13 @@ export class Runner {
|
|
|
80
98
|
await DBOS.registerQueue(this.queue(project.id), {
|
|
81
99
|
globalConcurrency: 1,
|
|
82
100
|
workerConcurrency: 1,
|
|
83
|
-
minPollingIntervalMs:
|
|
101
|
+
minPollingIntervalMs: runnerPollIntervalMs,
|
|
84
102
|
});
|
|
85
103
|
this.timer = setInterval(() => {
|
|
86
104
|
void this.tick().catch((error) => this.store.emit(null, "runner-error", {
|
|
87
105
|
error: this.redact(String(error)),
|
|
88
106
|
}));
|
|
89
|
-
},
|
|
107
|
+
}, runnerPollIntervalMs);
|
|
90
108
|
await this.tick();
|
|
91
109
|
}
|
|
92
110
|
redact = (text) => createRedactor([
|
|
@@ -162,6 +180,8 @@ export class Runner {
|
|
|
162
180
|
await this.stop(request.target);
|
|
163
181
|
else if (request.kind === "retry")
|
|
164
182
|
await this.retry(request.target, request.id);
|
|
183
|
+
else if (request.kind === "recover")
|
|
184
|
+
await this.recover(request.target, request.id);
|
|
165
185
|
else
|
|
166
186
|
throw new Error("Unknown command");
|
|
167
187
|
await this.store.finishCommand(request.id);
|
|
@@ -197,10 +217,7 @@ export class Runner {
|
|
|
197
217
|
(r.outcome === "queued" || r.outcome === "running"));
|
|
198
218
|
if (!run || this.active.has(run.id))
|
|
199
219
|
continue;
|
|
200
|
-
const handle = await
|
|
201
|
-
workflowID: run.id,
|
|
202
|
-
queueName: this.queue(project.id),
|
|
203
|
-
})(run.id);
|
|
220
|
+
const handle = await this.dispatch(run, project);
|
|
204
221
|
const result = handle
|
|
205
222
|
.getResult()
|
|
206
223
|
.catch(async (error) => {
|
|
@@ -210,18 +227,40 @@ export class Runner {
|
|
|
210
227
|
this.active.set(run.id, result);
|
|
211
228
|
}
|
|
212
229
|
}
|
|
230
|
+
async dispatch(run, project) {
|
|
231
|
+
const execution = run.executions?.at(-1);
|
|
232
|
+
if (execution?.recoveryOf) {
|
|
233
|
+
// Intent is committed first. On restart, adopt an existing fork instead of
|
|
234
|
+
// creating it twice after an uncertain DBOS response.
|
|
235
|
+
if (await DBOS.getWorkflowStatus(execution.id))
|
|
236
|
+
return DBOS.retrieveWorkflow(execution.id);
|
|
237
|
+
return DBOS.forkWorkflow(execution.recoveryOf, execution.startStep, {
|
|
238
|
+
newWorkflowID: execution.id,
|
|
239
|
+
queueName: this.queue(project.id),
|
|
240
|
+
applicationVersion: `${this.config.id}-${this.options.workflowVersion ?? "phase1-v2"}`,
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
return DBOS.startWorkflow(this.workflow, {
|
|
244
|
+
workflowID: run.id,
|
|
245
|
+
queueName: this.queue(project.id),
|
|
246
|
+
})(run.id);
|
|
247
|
+
}
|
|
213
248
|
async recordWorkflowFailure(run, error) {
|
|
214
249
|
const message = this.redact(String(error));
|
|
215
|
-
|
|
250
|
+
const executionId = run.executions?.at(-1)?.id ?? run.id;
|
|
251
|
+
await this.store.emit(run.id, "workflow-error", {
|
|
252
|
+
executionId,
|
|
253
|
+
error: message,
|
|
254
|
+
});
|
|
216
255
|
const current = await this.store.run(run.id);
|
|
256
|
+
if ((current.executions?.at(-1)?.id ?? current.id) !== executionId)
|
|
257
|
+
return;
|
|
217
258
|
if (["queued", "running"].includes(current.outcome)) {
|
|
218
259
|
await this.store.patchRun(run.id, {
|
|
219
260
|
outcome: "blocked",
|
|
220
261
|
error: message,
|
|
221
262
|
});
|
|
222
|
-
await this.store.
|
|
223
|
-
blocked: `DBOS execution failed for ${run.id}; inspect recovery evidence`,
|
|
224
|
-
});
|
|
263
|
+
await this.store.blockProject(run.projectId, `DBOS execution failed for ${run.id}; inspect recovery evidence`);
|
|
225
264
|
}
|
|
226
265
|
}
|
|
227
266
|
async execute(runId) {
|
|
@@ -230,6 +269,8 @@ export class Runner {
|
|
|
230
269
|
controller.abort();
|
|
231
270
|
this.controllers.set(runId, controller);
|
|
232
271
|
try {
|
|
272
|
+
if (DBOS.workflowID !== runId && this.options.workflow)
|
|
273
|
+
throw new BlockedError("Publication recovery supports the default workflow only");
|
|
233
274
|
await this.executeOwned(runId, controller);
|
|
234
275
|
}
|
|
235
276
|
finally {
|
|
@@ -237,14 +278,16 @@ export class Runner {
|
|
|
237
278
|
}
|
|
238
279
|
}
|
|
239
280
|
async executeOwned(runId, controller) {
|
|
240
|
-
const run = await DBOS.runStep(() => this.
|
|
281
|
+
const run = await DBOS.runStep(() => this.initializeExecution(runId), {
|
|
241
282
|
name: "load-run",
|
|
242
283
|
});
|
|
243
284
|
const project = this.config.projects.find((p) => p.id === run.projectId);
|
|
244
285
|
if (!project)
|
|
245
286
|
throw new Error("Project removed from configuration");
|
|
246
287
|
const workspace = this.options.workspace ?? new ExistingCheckout();
|
|
288
|
+
let recoveryChecked = false;
|
|
247
289
|
const operations = new Operations(runId, {
|
|
290
|
+
promptBaseDirectory: this.promptBaseDirectory,
|
|
248
291
|
store: this.store,
|
|
249
292
|
project,
|
|
250
293
|
workspace,
|
|
@@ -253,6 +296,36 @@ export class Runner {
|
|
|
253
296
|
artifacts: resolve(this.config.stateDirectory),
|
|
254
297
|
signal: controller.signal,
|
|
255
298
|
redact: this.redact,
|
|
299
|
+
executionFingerprint: () => executionFingerprint(project, this.options.workflowVersion ?? "phase1-v2", controller.signal),
|
|
300
|
+
beforeStep: async () => {
|
|
301
|
+
if (recoveryChecked)
|
|
302
|
+
return;
|
|
303
|
+
const current = await this.store.run(runId);
|
|
304
|
+
const execution = current.executions?.at(-1);
|
|
305
|
+
if (!execution?.recoveryOf) {
|
|
306
|
+
await this.store.patchRun(runId, { outcome: "running" });
|
|
307
|
+
recoveryChecked = true;
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
if (execution.id !== DBOS.workflowID)
|
|
311
|
+
throw new BlockedError("Execution has been superseded");
|
|
312
|
+
if (DBOS.stepID < execution.startStep)
|
|
313
|
+
throw new BlockedError("A reused checkpoint is missing; recovery refused");
|
|
314
|
+
// This runs inside the first non-replayed step, so copied start-gate
|
|
315
|
+
// checkpoints cannot bypass today's pause or checkout checks.
|
|
316
|
+
while (true) {
|
|
317
|
+
controller.signal.throwIfAborted();
|
|
318
|
+
const state = await this.store.project(project.id);
|
|
319
|
+
if (state.blocked)
|
|
320
|
+
throw new BlockedError(state.blocked);
|
|
321
|
+
if (!state.paused)
|
|
322
|
+
break;
|
|
323
|
+
await new Promise((resolve) => setTimeout(resolve, runnerPollIntervalMs));
|
|
324
|
+
}
|
|
325
|
+
await this.store.patchRun(runId, { outcome: "running" });
|
|
326
|
+
await this.checkRecoveryState(current, project, controller.signal);
|
|
327
|
+
recoveryChecked = true;
|
|
328
|
+
},
|
|
256
329
|
});
|
|
257
330
|
try {
|
|
258
331
|
while (true) {
|
|
@@ -293,9 +366,7 @@ export class Runner {
|
|
|
293
366
|
error: this.redact(String(error)),
|
|
294
367
|
});
|
|
295
368
|
if (unsafe || isBlockedError(error))
|
|
296
|
-
await this.store.
|
|
297
|
-
blocked: `Run ${runId} requires recovery: ${this.redact(String(error))}`,
|
|
298
|
-
});
|
|
369
|
+
await this.store.blockProject(project.id, `Run ${runId} requires recovery: ${this.redact(String(error))}`);
|
|
299
370
|
}, { name: "record-failure", retriesAllowed: false });
|
|
300
371
|
}
|
|
301
372
|
finally {
|
|
@@ -306,6 +377,27 @@ export class Runner {
|
|
|
306
377
|
await this.store.project(projectId);
|
|
307
378
|
await this.store.setProject(projectId, { paused: true });
|
|
308
379
|
}
|
|
380
|
+
async initializeExecution(runId) {
|
|
381
|
+
const run = await this.store.run(runId);
|
|
382
|
+
if (run.executions?.length)
|
|
383
|
+
return run;
|
|
384
|
+
const project = this.config.projects.find((item) => item.id === run.projectId);
|
|
385
|
+
if (!project)
|
|
386
|
+
throw new Error("Project removed from configuration");
|
|
387
|
+
return this.store.patchRun(runId, {
|
|
388
|
+
executions: [
|
|
389
|
+
{
|
|
390
|
+
id: run.id,
|
|
391
|
+
// Capture inputs after prepare fetches and checks out the actual base.
|
|
392
|
+
fingerprint: "",
|
|
393
|
+
recoverySupported: !this.options.workflow,
|
|
394
|
+
createdAt: run.createdAt,
|
|
395
|
+
outcome: run.outcome,
|
|
396
|
+
phase: run.phase,
|
|
397
|
+
},
|
|
398
|
+
],
|
|
399
|
+
});
|
|
400
|
+
}
|
|
309
401
|
async resume(projectId) {
|
|
310
402
|
const state = await this.store.project(projectId);
|
|
311
403
|
if (state.blocked)
|
|
@@ -318,11 +410,11 @@ export class Runner {
|
|
|
318
410
|
if (controller) {
|
|
319
411
|
controller.abort();
|
|
320
412
|
while (this.controllers.has(runId))
|
|
321
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
413
|
+
await new Promise((resolve) => setTimeout(resolve, shutdownPollIntervalMs));
|
|
322
414
|
return;
|
|
323
415
|
}
|
|
324
416
|
if (run.outcome === "queued") {
|
|
325
|
-
await DBOS.cancelWorkflow(runId);
|
|
417
|
+
await DBOS.cancelWorkflow(run.executions?.at(-1)?.id ?? runId);
|
|
326
418
|
await this.store.patchRun(runId, { outcome: "cancelled" });
|
|
327
419
|
return;
|
|
328
420
|
}
|
|
@@ -347,6 +439,51 @@ export class Runner {
|
|
|
347
439
|
},
|
|
348
440
|
});
|
|
349
441
|
}
|
|
442
|
+
async recover(runId, commandId) {
|
|
443
|
+
return this.store.admitRecovery(runId, {
|
|
444
|
+
commandId,
|
|
445
|
+
checkSafety: async (run) => {
|
|
446
|
+
if (!this.ownsRuntime || this.stopping)
|
|
447
|
+
throw new Error("Recovery requires an active runner");
|
|
448
|
+
if (this.options.workflow)
|
|
449
|
+
throw new Error("Publication recovery currently supports the default workflow only");
|
|
450
|
+
if (this.controllers.has(runId))
|
|
451
|
+
throw new Error("Work has not stopped");
|
|
452
|
+
const project = this.config.projects.find((item) => item.id === run.projectId);
|
|
453
|
+
if (!project)
|
|
454
|
+
throw new Error("Project removed from configuration");
|
|
455
|
+
const execution = run.executions.at(-1);
|
|
456
|
+
const status = await DBOS.getWorkflowStatus(execution.id);
|
|
457
|
+
if (!status || !["SUCCESS", "ERROR"].includes(status.status))
|
|
458
|
+
throw new Error("Source execution has not finished");
|
|
459
|
+
if (status.applicationVersion !==
|
|
460
|
+
`${this.config.id}-${this.options.workflowVersion ?? "phase1-v2"}`)
|
|
461
|
+
throw new Error("Workflow version changed; use retry");
|
|
462
|
+
const steps = await DBOS.listWorkflowSteps(execution.id);
|
|
463
|
+
const failed = steps?.find((step) => step.functionID === execution.failedStep);
|
|
464
|
+
if (!failed?.error || failed.name !== run.phase)
|
|
465
|
+
throw new Error("Failed publication checkpoint is unavailable");
|
|
466
|
+
const prefix = steps.filter((step) => step.functionID < failed.functionID);
|
|
467
|
+
if (prefix.length !== failed.functionID ||
|
|
468
|
+
prefix.some((step, index) => step.error || step.functionID !== index) ||
|
|
469
|
+
!prefix.some((step) => step.name === "commit"))
|
|
470
|
+
throw new Error("Completed publication checkpoints are unavailable");
|
|
471
|
+
await this.checkRecoveryState(run, project);
|
|
472
|
+
return prefix.map((step) => step.name);
|
|
473
|
+
},
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
async checkRecoveryState(run, project, signal) {
|
|
477
|
+
await verifyPublicationRecovery(run, {
|
|
478
|
+
project,
|
|
479
|
+
signal,
|
|
480
|
+
store: this.store,
|
|
481
|
+
workspace: this.options.workspace ?? new ExistingCheckout(),
|
|
482
|
+
hosting: this.hosting.get(project.id),
|
|
483
|
+
stateDirectory: this.config.stateDirectory,
|
|
484
|
+
workflowVersion: this.options.workflowVersion ?? "phase1-v2",
|
|
485
|
+
});
|
|
486
|
+
}
|
|
350
487
|
async shutdown() {
|
|
351
488
|
this.stopping = true;
|
|
352
489
|
if (this.timer)
|
|
@@ -354,7 +491,7 @@ export class Runner {
|
|
|
354
491
|
for (const controller of this.controllers.values())
|
|
355
492
|
controller.abort();
|
|
356
493
|
while (this.controllers.size || this.tickBusy)
|
|
357
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
494
|
+
await new Promise((resolve) => setTimeout(resolve, shutdownPollIntervalMs));
|
|
358
495
|
await Promise.all(this.active.values());
|
|
359
496
|
await Promise.allSettled(this.polling.values());
|
|
360
497
|
if (this.ownsRuntime) {
|
|
@@ -368,3 +505,18 @@ export class Runner {
|
|
|
368
505
|
await this.store.close();
|
|
369
506
|
}
|
|
370
507
|
}
|
|
508
|
+
function canonicalDirectory(path, label) {
|
|
509
|
+
const resolved = resolve(path);
|
|
510
|
+
let canonical;
|
|
511
|
+
try {
|
|
512
|
+
canonical = realpathSync(resolved);
|
|
513
|
+
}
|
|
514
|
+
catch (error) {
|
|
515
|
+
throw new Error(`${label} is not an existing directory: ${resolved}`, {
|
|
516
|
+
cause: error,
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
if (!statSync(canonical).isDirectory())
|
|
520
|
+
throw new Error(`${label} is not an existing directory: ${resolved}`);
|
|
521
|
+
return canonical;
|
|
522
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export declare const maxCapturedOutputBytes: number;
|
|
1
2
|
export interface CommandOptions {
|
|
2
3
|
cwd: string;
|
|
3
4
|
processFile?: string;
|
|
@@ -7,6 +8,7 @@ export interface CommandOptions {
|
|
|
7
8
|
env?: NodeJS.ProcessEnv;
|
|
8
9
|
allowFailure?: boolean;
|
|
9
10
|
captureOutput?: boolean;
|
|
11
|
+
strictUtf8?: boolean;
|
|
10
12
|
onOutput?: (chunk: string) => void;
|
|
11
13
|
}
|
|
12
14
|
export interface CommandResult {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
export const maxCapturedOutputBytes = 32 * 1024 * 1024;
|
|
3
4
|
/** Resolves only after the child closes. Cancellation terminates its process group. */
|
|
4
5
|
export function command(executable, args, options) {
|
|
5
6
|
options.signal?.throwIfAborted();
|
|
@@ -23,10 +24,14 @@ export function command(executable, args, options) {
|
|
|
23
24
|
}
|
|
24
25
|
catch (error) {
|
|
25
26
|
if (error.code !== "ESRCH")
|
|
26
|
-
failure =
|
|
27
|
+
failure = failure
|
|
28
|
+
? new AggregateError([failure, error], `${String(failure)}; process group termination failed: ${String(error)}`)
|
|
29
|
+
: error;
|
|
27
30
|
}
|
|
28
31
|
};
|
|
29
32
|
const stop = () => {
|
|
33
|
+
if (cancelled)
|
|
34
|
+
return;
|
|
30
35
|
cancelled = true;
|
|
31
36
|
kill("SIGTERM");
|
|
32
37
|
killTimer ??= setTimeout(() => kill("SIGKILL"), options.killGraceMs ?? 2000);
|
|
@@ -43,27 +48,51 @@ export function command(executable, args, options) {
|
|
|
43
48
|
}
|
|
44
49
|
const timeout = setTimeout(stop, options.timeoutMs ?? 300_000);
|
|
45
50
|
options.signal?.addEventListener("abort", stop, { once: true });
|
|
46
|
-
const
|
|
47
|
-
|
|
51
|
+
const decoders = {
|
|
52
|
+
stdout: new TextDecoder("utf-8", {
|
|
53
|
+
fatal: options.strictUtf8,
|
|
54
|
+
ignoreBOM: true,
|
|
55
|
+
}),
|
|
56
|
+
stderr: new TextDecoder("utf-8", {
|
|
57
|
+
fatal: options.strictUtf8,
|
|
58
|
+
ignoreBOM: true,
|
|
59
|
+
}),
|
|
60
|
+
};
|
|
61
|
+
let capturedBytes = 0;
|
|
62
|
+
const decodeOutput = (target, chunk) => {
|
|
48
63
|
try {
|
|
49
|
-
|
|
64
|
+
const value = decoders[target].decode(chunk, {
|
|
65
|
+
stream: chunk !== undefined,
|
|
66
|
+
});
|
|
67
|
+
if (value)
|
|
68
|
+
options.onOutput?.(value);
|
|
69
|
+
if (options.captureOutput !== false) {
|
|
70
|
+
if (target === "stdout")
|
|
71
|
+
stdout += value;
|
|
72
|
+
else
|
|
73
|
+
stderr += value;
|
|
74
|
+
}
|
|
50
75
|
}
|
|
51
76
|
catch (error) {
|
|
52
|
-
failure
|
|
77
|
+
failure ??= error;
|
|
53
78
|
stop();
|
|
54
79
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
if (stdout.length + stderr.length > 32 * 1024 * 1024)
|
|
80
|
+
};
|
|
81
|
+
const collect = (target, chunk) => {
|
|
82
|
+
if (options.captureOutput !== false) {
|
|
83
|
+
capturedBytes += chunk.length;
|
|
84
|
+
if (capturedBytes > maxCapturedOutputBytes) {
|
|
85
|
+
failure ??= new Error(`Command output size ${capturedBytes} bytes exceeds capture limit ${maxCapturedOutputBytes} bytes`);
|
|
62
86
|
stop();
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
63
89
|
}
|
|
90
|
+
decodeOutput(target, chunk);
|
|
64
91
|
};
|
|
65
92
|
child.stdout.on("data", (chunk) => collect("stdout", chunk));
|
|
66
93
|
child.stderr.on("data", (chunk) => collect("stderr", chunk));
|
|
94
|
+
child.stdout.on("end", () => decodeOutput("stdout"));
|
|
95
|
+
child.stderr.on("end", () => decodeOutput("stderr"));
|
|
67
96
|
const cleanup = () => {
|
|
68
97
|
clearTimeout(timeout);
|
|
69
98
|
if (killTimer)
|
package/dist/src/store.d.ts
CHANGED
|
@@ -25,7 +25,12 @@ export interface InvocationRecord {
|
|
|
25
25
|
requested: unknown;
|
|
26
26
|
effective: unknown;
|
|
27
27
|
prompt: string;
|
|
28
|
-
skills
|
|
28
|
+
/** Historical records only. New stages use runtime-managed skills. */
|
|
29
|
+
skills?: unknown;
|
|
30
|
+
taskPrompt?: import("./prompts.js").ResolvedPrompt;
|
|
31
|
+
outputContract?: string;
|
|
32
|
+
evidence?: import("./evidence.js").ChangeEvidence;
|
|
33
|
+
validationError?: string;
|
|
29
34
|
outcome: string;
|
|
30
35
|
startedAt: string;
|
|
31
36
|
finishedAt?: string;
|
|
@@ -38,6 +43,10 @@ interface RetryAdmission {
|
|
|
38
43
|
branchTemplate: string;
|
|
39
44
|
}>;
|
|
40
45
|
}
|
|
46
|
+
interface RecoveryAdmission {
|
|
47
|
+
commandId?: string;
|
|
48
|
+
checkSafety: (run: RunRecord) => Promise<string[]>;
|
|
49
|
+
}
|
|
41
50
|
/** Public persisted query surface. All queries work without a running executor. */
|
|
42
51
|
export declare class Store {
|
|
43
52
|
readonly scope: string;
|
|
@@ -58,11 +67,21 @@ export declare class Store {
|
|
|
58
67
|
blocked?: string | null;
|
|
59
68
|
}): Promise<void>;
|
|
60
69
|
insertRun(run: RunRecord): Promise<boolean>;
|
|
70
|
+
blockProject(id: string, reason: string): Promise<void>;
|
|
61
71
|
/**
|
|
62
72
|
* Admit a retry and its events atomically. Safety checks run under the project
|
|
63
73
|
* lock only for new admissions; they must not write through this Store.
|
|
64
74
|
*/
|
|
65
75
|
admitRetry(runId: string, admission: RetryAdmission): Promise<string>;
|
|
76
|
+
/** Persist recovery intent before dispatch; the command ID is its durable DBOS ID. */
|
|
77
|
+
admitRecovery(runId: string, admission: RecoveryAdmission): Promise<string>;
|
|
78
|
+
recoveryPlan(runId: string): Promise<{
|
|
79
|
+
eligible: boolean;
|
|
80
|
+
reason: string | null;
|
|
81
|
+
fromStep: string;
|
|
82
|
+
reuses: string;
|
|
83
|
+
checks: string;
|
|
84
|
+
}>;
|
|
66
85
|
run(id: string): Promise<RunRecord>;
|
|
67
86
|
runs(): Promise<RunRecord[]>;
|
|
68
87
|
patchRun(id: string, patch: Partial<RunRecord>): Promise<RunRecord>;
|
|
@@ -74,7 +93,7 @@ export declare class Store {
|
|
|
74
93
|
after?: number;
|
|
75
94
|
intervalMs?: number;
|
|
76
95
|
}): () => void;
|
|
77
|
-
request(kind: "pause" | "resume" | "stop" | "retry", target: string): Promise<string>;
|
|
96
|
+
request(kind: "pause" | "resume" | "stop" | "retry" | "recover", target: string): Promise<string>;
|
|
78
97
|
commands(): Promise<Array<{
|
|
79
98
|
id: string;
|
|
80
99
|
kind: string;
|
package/dist/src/store.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { and, eq, gt, Param, SQL } from "drizzle-orm";
|
|
2
|
+
import { and, eq, gt, isNull, Param, SQL } from "drizzle-orm";
|
|
3
3
|
import { drizzle } from "drizzle-orm/node-postgres";
|
|
4
4
|
import { Pool } from "pg";
|
|
5
5
|
import { tryLock, unlockAll } from "./db/locks.js";
|
|
6
6
|
import { migrateDatabase } from "./db/migrations.js";
|
|
7
7
|
import * as tables from "./db/schema.js";
|
|
8
|
+
import { recoveryUnavailable } from "./recovery.js";
|
|
8
9
|
import { createQueuedRun } from "./run-record.js";
|
|
9
10
|
import { redactValue } from "./runtime/redaction.js";
|
|
10
11
|
/** Public persisted query surface. All queries work without a running executor. */
|
|
@@ -113,6 +114,16 @@ export class Store {
|
|
|
113
114
|
await this.emit(run.id, "run", run);
|
|
114
115
|
return inserted.length > 0;
|
|
115
116
|
}
|
|
117
|
+
async blockProject(id, reason) {
|
|
118
|
+
const { projects } = tables;
|
|
119
|
+
const updated = await this.db
|
|
120
|
+
.update(projects)
|
|
121
|
+
.set({ blocked: this.redact(reason) })
|
|
122
|
+
.where(and(eq(projects.scope, this.scope), eq(projects.id, id), isNull(projects.blocked)))
|
|
123
|
+
.returning({ id: projects.id });
|
|
124
|
+
if (updated.length)
|
|
125
|
+
await this.emit(null, "project", { id, blocked: reason });
|
|
126
|
+
}
|
|
116
127
|
/**
|
|
117
128
|
* Admit a retry and its events atomically. Safety checks run under the project
|
|
118
129
|
* lock only for new admissions; they must not write through this Store.
|
|
@@ -192,6 +203,104 @@ export class Store {
|
|
|
192
203
|
return id;
|
|
193
204
|
});
|
|
194
205
|
}
|
|
206
|
+
/** Persist recovery intent before dispatch; the command ID is its durable DBOS ID. */
|
|
207
|
+
async admitRecovery(runId, admission) {
|
|
208
|
+
const original = await this.run(runId);
|
|
209
|
+
const { projects, runs, events } = tables;
|
|
210
|
+
const projectPredicate = and(eq(projects.scope, this.scope), eq(projects.id, original.projectId));
|
|
211
|
+
return this.db.transaction(async (tx) => {
|
|
212
|
+
const [project] = await tx
|
|
213
|
+
.select()
|
|
214
|
+
.from(projects)
|
|
215
|
+
.where(projectPredicate)
|
|
216
|
+
.for("update");
|
|
217
|
+
if (!project)
|
|
218
|
+
throw new Error(`Unknown project: ${original.projectId}`);
|
|
219
|
+
const history = await tx
|
|
220
|
+
.select({ record: runs.record })
|
|
221
|
+
.from(runs)
|
|
222
|
+
.where(and(eq(runs.scope, this.scope), eq(runs.taskKey, original.taskKey)))
|
|
223
|
+
.orderBy(runs.id)
|
|
224
|
+
.for("update");
|
|
225
|
+
const previous = history.find(({ record }) => record.id === runId)?.record;
|
|
226
|
+
if (!previous)
|
|
227
|
+
throw new Error(`Unknown run: ${runId}`);
|
|
228
|
+
if (admission.commandId) {
|
|
229
|
+
const records = await tx
|
|
230
|
+
.select({ record: runs.record })
|
|
231
|
+
.from(runs)
|
|
232
|
+
.where(eq(runs.scope, this.scope));
|
|
233
|
+
const owner = records.find(({ record }) => record.id === admission.commandId ||
|
|
234
|
+
record.executions?.some((item) => item.id === admission.commandId))?.record;
|
|
235
|
+
if (owner) {
|
|
236
|
+
if (owner.id !== runId ||
|
|
237
|
+
!owner.executions?.some((item) => item.id === admission.commandId && item.recoveryOf))
|
|
238
|
+
throw new Error("Recovery command identity belongs to a different execution");
|
|
239
|
+
return admission.commandId;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
const unavailable = recoveryUnavailable(previous);
|
|
243
|
+
if (unavailable)
|
|
244
|
+
throw new Error(unavailable);
|
|
245
|
+
if (history.some(({ record }) => record.attempt > previous.attempt ||
|
|
246
|
+
(record.id !== runId &&
|
|
247
|
+
["queued", "running"].includes(record.outcome))))
|
|
248
|
+
throw new Error("A newer attempt has superseded this run");
|
|
249
|
+
// Recovery must never remove a block imposed by a different operation.
|
|
250
|
+
if (project.blocked)
|
|
251
|
+
throw new Error(project.blocked);
|
|
252
|
+
const reusedSteps = await admission.checkSafety(previous);
|
|
253
|
+
const source = previous.executions.at(-1);
|
|
254
|
+
const id = admission.commandId ?? randomUUID();
|
|
255
|
+
const now = new Date().toISOString();
|
|
256
|
+
const execution = {
|
|
257
|
+
id,
|
|
258
|
+
recoveryOf: source.id,
|
|
259
|
+
startStep: source.failedStep,
|
|
260
|
+
reusedSteps,
|
|
261
|
+
fingerprint: source.fingerprint,
|
|
262
|
+
createdAt: now,
|
|
263
|
+
outcome: "queued",
|
|
264
|
+
phase: previous.phase,
|
|
265
|
+
recoverySupported: source.recoverySupported,
|
|
266
|
+
};
|
|
267
|
+
const recovered = {
|
|
268
|
+
...previous,
|
|
269
|
+
outcome: "queued",
|
|
270
|
+
updatedAt: now,
|
|
271
|
+
executions: [...previous.executions, execution],
|
|
272
|
+
};
|
|
273
|
+
delete recovered.error;
|
|
274
|
+
delete recovered.failedStep;
|
|
275
|
+
const persisted = redactValue(recovered, this.redact);
|
|
276
|
+
await tx
|
|
277
|
+
.update(runs)
|
|
278
|
+
.set({ record: persisted })
|
|
279
|
+
.where(and(eq(runs.scope, this.scope), eq(runs.id, runId)));
|
|
280
|
+
await tx.insert(events).values({
|
|
281
|
+
scope: this.scope,
|
|
282
|
+
runId,
|
|
283
|
+
kind: "recovery",
|
|
284
|
+
payload: redactValue(execution, this.redact),
|
|
285
|
+
});
|
|
286
|
+
return id;
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
async recoveryPlan(runId) {
|
|
290
|
+
const run = await this.run(runId);
|
|
291
|
+
const reason = recoveryUnavailable(run) ??
|
|
292
|
+
((await this.runs()).some((item) => item.taskKey === run.taskKey && item.attempt > run.attempt)
|
|
293
|
+
? "A newer attempt has superseded this run"
|
|
294
|
+
: undefined) ??
|
|
295
|
+
(await this.project(run.projectId)).blocked;
|
|
296
|
+
return {
|
|
297
|
+
eligible: !reason,
|
|
298
|
+
reason: reason ?? null,
|
|
299
|
+
fromStep: run.phase,
|
|
300
|
+
reuses: "Completed steps before the failed publication step, including implementation, validation and commit",
|
|
301
|
+
checks: "Runner verifies checkpoints, configuration, artifacts and checkout before recovery",
|
|
302
|
+
};
|
|
303
|
+
}
|
|
195
304
|
async run(id) {
|
|
196
305
|
const { runs } = tables;
|
|
197
306
|
const [row] = await this.db
|
|
@@ -232,6 +341,18 @@ export class Store {
|
|
|
232
341
|
// Match JSON serialization: undefined patch fields leave stored fields intact.
|
|
233
342
|
const persistedChange = JSON.parse(JSON.stringify(redactValue(change, this.redact)));
|
|
234
343
|
const merged = { ...row.record, ...persistedChange };
|
|
344
|
+
const execution = merged.executions?.at(-1);
|
|
345
|
+
if (execution) {
|
|
346
|
+
execution.outcome = merged.outcome;
|
|
347
|
+
if (merged.outcome === "running" && !execution.startedAt)
|
|
348
|
+
execution.startedAt = change.updatedAt;
|
|
349
|
+
execution.phase = merged.phase;
|
|
350
|
+
execution.error = merged.error;
|
|
351
|
+
execution.failedStep = merged.failedStep;
|
|
352
|
+
if (!execution.finishedAt &&
|
|
353
|
+
!["queued", "running"].includes(merged.outcome))
|
|
354
|
+
execution.finishedAt = change.updatedAt;
|
|
355
|
+
}
|
|
235
356
|
await tx.update(runs).set({ record: merged }).where(predicate);
|
|
236
357
|
return merged;
|
|
237
358
|
});
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { RunRecord } from "../domain.js";
|
|
2
|
+
import type { ProjectState } from "../store.js";
|
|
3
|
+
/** Display eligibility only; the runner still validates command admission. */
|
|
4
|
+
export declare function actionAvailability({ run, project, projectRuns, pending, }: {
|
|
5
|
+
run?: RunRecord;
|
|
6
|
+
project?: ProjectState;
|
|
7
|
+
projectRuns: RunRecord[];
|
|
8
|
+
pending: boolean;
|
|
9
|
+
}): {
|
|
10
|
+
recoveryReason: string | undefined;
|
|
11
|
+
available: {
|
|
12
|
+
stop: boolean;
|
|
13
|
+
retry: boolean;
|
|
14
|
+
recover: boolean;
|
|
15
|
+
};
|
|
16
|
+
};
|