@mingchuno/agent-workflows 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +37 -6
  2. package/dist/src/adapters/agents.js +6 -3
  3. package/dist/src/adapters/hosting.js +16 -10
  4. package/dist/src/adapters/sdk-protocol.d.ts +3 -3
  5. package/dist/src/adapters/sdk-protocol.js +9 -7
  6. package/dist/src/attribution.d.ts +9 -0
  7. package/dist/src/attribution.js +57 -0
  8. package/dist/src/cli.d.ts +1 -1
  9. package/dist/src/cli.js +80 -25
  10. package/dist/src/config.d.ts +24 -30
  11. package/dist/src/config.js +32 -27
  12. package/dist/src/defaults.d.ts +2 -0
  13. package/dist/src/defaults.js +2 -0
  14. package/dist/src/domain.d.ts +31 -4
  15. package/dist/src/domain.js +10 -3
  16. package/dist/src/evidence.d.ts +54 -0
  17. package/dist/src/evidence.js +214 -0
  18. package/dist/src/index.d.ts +1 -0
  19. package/dist/src/index.js +1 -0
  20. package/dist/src/invocation.d.ts +25 -0
  21. package/dist/src/invocation.js +166 -0
  22. package/dist/src/operations.d.ts +8 -2
  23. package/dist/src/operations.js +100 -136
  24. package/dist/src/prompts.d.ts +28 -0
  25. package/dist/src/prompts.js +63 -0
  26. package/dist/src/recovery.d.ts +19 -0
  27. package/dist/src/recovery.js +99 -0
  28. package/dist/src/runner.d.ts +7 -0
  29. package/dist/src/runner.js +170 -18
  30. package/dist/src/runtime/process.d.ts +2 -0
  31. package/dist/src/runtime/process.js +41 -12
  32. package/dist/src/store.d.ts +21 -2
  33. package/dist/src/store.js +122 -1
  34. package/dist/src/tui/actions.d.ts +16 -0
  35. package/dist/src/tui/actions.js +23 -0
  36. package/dist/src/tui/constants.d.ts +6 -0
  37. package/dist/src/tui/constants.js +3 -0
  38. package/dist/src/{tui-data.d.ts → tui/data.d.ts} +10 -7
  39. package/dist/src/tui/data.js +146 -0
  40. package/dist/src/tui/dialogs.d.ts +17 -0
  41. package/dist/src/tui/dialogs.js +149 -0
  42. package/dist/src/tui/format.d.ts +7 -0
  43. package/dist/src/tui/format.js +62 -0
  44. package/dist/src/tui/index.d.ts +3 -0
  45. package/dist/src/tui/index.js +2 -0
  46. package/dist/src/tui/layout.d.ts +25 -0
  47. package/dist/src/tui/layout.js +36 -0
  48. package/dist/src/tui/log-file.d.ts +26 -0
  49. package/dist/src/tui/log-file.js +156 -0
  50. package/dist/src/tui/log.d.ts +11 -0
  51. package/dist/src/tui/log.js +90 -0
  52. package/dist/src/tui/monitor.d.ts +10 -0
  53. package/dist/src/tui/monitor.js +284 -0
  54. package/dist/src/tui/notifications.d.ts +23 -0
  55. package/dist/src/tui/notifications.js +104 -0
  56. package/dist/src/tui/text.d.ts +3 -0
  57. package/dist/src/tui/text.js +10 -0
  58. package/dist/src/tui/use-log-controller.d.ts +27 -0
  59. package/dist/src/tui/use-log-controller.js +192 -0
  60. package/dist/src/tui/views.d.ts +25 -0
  61. package/dist/src/tui/views.js +327 -0
  62. package/dist/src/workspace.js +21 -8
  63. package/docs/api.md +132 -8
  64. package/docs/architecture.md +21 -4
  65. package/docs/configuration.md +181 -8
  66. package/docs/database.md +7 -0
  67. package/docs/operations.md +160 -5
  68. package/docs/providers.md +58 -2
  69. package/docs/releases.md +34 -79
  70. package/examples/config.ts +6 -6
  71. package/examples/run.ts +4 -1
  72. package/package.json +4 -2
  73. package/dist/src/tui-data.js +0 -89
  74. package/dist/src/tui.d.ts +0 -5
  75. package/dist/src/tui.js +0 -69
  76. package/docs/acceptance.md +0 -35
@@ -1,15 +1,21 @@
1
1
  import { randomUUID } from "node:crypto";
2
+ import { realpathSync, statSync } from "node:fs";
2
3
  import { mkdir, realpath } from "node:fs/promises";
3
4
  import { resolve } from "node:path";
4
5
  import { DBOS } from "@dbos-inc/dbos-sdk";
5
6
  import { configSchema } from "./config.js";
6
7
  import { BlockedError, isBlockedError, } from "./domain.js";
8
+ import { assertEvidenceDirectory } from "./evidence.js";
7
9
  import { defaultWorkflow, Operations } from "./operations.js";
10
+ import { projectPrompts } from "./prompts.js";
11
+ import { executionFingerprint, verifyPublicationRecovery } from "./recovery.js";
8
12
  import { createQueuedRun } from "./run-record.js";
9
13
  import { assertProcessesStopped, CheckoutOwnership, } from "./runtime/ownership.js";
10
14
  import { createRedactor, runtimeLogger } from "./runtime/redaction.js";
11
15
  import { Store } from "./store.js";
12
16
  import { ExistingCheckout } from "./workspace.js";
17
+ const runnerPollIntervalMs = 100;
18
+ const shutdownPollIntervalMs = 20;
13
19
  export class Runner {
14
20
  options;
15
21
  store;
@@ -17,6 +23,7 @@ export class Runner {
17
23
  controllers = new Map();
18
24
  active = new Map();
19
25
  hosting = new Map();
26
+ promptBaseDirectory;
20
27
  workflow;
21
28
  ownership = new CheckoutOwnership();
22
29
  stopping = false;
@@ -28,6 +35,16 @@ export class Runner {
28
35
  constructor(options) {
29
36
  this.options = options;
30
37
  this.config = configSchema.parse(options.config);
38
+ const pathBaseDirectory = canonicalDirectory(options.pathBaseDirectory ?? process.cwd(), "Configuration path base");
39
+ this.config.stateDirectory = resolve(pathBaseDirectory, this.config.stateDirectory);
40
+ for (const project of this.config.projects)
41
+ project.checkout = resolve(pathBaseDirectory, project.checkout);
42
+ const promptBaseDirectory = options.promptBaseDirectory
43
+ ? resolve(options.promptBaseDirectory)
44
+ : pathBaseDirectory;
45
+ this.promptBaseDirectory = promptBaseDirectory;
46
+ for (const project of this.config.projects)
47
+ projectPrompts(project, promptBaseDirectory);
31
48
  this.store = new Store(options.databaseUrl, this.config.id, this.redact);
32
49
  }
33
50
  queue(id) {
@@ -42,6 +59,7 @@ export class Runner {
42
59
  const ids = new Set();
43
60
  for (const project of this.config.projects) {
44
61
  project.checkout = await realpath(project.checkout);
62
+ this.config.stateDirectory = await assertEvidenceDirectory(project.checkout, this.config.stateDirectory);
45
63
  if (canonical.has(project.checkout) || ids.has(project.id))
46
64
  throw new Error("Duplicate project identity or canonical checkout");
47
65
  canonical.add(project.checkout);
@@ -69,7 +87,7 @@ export class Runner {
69
87
  DBOS.setConfig({
70
88
  name: `agent-workflows-${this.config.id}`,
71
89
  systemDatabaseUrl: this.options.databaseUrl,
72
- applicationVersion: `${this.config.id}-${this.options.workflowVersion ?? "phase1-v1"}`,
90
+ applicationVersion: `${this.config.id}-${this.options.workflowVersion ?? "phase1-v2"}`,
73
91
  executorID: this.config.id,
74
92
  listenQueues: this.config.projects.map((p) => this.queue(p.id)),
75
93
  logger: runtimeLogger(this.redact),
@@ -80,13 +98,13 @@ export class Runner {
80
98
  await DBOS.registerQueue(this.queue(project.id), {
81
99
  globalConcurrency: 1,
82
100
  workerConcurrency: 1,
83
- minPollingIntervalMs: 100,
101
+ minPollingIntervalMs: runnerPollIntervalMs,
84
102
  });
85
103
  this.timer = setInterval(() => {
86
104
  void this.tick().catch((error) => this.store.emit(null, "runner-error", {
87
105
  error: this.redact(String(error)),
88
106
  }));
89
- }, 100);
107
+ }, runnerPollIntervalMs);
90
108
  await this.tick();
91
109
  }
92
110
  redact = (text) => createRedactor([
@@ -162,6 +180,8 @@ export class Runner {
162
180
  await this.stop(request.target);
163
181
  else if (request.kind === "retry")
164
182
  await this.retry(request.target, request.id);
183
+ else if (request.kind === "recover")
184
+ await this.recover(request.target, request.id);
165
185
  else
166
186
  throw new Error("Unknown command");
167
187
  await this.store.finishCommand(request.id);
@@ -197,10 +217,7 @@ export class Runner {
197
217
  (r.outcome === "queued" || r.outcome === "running"));
198
218
  if (!run || this.active.has(run.id))
199
219
  continue;
200
- const handle = await DBOS.startWorkflow(this.workflow, {
201
- workflowID: run.id,
202
- queueName: this.queue(project.id),
203
- })(run.id);
220
+ const handle = await this.dispatch(run, project);
204
221
  const result = handle
205
222
  .getResult()
206
223
  .catch(async (error) => {
@@ -210,18 +227,40 @@ export class Runner {
210
227
  this.active.set(run.id, result);
211
228
  }
212
229
  }
230
+ async dispatch(run, project) {
231
+ const execution = run.executions?.at(-1);
232
+ if (execution?.recoveryOf) {
233
+ // Intent is committed first. On restart, adopt an existing fork instead of
234
+ // creating it twice after an uncertain DBOS response.
235
+ if (await DBOS.getWorkflowStatus(execution.id))
236
+ return DBOS.retrieveWorkflow(execution.id);
237
+ return DBOS.forkWorkflow(execution.recoveryOf, execution.startStep, {
238
+ newWorkflowID: execution.id,
239
+ queueName: this.queue(project.id),
240
+ applicationVersion: `${this.config.id}-${this.options.workflowVersion ?? "phase1-v2"}`,
241
+ });
242
+ }
243
+ return DBOS.startWorkflow(this.workflow, {
244
+ workflowID: run.id,
245
+ queueName: this.queue(project.id),
246
+ })(run.id);
247
+ }
213
248
  async recordWorkflowFailure(run, error) {
214
249
  const message = this.redact(String(error));
215
- await this.store.emit(run.id, "workflow-error", { error: message });
250
+ const executionId = run.executions?.at(-1)?.id ?? run.id;
251
+ await this.store.emit(run.id, "workflow-error", {
252
+ executionId,
253
+ error: message,
254
+ });
216
255
  const current = await this.store.run(run.id);
256
+ if ((current.executions?.at(-1)?.id ?? current.id) !== executionId)
257
+ return;
217
258
  if (["queued", "running"].includes(current.outcome)) {
218
259
  await this.store.patchRun(run.id, {
219
260
  outcome: "blocked",
220
261
  error: message,
221
262
  });
222
- await this.store.setProject(run.projectId, {
223
- blocked: `DBOS execution failed for ${run.id}; inspect recovery evidence`,
224
- });
263
+ await this.store.blockProject(run.projectId, `DBOS execution failed for ${run.id}; inspect recovery evidence`);
225
264
  }
226
265
  }
227
266
  async execute(runId) {
@@ -230,6 +269,8 @@ export class Runner {
230
269
  controller.abort();
231
270
  this.controllers.set(runId, controller);
232
271
  try {
272
+ if (DBOS.workflowID !== runId && this.options.workflow)
273
+ throw new BlockedError("Publication recovery supports the default workflow only");
233
274
  await this.executeOwned(runId, controller);
234
275
  }
235
276
  finally {
@@ -237,14 +278,16 @@ export class Runner {
237
278
  }
238
279
  }
239
280
  async executeOwned(runId, controller) {
240
- const run = await DBOS.runStep(() => this.store.run(runId), {
281
+ const run = await DBOS.runStep(() => this.initializeExecution(runId), {
241
282
  name: "load-run",
242
283
  });
243
284
  const project = this.config.projects.find((p) => p.id === run.projectId);
244
285
  if (!project)
245
286
  throw new Error("Project removed from configuration");
246
287
  const workspace = this.options.workspace ?? new ExistingCheckout();
288
+ let recoveryChecked = false;
247
289
  const operations = new Operations(runId, {
290
+ promptBaseDirectory: this.promptBaseDirectory,
248
291
  store: this.store,
249
292
  project,
250
293
  workspace,
@@ -253,6 +296,36 @@ export class Runner {
253
296
  artifacts: resolve(this.config.stateDirectory),
254
297
  signal: controller.signal,
255
298
  redact: this.redact,
299
+ executionFingerprint: () => executionFingerprint(project, this.options.workflowVersion ?? "phase1-v2", controller.signal),
300
+ beforeStep: async () => {
301
+ if (recoveryChecked)
302
+ return;
303
+ const current = await this.store.run(runId);
304
+ const execution = current.executions?.at(-1);
305
+ if (!execution?.recoveryOf) {
306
+ await this.store.patchRun(runId, { outcome: "running" });
307
+ recoveryChecked = true;
308
+ return;
309
+ }
310
+ if (execution.id !== DBOS.workflowID)
311
+ throw new BlockedError("Execution has been superseded");
312
+ if (DBOS.stepID < execution.startStep)
313
+ throw new BlockedError("A reused checkpoint is missing; recovery refused");
314
+ // This runs inside the first non-replayed step, so copied start-gate
315
+ // checkpoints cannot bypass today's pause or checkout checks.
316
+ while (true) {
317
+ controller.signal.throwIfAborted();
318
+ const state = await this.store.project(project.id);
319
+ if (state.blocked)
320
+ throw new BlockedError(state.blocked);
321
+ if (!state.paused)
322
+ break;
323
+ await new Promise((resolve) => setTimeout(resolve, runnerPollIntervalMs));
324
+ }
325
+ await this.store.patchRun(runId, { outcome: "running" });
326
+ await this.checkRecoveryState(current, project, controller.signal);
327
+ recoveryChecked = true;
328
+ },
256
329
  });
257
330
  try {
258
331
  while (true) {
@@ -293,9 +366,7 @@ export class Runner {
293
366
  error: this.redact(String(error)),
294
367
  });
295
368
  if (unsafe || isBlockedError(error))
296
- await this.store.setProject(project.id, {
297
- blocked: `Run ${runId} requires recovery: ${this.redact(String(error))}`,
298
- });
369
+ await this.store.blockProject(project.id, `Run ${runId} requires recovery: ${this.redact(String(error))}`);
299
370
  }, { name: "record-failure", retriesAllowed: false });
300
371
  }
301
372
  finally {
@@ -306,6 +377,27 @@ export class Runner {
306
377
  await this.store.project(projectId);
307
378
  await this.store.setProject(projectId, { paused: true });
308
379
  }
380
+ async initializeExecution(runId) {
381
+ const run = await this.store.run(runId);
382
+ if (run.executions?.length)
383
+ return run;
384
+ const project = this.config.projects.find((item) => item.id === run.projectId);
385
+ if (!project)
386
+ throw new Error("Project removed from configuration");
387
+ return this.store.patchRun(runId, {
388
+ executions: [
389
+ {
390
+ id: run.id,
391
+ // Capture inputs after prepare fetches and checks out the actual base.
392
+ fingerprint: "",
393
+ recoverySupported: !this.options.workflow,
394
+ createdAt: run.createdAt,
395
+ outcome: run.outcome,
396
+ phase: run.phase,
397
+ },
398
+ ],
399
+ });
400
+ }
309
401
  async resume(projectId) {
310
402
  const state = await this.store.project(projectId);
311
403
  if (state.blocked)
@@ -318,11 +410,11 @@ export class Runner {
318
410
  if (controller) {
319
411
  controller.abort();
320
412
  while (this.controllers.has(runId))
321
- await new Promise((resolve) => setTimeout(resolve, 20));
413
+ await new Promise((resolve) => setTimeout(resolve, shutdownPollIntervalMs));
322
414
  return;
323
415
  }
324
416
  if (run.outcome === "queued") {
325
- await DBOS.cancelWorkflow(runId);
417
+ await DBOS.cancelWorkflow(run.executions?.at(-1)?.id ?? runId);
326
418
  await this.store.patchRun(runId, { outcome: "cancelled" });
327
419
  return;
328
420
  }
@@ -347,6 +439,51 @@ export class Runner {
347
439
  },
348
440
  });
349
441
  }
442
+ async recover(runId, commandId) {
443
+ return this.store.admitRecovery(runId, {
444
+ commandId,
445
+ checkSafety: async (run) => {
446
+ if (!this.ownsRuntime || this.stopping)
447
+ throw new Error("Recovery requires an active runner");
448
+ if (this.options.workflow)
449
+ throw new Error("Publication recovery currently supports the default workflow only");
450
+ if (this.controllers.has(runId))
451
+ throw new Error("Work has not stopped");
452
+ const project = this.config.projects.find((item) => item.id === run.projectId);
453
+ if (!project)
454
+ throw new Error("Project removed from configuration");
455
+ const execution = run.executions.at(-1);
456
+ const status = await DBOS.getWorkflowStatus(execution.id);
457
+ if (!status || !["SUCCESS", "ERROR"].includes(status.status))
458
+ throw new Error("Source execution has not finished");
459
+ if (status.applicationVersion !==
460
+ `${this.config.id}-${this.options.workflowVersion ?? "phase1-v2"}`)
461
+ throw new Error("Workflow version changed; use retry");
462
+ const steps = await DBOS.listWorkflowSteps(execution.id);
463
+ const failed = steps?.find((step) => step.functionID === execution.failedStep);
464
+ if (!failed?.error || failed.name !== run.phase)
465
+ throw new Error("Failed publication checkpoint is unavailable");
466
+ const prefix = steps.filter((step) => step.functionID < failed.functionID);
467
+ if (prefix.length !== failed.functionID ||
468
+ prefix.some((step, index) => step.error || step.functionID !== index) ||
469
+ !prefix.some((step) => step.name === "commit"))
470
+ throw new Error("Completed publication checkpoints are unavailable");
471
+ await this.checkRecoveryState(run, project);
472
+ return prefix.map((step) => step.name);
473
+ },
474
+ });
475
+ }
476
+ async checkRecoveryState(run, project, signal) {
477
+ await verifyPublicationRecovery(run, {
478
+ project,
479
+ signal,
480
+ store: this.store,
481
+ workspace: this.options.workspace ?? new ExistingCheckout(),
482
+ hosting: this.hosting.get(project.id),
483
+ stateDirectory: this.config.stateDirectory,
484
+ workflowVersion: this.options.workflowVersion ?? "phase1-v2",
485
+ });
486
+ }
350
487
  async shutdown() {
351
488
  this.stopping = true;
352
489
  if (this.timer)
@@ -354,7 +491,7 @@ export class Runner {
354
491
  for (const controller of this.controllers.values())
355
492
  controller.abort();
356
493
  while (this.controllers.size || this.tickBusy)
357
- await new Promise((resolve) => setTimeout(resolve, 20));
494
+ await new Promise((resolve) => setTimeout(resolve, shutdownPollIntervalMs));
358
495
  await Promise.all(this.active.values());
359
496
  await Promise.allSettled(this.polling.values());
360
497
  if (this.ownsRuntime) {
@@ -368,3 +505,18 @@ export class Runner {
368
505
  await this.store.close();
369
506
  }
370
507
  }
508
+ function canonicalDirectory(path, label) {
509
+ const resolved = resolve(path);
510
+ let canonical;
511
+ try {
512
+ canonical = realpathSync(resolved);
513
+ }
514
+ catch (error) {
515
+ throw new Error(`${label} is not an existing directory: ${resolved}`, {
516
+ cause: error,
517
+ });
518
+ }
519
+ if (!statSync(canonical).isDirectory())
520
+ throw new Error(`${label} is not an existing directory: ${resolved}`);
521
+ return canonical;
522
+ }
@@ -1,3 +1,4 @@
1
+ export declare const maxCapturedOutputBytes: number;
1
2
  export interface CommandOptions {
2
3
  cwd: string;
3
4
  processFile?: string;
@@ -7,6 +8,7 @@ export interface CommandOptions {
7
8
  env?: NodeJS.ProcessEnv;
8
9
  allowFailure?: boolean;
9
10
  captureOutput?: boolean;
11
+ strictUtf8?: boolean;
10
12
  onOutput?: (chunk: string) => void;
11
13
  }
12
14
  export interface CommandResult {
@@ -1,5 +1,6 @@
1
1
  import { spawn } from "node:child_process";
2
2
  import { unlinkSync, writeFileSync } from "node:fs";
3
+ export const maxCapturedOutputBytes = 32 * 1024 * 1024;
3
4
  /** Resolves only after the child closes. Cancellation terminates its process group. */
4
5
  export function command(executable, args, options) {
5
6
  options.signal?.throwIfAborted();
@@ -23,10 +24,14 @@ export function command(executable, args, options) {
23
24
  }
24
25
  catch (error) {
25
26
  if (error.code !== "ESRCH")
26
- failure = error;
27
+ failure = failure
28
+ ? new AggregateError([failure, error], `${String(failure)}; process group termination failed: ${String(error)}`)
29
+ : error;
27
30
  }
28
31
  };
29
32
  const stop = () => {
33
+ if (cancelled)
34
+ return;
30
35
  cancelled = true;
31
36
  kill("SIGTERM");
32
37
  killTimer ??= setTimeout(() => kill("SIGKILL"), options.killGraceMs ?? 2000);
@@ -43,27 +48,51 @@ export function command(executable, args, options) {
43
48
  }
44
49
  const timeout = setTimeout(stop, options.timeoutMs ?? 300_000);
45
50
  options.signal?.addEventListener("abort", stop, { once: true });
46
- const collect = (target, chunk) => {
47
- const value = chunk.toString();
51
+ const decoders = {
52
+ stdout: new TextDecoder("utf-8", {
53
+ fatal: options.strictUtf8,
54
+ ignoreBOM: true,
55
+ }),
56
+ stderr: new TextDecoder("utf-8", {
57
+ fatal: options.strictUtf8,
58
+ ignoreBOM: true,
59
+ }),
60
+ };
61
+ let capturedBytes = 0;
62
+ const decodeOutput = (target, chunk) => {
48
63
  try {
49
- options.onOutput?.(value);
64
+ const value = decoders[target].decode(chunk, {
65
+ stream: chunk !== undefined,
66
+ });
67
+ if (value)
68
+ options.onOutput?.(value);
69
+ if (options.captureOutput !== false) {
70
+ if (target === "stdout")
71
+ stdout += value;
72
+ else
73
+ stderr += value;
74
+ }
50
75
  }
51
76
  catch (error) {
52
- failure = error;
77
+ failure ??= error;
53
78
  stop();
54
79
  }
55
- if (options.captureOutput !== false &&
56
- stdout.length + stderr.length <= 32 * 1024 * 1024) {
57
- if (target === "stdout")
58
- stdout += value;
59
- else
60
- stderr += value;
61
- if (stdout.length + stderr.length > 32 * 1024 * 1024)
80
+ };
81
+ const collect = (target, chunk) => {
82
+ if (options.captureOutput !== false) {
83
+ capturedBytes += chunk.length;
84
+ if (capturedBytes > maxCapturedOutputBytes) {
85
+ failure ??= new Error(`Command output size ${capturedBytes} bytes exceeds capture limit ${maxCapturedOutputBytes} bytes`);
62
86
  stop();
87
+ return;
88
+ }
63
89
  }
90
+ decodeOutput(target, chunk);
64
91
  };
65
92
  child.stdout.on("data", (chunk) => collect("stdout", chunk));
66
93
  child.stderr.on("data", (chunk) => collect("stderr", chunk));
94
+ child.stdout.on("end", () => decodeOutput("stdout"));
95
+ child.stderr.on("end", () => decodeOutput("stderr"));
67
96
  const cleanup = () => {
68
97
  clearTimeout(timeout);
69
98
  if (killTimer)
@@ -25,7 +25,12 @@ export interface InvocationRecord {
25
25
  requested: unknown;
26
26
  effective: unknown;
27
27
  prompt: string;
28
- skills: unknown;
28
+ /** Historical records only. New stages use runtime-managed skills. */
29
+ skills?: unknown;
30
+ taskPrompt?: import("./prompts.js").ResolvedPrompt;
31
+ outputContract?: string;
32
+ evidence?: import("./evidence.js").ChangeEvidence;
33
+ validationError?: string;
29
34
  outcome: string;
30
35
  startedAt: string;
31
36
  finishedAt?: string;
@@ -38,6 +43,10 @@ interface RetryAdmission {
38
43
  branchTemplate: string;
39
44
  }>;
40
45
  }
46
+ interface RecoveryAdmission {
47
+ commandId?: string;
48
+ checkSafety: (run: RunRecord) => Promise<string[]>;
49
+ }
41
50
  /** Public persisted query surface. All queries work without a running executor. */
42
51
  export declare class Store {
43
52
  readonly scope: string;
@@ -58,11 +67,21 @@ export declare class Store {
58
67
  blocked?: string | null;
59
68
  }): Promise<void>;
60
69
  insertRun(run: RunRecord): Promise<boolean>;
70
+ blockProject(id: string, reason: string): Promise<void>;
61
71
  /**
62
72
  * Admit a retry and its events atomically. Safety checks run under the project
63
73
  * lock only for new admissions; they must not write through this Store.
64
74
  */
65
75
  admitRetry(runId: string, admission: RetryAdmission): Promise<string>;
76
+ /** Persist recovery intent before dispatch; the command ID is its durable DBOS ID. */
77
+ admitRecovery(runId: string, admission: RecoveryAdmission): Promise<string>;
78
+ recoveryPlan(runId: string): Promise<{
79
+ eligible: boolean;
80
+ reason: string | null;
81
+ fromStep: string;
82
+ reuses: string;
83
+ checks: string;
84
+ }>;
66
85
  run(id: string): Promise<RunRecord>;
67
86
  runs(): Promise<RunRecord[]>;
68
87
  patchRun(id: string, patch: Partial<RunRecord>): Promise<RunRecord>;
@@ -74,7 +93,7 @@ export declare class Store {
74
93
  after?: number;
75
94
  intervalMs?: number;
76
95
  }): () => void;
77
- request(kind: "pause" | "resume" | "stop" | "retry", target: string): Promise<string>;
96
+ request(kind: "pause" | "resume" | "stop" | "retry" | "recover", target: string): Promise<string>;
78
97
  commands(): Promise<Array<{
79
98
  id: string;
80
99
  kind: string;
package/dist/src/store.js CHANGED
@@ -1,10 +1,11 @@
1
1
  import { randomUUID } from "node:crypto";
2
- import { and, eq, gt, Param, SQL } from "drizzle-orm";
2
+ import { and, eq, gt, isNull, Param, SQL } from "drizzle-orm";
3
3
  import { drizzle } from "drizzle-orm/node-postgres";
4
4
  import { Pool } from "pg";
5
5
  import { tryLock, unlockAll } from "./db/locks.js";
6
6
  import { migrateDatabase } from "./db/migrations.js";
7
7
  import * as tables from "./db/schema.js";
8
+ import { recoveryUnavailable } from "./recovery.js";
8
9
  import { createQueuedRun } from "./run-record.js";
9
10
  import { redactValue } from "./runtime/redaction.js";
10
11
  /** Public persisted query surface. All queries work without a running executor. */
@@ -113,6 +114,16 @@ export class Store {
113
114
  await this.emit(run.id, "run", run);
114
115
  return inserted.length > 0;
115
116
  }
117
+ async blockProject(id, reason) {
118
+ const { projects } = tables;
119
+ const updated = await this.db
120
+ .update(projects)
121
+ .set({ blocked: this.redact(reason) })
122
+ .where(and(eq(projects.scope, this.scope), eq(projects.id, id), isNull(projects.blocked)))
123
+ .returning({ id: projects.id });
124
+ if (updated.length)
125
+ await this.emit(null, "project", { id, blocked: reason });
126
+ }
116
127
  /**
117
128
  * Admit a retry and its events atomically. Safety checks run under the project
118
129
  * lock only for new admissions; they must not write through this Store.
@@ -192,6 +203,104 @@ export class Store {
192
203
  return id;
193
204
  });
194
205
  }
206
+ /** Persist recovery intent before dispatch; the command ID is its durable DBOS ID. */
207
+ async admitRecovery(runId, admission) {
208
+ const original = await this.run(runId);
209
+ const { projects, runs, events } = tables;
210
+ const projectPredicate = and(eq(projects.scope, this.scope), eq(projects.id, original.projectId));
211
+ return this.db.transaction(async (tx) => {
212
+ const [project] = await tx
213
+ .select()
214
+ .from(projects)
215
+ .where(projectPredicate)
216
+ .for("update");
217
+ if (!project)
218
+ throw new Error(`Unknown project: ${original.projectId}`);
219
+ const history = await tx
220
+ .select({ record: runs.record })
221
+ .from(runs)
222
+ .where(and(eq(runs.scope, this.scope), eq(runs.taskKey, original.taskKey)))
223
+ .orderBy(runs.id)
224
+ .for("update");
225
+ const previous = history.find(({ record }) => record.id === runId)?.record;
226
+ if (!previous)
227
+ throw new Error(`Unknown run: ${runId}`);
228
+ if (admission.commandId) {
229
+ const records = await tx
230
+ .select({ record: runs.record })
231
+ .from(runs)
232
+ .where(eq(runs.scope, this.scope));
233
+ const owner = records.find(({ record }) => record.id === admission.commandId ||
234
+ record.executions?.some((item) => item.id === admission.commandId))?.record;
235
+ if (owner) {
236
+ if (owner.id !== runId ||
237
+ !owner.executions?.some((item) => item.id === admission.commandId && item.recoveryOf))
238
+ throw new Error("Recovery command identity belongs to a different execution");
239
+ return admission.commandId;
240
+ }
241
+ }
242
+ const unavailable = recoveryUnavailable(previous);
243
+ if (unavailable)
244
+ throw new Error(unavailable);
245
+ if (history.some(({ record }) => record.attempt > previous.attempt ||
246
+ (record.id !== runId &&
247
+ ["queued", "running"].includes(record.outcome))))
248
+ throw new Error("A newer attempt has superseded this run");
249
+ // Recovery must never remove a block imposed by a different operation.
250
+ if (project.blocked)
251
+ throw new Error(project.blocked);
252
+ const reusedSteps = await admission.checkSafety(previous);
253
+ const source = previous.executions.at(-1);
254
+ const id = admission.commandId ?? randomUUID();
255
+ const now = new Date().toISOString();
256
+ const execution = {
257
+ id,
258
+ recoveryOf: source.id,
259
+ startStep: source.failedStep,
260
+ reusedSteps,
261
+ fingerprint: source.fingerprint,
262
+ createdAt: now,
263
+ outcome: "queued",
264
+ phase: previous.phase,
265
+ recoverySupported: source.recoverySupported,
266
+ };
267
+ const recovered = {
268
+ ...previous,
269
+ outcome: "queued",
270
+ updatedAt: now,
271
+ executions: [...previous.executions, execution],
272
+ };
273
+ delete recovered.error;
274
+ delete recovered.failedStep;
275
+ const persisted = redactValue(recovered, this.redact);
276
+ await tx
277
+ .update(runs)
278
+ .set({ record: persisted })
279
+ .where(and(eq(runs.scope, this.scope), eq(runs.id, runId)));
280
+ await tx.insert(events).values({
281
+ scope: this.scope,
282
+ runId,
283
+ kind: "recovery",
284
+ payload: redactValue(execution, this.redact),
285
+ });
286
+ return id;
287
+ });
288
+ }
289
+ async recoveryPlan(runId) {
290
+ const run = await this.run(runId);
291
+ const reason = recoveryUnavailable(run) ??
292
+ ((await this.runs()).some((item) => item.taskKey === run.taskKey && item.attempt > run.attempt)
293
+ ? "A newer attempt has superseded this run"
294
+ : undefined) ??
295
+ (await this.project(run.projectId)).blocked;
296
+ return {
297
+ eligible: !reason,
298
+ reason: reason ?? null,
299
+ fromStep: run.phase,
300
+ reuses: "Completed steps before the failed publication step, including implementation, validation and commit",
301
+ checks: "Runner verifies checkpoints, configuration, artifacts and checkout before recovery",
302
+ };
303
+ }
195
304
  async run(id) {
196
305
  const { runs } = tables;
197
306
  const [row] = await this.db
@@ -232,6 +341,18 @@ export class Store {
232
341
  // Match JSON serialization: undefined patch fields leave stored fields intact.
233
342
  const persistedChange = JSON.parse(JSON.stringify(redactValue(change, this.redact)));
234
343
  const merged = { ...row.record, ...persistedChange };
344
+ const execution = merged.executions?.at(-1);
345
+ if (execution) {
346
+ execution.outcome = merged.outcome;
347
+ if (merged.outcome === "running" && !execution.startedAt)
348
+ execution.startedAt = change.updatedAt;
349
+ execution.phase = merged.phase;
350
+ execution.error = merged.error;
351
+ execution.failedStep = merged.failedStep;
352
+ if (!execution.finishedAt &&
353
+ !["queued", "running"].includes(merged.outcome))
354
+ execution.finishedAt = change.updatedAt;
355
+ }
235
356
  await tx.update(runs).set({ record: merged }).where(predicate);
236
357
  return merged;
237
358
  });
@@ -0,0 +1,16 @@
1
+ import type { RunRecord } from "../domain.js";
2
+ import type { ProjectState } from "../store.js";
3
+ /** Display eligibility only; the runner still validates command admission. */
4
+ export declare function actionAvailability({ run, project, projectRuns, pending, }: {
5
+ run?: RunRecord;
6
+ project?: ProjectState;
7
+ projectRuns: RunRecord[];
8
+ pending: boolean;
9
+ }): {
10
+ recoveryReason: string | undefined;
11
+ available: {
12
+ stop: boolean;
13
+ retry: boolean;
14
+ recover: boolean;
15
+ };
16
+ };