@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/docs/http-api.md +80 -4
  2. package/package.json +1 -4
  3. package/src/api/__tests__/index.test.ts +443 -32
  4. package/src/api/index.ts +108 -127
  5. package/src/cli/__tests__/analyze-task.test.ts +40 -7
  6. package/src/cli/__tests__/index.test.ts +337 -17
  7. package/src/cli/__tests__/types.test.ts +0 -18
  8. package/src/cli/__tests__/update-pipeline-json.test.ts +19 -9
  9. package/src/cli/analyze-task.ts +3 -14
  10. package/src/cli/index.ts +73 -61
  11. package/src/cli/types.ts +0 -13
  12. package/src/cli/update-pipeline-json.ts +3 -14
  13. package/src/config/__tests__/paths.test.ts +46 -1
  14. package/src/config/__tests__/pipeline-registry.test.ts +767 -0
  15. package/src/config/__tests__/sse-events.test.ts +470 -0
  16. package/src/config/__tests__/statuses.test.ts +41 -0
  17. package/src/config/paths.ts +11 -2
  18. package/src/config/pipeline-registry.ts +258 -0
  19. package/src/config/sse-events.ts +122 -0
  20. package/src/config/statuses.ts +20 -1
  21. package/src/core/__tests__/agent-step.test.ts +115 -0
  22. package/src/core/__tests__/config.test.ts +545 -105
  23. package/src/core/__tests__/core-boot-resolution.test.ts +181 -0
  24. package/src/core/__tests__/job-concurrency.test.ts +260 -0
  25. package/src/core/__tests__/job-submission.test.ts +396 -0
  26. package/src/core/__tests__/job-view.test.ts +1341 -0
  27. package/src/core/__tests__/json-file.test.ts +111 -0
  28. package/src/core/__tests__/lifecycle-policy.test.ts +81 -7
  29. package/src/core/__tests__/logger.test.ts +22 -0
  30. package/src/core/__tests__/orchestrator-no-deprecated-exports.test.ts +41 -0
  31. package/src/core/__tests__/orchestrator.test.ts +615 -2
  32. package/src/core/__tests__/pipeline-runner.test.ts +946 -9
  33. package/src/core/__tests__/redact.test.ts +153 -0
  34. package/src/core/__tests__/runner-liveness.test.ts +910 -0
  35. package/src/core/__tests__/seed-naming.test.ts +57 -0
  36. package/src/core/__tests__/single-derivation.test.ts +159 -0
  37. package/src/core/__tests__/task-runner.test.ts +594 -3
  38. package/src/core/__tests__/task-telemetry.test.ts +241 -0
  39. package/src/core/__tests__/workspace-root-resolution-guard.test.ts +62 -0
  40. package/src/core/agent-step.ts +4 -1
  41. package/src/core/agent-types.ts +5 -4
  42. package/src/core/config.ts +134 -222
  43. package/src/core/control.ts +3 -0
  44. package/src/core/job-concurrency.ts +56 -20
  45. package/src/core/job-submission.ts +133 -0
  46. package/src/core/job-view.ts +473 -0
  47. package/src/core/json-file.ts +45 -0
  48. package/src/core/lifecycle-policy.ts +23 -8
  49. package/src/core/logger.ts +0 -29
  50. package/src/core/orchestrator.ts +200 -74
  51. package/src/core/pipeline-runner.ts +85 -53
  52. package/src/core/redact.ts +40 -0
  53. package/src/core/runner-liveness.ts +280 -0
  54. package/src/core/seed-naming.ts +9 -0
  55. package/src/core/status-writer.ts +27 -33
  56. package/src/core/task-runner.ts +356 -319
  57. package/src/core/task-telemetry.ts +107 -0
  58. package/src/harness/__tests__/subprocess.test.ts +112 -1
  59. package/src/harness/subprocess.ts +55 -16
  60. package/src/llm/__tests__/index.test.ts +684 -33
  61. package/src/llm/index.ts +310 -67
  62. package/src/providers/__tests__/alibaba.test.ts +67 -6
  63. package/src/providers/__tests__/anthropic.test.ts +35 -14
  64. package/src/providers/__tests__/base.test.ts +62 -0
  65. package/src/providers/__tests__/claude-code.test.ts +99 -14
  66. package/src/providers/__tests__/deepseek.test.ts +16 -6
  67. package/src/providers/__tests__/gemini.test.ts +47 -25
  68. package/src/providers/__tests__/moonshot.test.ts +27 -0
  69. package/src/providers/__tests__/openai.test.ts +262 -74
  70. package/src/providers/__tests__/opencode.test.ts +77 -0
  71. package/src/providers/__tests__/request-timeout-contract.test.ts +226 -0
  72. package/src/providers/__tests__/stream-accumulator.test.ts +52 -0
  73. package/src/providers/__tests__/types.test.ts +85 -0
  74. package/src/providers/__tests__/zero-fill-guard.test.ts +62 -0
  75. package/src/providers/__tests__/zhipu.test.ts +27 -14
  76. package/src/providers/alibaba.ts +20 -39
  77. package/src/providers/anthropic.ts +23 -11
  78. package/src/providers/base.ts +19 -0
  79. package/src/providers/claude-code.ts +27 -18
  80. package/src/providers/deepseek.ts +9 -28
  81. package/src/providers/gemini.ts +20 -58
  82. package/src/providers/moonshot.ts +15 -11
  83. package/src/providers/openai.ts +79 -61
  84. package/src/providers/opencode.ts +16 -33
  85. package/src/providers/stream-accumulator.ts +27 -21
  86. package/src/providers/types.ts +29 -4
  87. package/src/providers/zhipu.ts +15 -44
  88. package/src/task-analysis/__tests__/analyzer.test.ts +0 -0
  89. package/src/task-analysis/__tests__/enrichers-schema-deducer.test.ts +34 -2
  90. package/src/task-analysis/__tests__/no-ast-imports.test.ts +52 -0
  91. package/src/task-analysis/__tests__/repository.test.ts +65 -0
  92. package/src/task-analysis/__tests__/types.test.ts +63 -130
  93. package/src/task-analysis/analyzer.ts +91 -0
  94. package/src/task-analysis/enrichers/schema-deducer.ts +13 -2
  95. package/src/task-analysis/index.ts +2 -36
  96. package/src/task-analysis/repository.ts +45 -0
  97. package/src/task-analysis/types.ts +42 -58
  98. package/src/ui/client/__tests__/api.test.ts +143 -1
  99. package/src/ui/client/__tests__/bootstrap.test.ts +178 -62
  100. package/src/ui/client/__tests__/job-adapter.test.ts +203 -2
  101. package/src/ui/client/__tests__/load-state.test.ts +70 -0
  102. package/src/ui/client/__tests__/types.test.ts +66 -3
  103. package/src/ui/client/__tests__/useJobDetailWithUpdates.test.ts +390 -77
  104. package/src/ui/client/__tests__/useJobList.test.ts +198 -23
  105. package/src/ui/client/__tests__/useJobListWithUpdates.test.ts +218 -7
  106. package/src/ui/client/adapters/__tests__/job-adapter.test.ts +186 -0
  107. package/src/ui/client/adapters/job-adapter.ts +41 -16
  108. package/src/ui/client/api.ts +38 -15
  109. package/src/ui/client/bootstrap.ts +19 -14
  110. package/src/ui/client/hooks/useJobDetailWithUpdates.ts +73 -97
  111. package/src/ui/client/hooks/useJobList.ts +26 -31
  112. package/src/ui/client/hooks/useJobListWithUpdates.ts +42 -76
  113. package/src/ui/client/load-state.ts +20 -0
  114. package/src/ui/client/reducers/__tests__/job-events.test.ts +568 -0
  115. package/src/ui/client/reducers/job-events.ts +137 -0
  116. package/src/ui/client/types.ts +16 -20
  117. package/src/ui/components/DAGGrid.tsx +6 -1
  118. package/src/ui/components/JobDetail.tsx +12 -4
  119. package/src/ui/components/JobTable.tsx +41 -13
  120. package/src/ui/components/StageTimeline.tsx +8 -20
  121. package/src/ui/components/TaskAnalysisDisplay.tsx +48 -6
  122. package/src/ui/components/__tests__/DAGGrid.test.tsx +36 -0
  123. package/src/ui/components/__tests__/JobDetail.test.tsx +112 -0
  124. package/src/ui/components/__tests__/JobTable.test.tsx +137 -1
  125. package/src/ui/components/__tests__/StageTimeline.test.tsx +5 -6
  126. package/src/ui/components/__tests__/TaskAnalysisDisplay.test.tsx +64 -0
  127. package/src/ui/components/types.ts +35 -15
  128. package/src/ui/dist/assets/{index--RH3sAt3.js → index-DN3-zvtP.js} +4324 -263
  129. package/src/ui/dist/assets/index-DN3-zvtP.js.map +1 -0
  130. package/src/ui/dist/assets/style-CtZBnjlR.css +2 -0
  131. package/src/ui/dist/index.html +2 -2
  132. package/src/ui/pages/PipelineDetail.tsx +60 -4
  133. package/src/ui/pages/PromptPipelineDashboard.tsx +59 -7
  134. package/src/ui/pages/__tests__/PromptPipelineDashboard.test.tsx +114 -32
  135. package/src/ui/pages/__tests__/pages.test.tsx +236 -42
  136. package/src/ui/server/__tests__/concurrency-endpoint.test.ts +54 -0
  137. package/src/ui/server/__tests__/config-bridge-node.test.ts +34 -1
  138. package/src/ui/server/__tests__/config-bridge.test.ts +9 -1
  139. package/src/ui/server/__tests__/embedded-assets.test.ts +66 -0
  140. package/src/ui/server/__tests__/file-endpoints.test.ts +183 -5
  141. package/src/ui/server/__tests__/gate-endpoints.test.ts +55 -0
  142. package/src/ui/server/__tests__/index.test.ts +63 -0
  143. package/src/ui/server/__tests__/job-control-endpoints.test.ts +512 -2
  144. package/src/ui/server/__tests__/job-endpoints.test.ts +158 -2
  145. package/src/ui/server/__tests__/read-static-path-guard.test.ts +94 -0
  146. package/src/ui/server/__tests__/router-root-isolation.test.ts +204 -0
  147. package/src/ui/server/__tests__/router-threading.test.ts +148 -0
  148. package/src/ui/server/__tests__/router.test.ts +104 -0
  149. package/src/ui/server/__tests__/sse-broadcast.test.ts +44 -0
  150. package/src/ui/server/__tests__/sse-enhancer.test.ts +70 -0
  151. package/src/ui/server/config-bridge-node.ts +8 -26
  152. package/src/ui/server/config-bridge.ts +3 -4
  153. package/src/ui/server/embedded-assets-imports.d.ts +44 -0
  154. package/src/ui/server/embedded-assets.ts +13 -2
  155. package/src/ui/server/endpoints/__tests__/meta-endpoint.test.ts +1 -0
  156. package/src/ui/server/endpoints/__tests__/pipeline-analysis-endpoint.test.ts +167 -0
  157. package/src/ui/server/endpoints/__tests__/schema-file-endpoint.test.ts +104 -0
  158. package/src/ui/server/endpoints/__tests__/task-analysis-endpoint.test.ts +103 -0
  159. package/src/ui/server/endpoints/__tests__/upload-endpoints.test.ts +162 -96
  160. package/src/ui/server/endpoints/concurrency-endpoint.ts +2 -1
  161. package/src/ui/server/endpoints/create-pipeline-endpoint.ts +14 -29
  162. package/src/ui/server/endpoints/file-endpoints.ts +15 -10
  163. package/src/ui/server/endpoints/gate-endpoints.ts +2 -6
  164. package/src/ui/server/endpoints/job-control-endpoints.ts +129 -141
  165. package/src/ui/server/endpoints/job-endpoints.ts +7 -0
  166. package/src/ui/server/endpoints/meta-endpoint.ts +1 -1
  167. package/src/ui/server/endpoints/pipeline-analysis-endpoint.ts +99 -11
  168. package/src/ui/server/endpoints/schema-file-endpoint.ts +11 -3
  169. package/src/ui/server/endpoints/task-analysis-endpoint.ts +21 -5
  170. package/src/ui/server/endpoints/task-save-endpoint.ts +1 -2
  171. package/src/ui/server/endpoints/upload-endpoints.ts +5 -40
  172. package/src/ui/server/index.ts +19 -14
  173. package/src/ui/server/job-reader.ts +14 -2
  174. package/src/ui/server/router.ts +33 -10
  175. package/src/ui/server/sse-broadcast.ts +14 -3
  176. package/src/ui/server/sse-enhancer.ts +12 -2
  177. package/src/ui/state/__tests__/schema-loader.test.ts +11 -1
  178. package/src/ui/state/__tests__/snapshot.test.ts +120 -14
  179. package/src/ui/state/__tests__/types.test.ts +104 -5
  180. package/src/ui/state/snapshot.ts +2 -2
  181. package/src/ui/state/transformers/__tests__/list-transformer.test.ts +85 -2
  182. package/src/ui/state/transformers/__tests__/status-transformer.test.ts +250 -22
  183. package/src/ui/state/transformers/list-transformer.ts +13 -3
  184. package/src/ui/state/transformers/status-transformer.ts +36 -170
  185. package/src/ui/state/types.ts +15 -48
  186. package/src/utils/__tests__/path-containment.test.ts +160 -0
  187. package/src/{ui/server/utils → utils}/path-containment.ts +21 -0
  188. package/src/task-analysis/__tests__/enrichers-analysis-writer.test.ts +0 -86
  189. package/src/task-analysis/__tests__/enrichers-artifact-resolver.test.ts +0 -124
  190. package/src/task-analysis/__tests__/extractors-artifacts.test.ts +0 -133
  191. package/src/task-analysis/__tests__/extractors-llm-calls.test.ts +0 -46
  192. package/src/task-analysis/__tests__/extractors-stages.test.ts +0 -52
  193. package/src/task-analysis/__tests__/index.test.ts +0 -143
  194. package/src/task-analysis/__tests__/parser.test.ts +0 -41
  195. package/src/task-analysis/__tests__/utils-ast.test.ts +0 -82
  196. package/src/task-analysis/enrichers/analysis-writer.ts +0 -75
  197. package/src/task-analysis/enrichers/artifact-resolver.ts +0 -89
  198. package/src/task-analysis/extractors/artifacts.ts +0 -143
  199. package/src/task-analysis/extractors/llm-calls.ts +0 -117
  200. package/src/task-analysis/extractors/stages.ts +0 -45
  201. package/src/task-analysis/parser.ts +0 -20
  202. package/src/task-analysis/utils/ast.ts +0 -45
  203. package/src/ui/dist/assets/index--RH3sAt3.js.map +0 -1
  204. package/src/ui/dist/assets/style-CSSKuMOe.css +0 -2
  205. package/src/ui/embedded-assets.js +0 -12
  206. package/src/ui/server/__tests__/path-containment.test.ts +0 -54
@@ -1,19 +1,25 @@
1
- import { describe, test, expect } from "bun:test";
1
+ import { describe, test, expect, afterEach } from "bun:test";
2
+ import type { Subprocess } from "bun";
2
3
  import { mkdtemp, mkdir, writeFile, utimes, rm, readdir, readFile } from "node:fs/promises";
3
- import { existsSync } from "node:fs";
4
+ import { existsSync, mkdtempSync, rmSync } from "node:fs";
4
5
  import { tmpdir } from "node:os";
5
6
  import { join } from "node:path";
6
7
  import {
8
+ adoptOrphanedRunners,
7
9
  drainPendingQueue,
8
10
  handleChildExit,
9
11
  resolveDirs,
12
+ startOrchestrator,
10
13
  promoteSeedFile,
11
14
  restoreSeedFile,
12
15
  } from "../orchestrator";
16
+ import type { ChildHandle } from "../orchestrator";
13
17
  import {
14
18
  getConcurrencyRuntimePaths,
15
19
  getJobConcurrencyStatus,
16
20
  } from "../job-concurrency";
21
+ import { isProcessAlive } from "../runner-liveness";
22
+ import { resetConfig } from "../config";
17
23
 
18
24
  async function setupDir(prefix: string): Promise<string> {
19
25
  const dir = await mkdtemp(join(tmpdir(), prefix));
@@ -131,6 +137,7 @@ describe("drainPendingQueue", () => {
131
137
  dataDir: dir,
132
138
  maxConcurrentJobs: 2,
133
139
  lockTimeoutMs: 1000,
140
+ staleLeaseTimeoutMs: 1000,
134
141
  spawnRunner: fakeSpawnRunner(pids),
135
142
  });
136
143
  expect(result.promoted).toEqual(["job-a", "job-b"]);
@@ -154,6 +161,7 @@ describe("drainPendingQueue", () => {
154
161
  dataDir: dir,
155
162
  maxConcurrentJobs: 0,
156
163
  lockTimeoutMs: 1000,
164
+ staleLeaseTimeoutMs: 1000,
157
165
  spawnRunner: fakeSpawnRunner(pids),
158
166
  });
159
167
  expect(result.promoted).toEqual([]);
@@ -181,6 +189,7 @@ describe("drainPendingQueue", () => {
181
189
  dataDir: dir,
182
190
  maxConcurrentJobs: 5,
183
191
  lockTimeoutMs: 1000,
192
+ staleLeaseTimeoutMs: 1000,
184
193
  spawnRunner: fakeSpawnRunner(pids),
185
194
  });
186
195
  expect(result.promoted).toEqual(["job-fresh"]);
@@ -206,6 +215,7 @@ describe("drainPendingQueue", () => {
206
215
  dataDir: dir,
207
216
  maxConcurrentJobs: 5,
208
217
  lockTimeoutMs: 1000,
218
+ staleLeaseTimeoutMs: 1000,
209
219
  spawnRunner: fakeSpawnRunner(pids),
210
220
  });
211
221
  expect(result.promoted).toEqual(["job-staged"]);
@@ -232,6 +242,7 @@ describe("drainPendingQueue", () => {
232
242
  dataDir: dir,
233
243
  maxConcurrentJobs: 5,
234
244
  lockTimeoutMs: 1000,
245
+ staleLeaseTimeoutMs: 1000,
235
246
  spawnRunner: fakeSpawnRunner(pids),
236
247
  });
237
248
  expect(result.promoted).toEqual(["job-heal"]);
@@ -255,6 +266,7 @@ describe("drainPendingQueue", () => {
255
266
  dataDir: dir,
256
267
  maxConcurrentJobs: 5,
257
268
  lockTimeoutMs: 1000,
269
+ staleLeaseTimeoutMs: 1000,
258
270
  spawnRunner: fakeSpawnRunner(pids),
259
271
  });
260
272
  expect(result.promoted).toEqual(["job-a", "job-b", "job-c"]);
@@ -274,6 +286,7 @@ describe("drainPendingQueue", () => {
274
286
  dataDir: dir,
275
287
  maxConcurrentJobs: 2,
276
288
  lockTimeoutMs: 1000,
289
+ staleLeaseTimeoutMs: 1000,
277
290
  spawnRunner: failingSpawn,
278
291
  });
279
292
  expect(result.promoted).toEqual([]);
@@ -300,6 +313,7 @@ describe("drainPendingQueue", () => {
300
313
  dataDir: dir,
301
314
  maxConcurrentJobs: 2,
302
315
  lockTimeoutMs: 1000,
316
+ staleLeaseTimeoutMs: 1000,
303
317
  spawnRunner: failingSpawn,
304
318
  });
305
319
  }
@@ -328,6 +342,7 @@ describe("drainPendingQueue", () => {
328
342
  dataDir: dir,
329
343
  maxConcurrentJobs: 2,
330
344
  lockTimeoutMs: 1000,
345
+ staleLeaseTimeoutMs: 1000,
331
346
  spawnRunner: failingSpawn,
332
347
  });
333
348
  }
@@ -354,6 +369,7 @@ describe("drainPendingQueue", () => {
354
369
  dataDir: dir,
355
370
  maxConcurrentJobs: 2,
356
371
  lockTimeoutMs: 1000,
372
+ staleLeaseTimeoutMs: 1000,
357
373
  spawnRunner: fakeSpawnRunner(new Map()),
358
374
  });
359
375
  }
@@ -377,6 +393,7 @@ describe("drainPendingQueue", () => {
377
393
  dataDir: dir,
378
394
  maxConcurrentJobs: 2,
379
395
  lockTimeoutMs: 10,
396
+ staleLeaseTimeoutMs: 10,
380
397
  spawnRunner: async () => {
381
398
  await mkdir(lockDir, { recursive: true });
382
399
  await writeFile(join(lockDir, "owner.json"), JSON.stringify({ pid: process.pid, acquiredAt: new Date().toISOString() }));
@@ -412,6 +429,7 @@ describe("handleChildExit", () => {
412
429
  dataDir: dir,
413
430
  maxConcurrentJobs: 1,
414
431
  lockTimeoutMs: 1000,
432
+ staleLeaseTimeoutMs: 1000,
415
433
  spawnRunner: spawn,
416
434
  });
417
435
  expect(first.promoted).toEqual(["job-a"]);
@@ -423,6 +441,7 @@ describe("handleChildExit", () => {
423
441
  dataDir: dir,
424
442
  maxConcurrentJobs: 1,
425
443
  lockTimeoutMs: 1000,
444
+ staleLeaseTimeoutMs: 1000,
426
445
  spawnRunner: spawn,
427
446
  }).then((r) => {
428
447
  drainResult = r;
@@ -463,6 +482,146 @@ describe("handleChildExit", () => {
463
482
  await rm(dir, { recursive: true });
464
483
  }
465
484
  });
485
+
486
+ test("handleChildExit still attempts reconciliation when slot release fails", async () => {
487
+ const dir = await setupDir("exit-release-fails-");
488
+ try {
489
+ const releaseError = new Error("release failed");
490
+ let drained = false;
491
+ let reconciled = false;
492
+
493
+ await expect(handleChildExit({
494
+ dataDir: dir,
495
+ jobId: "job-release-fails",
496
+ triggerDrain: () => {
497
+ drained = true;
498
+ },
499
+ lockTimeoutMs: 1000,
500
+ releaseSlot: async () => {
501
+ throw releaseError;
502
+ },
503
+ reconcileJob: async () => {
504
+ reconciled = true;
505
+ return { reconciled: false, reason: "unreadable" };
506
+ },
507
+ })).rejects.toThrow("release failed");
508
+
509
+ expect(reconciled).toBe(true);
510
+ expect(drained).toBe(true);
511
+ } finally {
512
+ await rm(dir, { recursive: true });
513
+ }
514
+ });
515
+
516
+ test("handleChildExit reports both cleanup errors when slot release and reconciliation fail", async () => {
517
+ const dir = await setupDir("exit-both-fail-");
518
+ try {
519
+ const releaseError = new Error("release failed");
520
+ const reconcileError = new Error("reconcile failed");
521
+ let drained = false;
522
+
523
+ let thrown: unknown;
524
+ try {
525
+ await handleChildExit({
526
+ dataDir: dir,
527
+ jobId: "job-both-fail",
528
+ triggerDrain: () => {
529
+ drained = true;
530
+ },
531
+ lockTimeoutMs: 1000,
532
+ releaseSlot: async () => {
533
+ throw releaseError;
534
+ },
535
+ reconcileJob: async () => {
536
+ throw reconcileError;
537
+ },
538
+ });
539
+ } catch (err) {
540
+ thrown = err;
541
+ }
542
+
543
+ expect(thrown).toBeInstanceOf(AggregateError);
544
+ expect((thrown as AggregateError).message).toBe("child exit cleanup failed");
545
+ expect((thrown as AggregateError).errors).toEqual([releaseError, reconcileError]);
546
+ expect(drained).toBe(true);
547
+ } finally {
548
+ await rm(dir, { recursive: true });
549
+ }
550
+ });
551
+
552
+ test("AC-3: a mid-job runner exit marks the running task and job failed, releases the slot, and triggers a drain", async () => {
553
+ const dir = await setupDir("exit-reconcile-");
554
+ try {
555
+ const jobId = "job-running";
556
+ const jobDir = join(dir, "current", jobId);
557
+ await mkdir(jobDir, { recursive: true });
558
+ await writeFile(
559
+ join(jobDir, "tasks-status.json"),
560
+ JSON.stringify({
561
+ id: jobId,
562
+ state: "running",
563
+ current: "research",
564
+ currentStage: null,
565
+ lastUpdated: "2026-06-19T11:00:00.000Z",
566
+ tasks: { research: { state: "running" } },
567
+ files: { artifacts: [], logs: [], tmp: [] },
568
+ }),
569
+ );
570
+ await writeFile(join(jobDir, "runner.pid"), "999999");
571
+
572
+ let drained = false;
573
+ await handleChildExit({
574
+ dataDir: dir,
575
+ jobId,
576
+ triggerDrain: () => {
577
+ drained = true;
578
+ },
579
+ code: 1,
580
+ signal: null,
581
+ lockTimeoutMs: 1000,
582
+ });
583
+
584
+ expect(drained).toBe(true);
585
+ const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
586
+ expect(existsSync(join(runningJobsDir, `${jobId}.json`))).toBe(false);
587
+
588
+ const snapshot = JSON.parse(await readFile(join(jobDir, "tasks-status.json"), "utf-8")) as Record<string, unknown>;
589
+ expect(snapshot["state"]).toBe("failed");
590
+ const tasks = snapshot["tasks"] as Record<string, Record<string, unknown>>;
591
+ expect(tasks["research"]?.["state"]).toBe("failed");
592
+ expect(tasks["research"]?.["error"]).toBeTruthy();
593
+ const errorContext = tasks["research"]?.["errorContext"] as Record<string, unknown>;
594
+ expect(errorContext?.["kind"]).toBe("runner-interrupted");
595
+ expect(errorContext?.["code"]).toBe(1);
596
+ } finally {
597
+ await rm(dir, { recursive: true });
598
+ }
599
+ });
600
+
601
+ test("AC-10: a job already moved to complete/ (no current/ status) is untouched, and the drain still fires", async () => {
602
+ const dir = await setupDir("exit-complete-");
603
+ try {
604
+ const jobId = "job-gone";
605
+ const statusPath = join(dir, "current", jobId, "tasks-status.json");
606
+
607
+ let drainCount = 0;
608
+ await handleChildExit({
609
+ dataDir: dir,
610
+ jobId,
611
+ triggerDrain: () => {
612
+ drainCount++;
613
+ },
614
+ code: 0,
615
+ signal: null,
616
+ lockTimeoutMs: 1000,
617
+ });
618
+
619
+ expect(drainCount).toBe(1);
620
+ expect(existsSync(statusPath)).toBe(false);
621
+ } finally {
622
+ await rm(dir, { recursive: true });
623
+ }
624
+ });
466
625
  });
467
626
 
468
627
  describe("drainPendingQueue concurrency", () => {
@@ -481,12 +640,14 @@ describe("drainPendingQueue concurrency", () => {
481
640
  dataDir: dir,
482
641
  maxConcurrentJobs: 2,
483
642
  lockTimeoutMs: 1000,
643
+ staleLeaseTimeoutMs: 1000,
484
644
  spawnRunner: spawn,
485
645
  }),
486
646
  drainPendingQueue({
487
647
  dataDir: dir,
488
648
  maxConcurrentJobs: 2,
489
649
  lockTimeoutMs: 1000,
650
+ staleLeaseTimeoutMs: 1000,
490
651
  spawnRunner: spawn,
491
652
  }),
492
653
  ]);
@@ -512,3 +673,455 @@ describe("drainPendingQueue concurrency", () => {
512
673
  }
513
674
  });
514
675
  });
676
+
677
+ describe("drainPendingQueue stale-lease age (AC-5)", () => {
678
+ test("does not prune a fresh null-pid in-flight lease when lockTimeoutMs is short but staleLeaseTimeoutMs is long", async () => {
679
+ const dir = await setupDir("drain-stale-age-");
680
+ try {
681
+ await writeLease(
682
+ dir,
683
+ "job-held",
684
+ null,
685
+ new Date(Date.now() - 1000).toISOString(),
686
+ );
687
+ await writeSeed(dir, "job-queued", { pipeline: "p" }, 1700000000);
688
+
689
+ const result = await drainPendingQueue({
690
+ dataDir: dir,
691
+ maxConcurrentJobs: 1,
692
+ lockTimeoutMs: 50,
693
+ staleLeaseTimeoutMs: 30000,
694
+ spawnRunner: fakeSpawnRunner(new Map()),
695
+ });
696
+
697
+ expect(result.promoted).toEqual([]);
698
+ const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
699
+ expect(existsSync(join(runningJobsDir, "job-held.json"))).toBe(true);
700
+ } finally {
701
+ await rm(dir, { recursive: true });
702
+ }
703
+ });
704
+ });
705
+
706
+ interface FakeWatcher {
707
+ on(event: "add", cb: (path: string) => void): FakeWatcher;
708
+ on(event: "ready", cb: () => void): FakeWatcher;
709
+ on(event: "error", cb: (err: Error) => void): FakeWatcher;
710
+ close(): Promise<void>;
711
+ }
712
+
713
+ function makeFakeWatcherFactory() {
714
+ return (_path: string, _options: Record<string, unknown>): FakeWatcher => {
715
+ const watcher = {
716
+ on(event: "add" | "ready" | "error", cb: () => void): FakeWatcher {
717
+ if (event === "ready") queueMicrotask(() => cb());
718
+ return watcher;
719
+ },
720
+ close(): Promise<void> {
721
+ return Promise.resolve();
722
+ },
723
+ } as unknown as FakeWatcher;
724
+ return watcher;
725
+ };
726
+ }
727
+
728
+ async function writeRunningJob(
729
+ dataDir: string,
730
+ jobId: string,
731
+ lastUpdated: string,
732
+ runnerPid: number,
733
+ ): Promise<void> {
734
+ const jobDir = join(dataDir, "current", jobId);
735
+ await mkdir(jobDir, { recursive: true });
736
+ await writeFile(
737
+ join(jobDir, "tasks-status.json"),
738
+ JSON.stringify({
739
+ id: jobId,
740
+ state: "running",
741
+ current: "research",
742
+ currentStage: null,
743
+ lastUpdated,
744
+ tasks: { research: { state: "running" } },
745
+ files: { artifacts: [], logs: [], tmp: [] },
746
+ }),
747
+ );
748
+ await writeFile(join(jobDir, "runner.pid"), String(runnerPid));
749
+ }
750
+
751
+ async function writeLease(
752
+ dataDir: string,
753
+ jobId: string,
754
+ pid: number | null,
755
+ acquiredAt: string,
756
+ ): Promise<string> {
757
+ const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
758
+ await mkdir(runningJobsDir, { recursive: true });
759
+ const slotPath = join(runningJobsDir, `${jobId}.json`);
760
+ await writeFile(slotPath, JSON.stringify({
761
+ jobId,
762
+ pid,
763
+ acquiredAt,
764
+ source: "orchestrator",
765
+ slotPath,
766
+ }));
767
+ return slotPath;
768
+ }
769
+
770
+ describe("startOrchestrator boot sweep (AC-16)", () => {
771
+ afterEach(() => {
772
+ resetConfig();
773
+ });
774
+
775
+ test("a stale orphaned job in current/ is marked failed and the handle still resolves", async () => {
776
+ const root = await mkdtemp(join(tmpdir(), "boot-stale-"));
777
+ const dataDir = join(root, "pipeline-data");
778
+ try {
779
+ await writeRunningJob(dataDir, "job-stale", "2020-01-01T00:00:00.000Z", 999999);
780
+
781
+ const handle = await startOrchestrator({
782
+ dataDir,
783
+ watcherFactory: makeFakeWatcherFactory(),
784
+ });
785
+
786
+ expect(handle).toBeDefined();
787
+ expect(typeof handle.stop).toBe("function");
788
+
789
+ const snapshot = JSON.parse(
790
+ await readFile(join(dataDir, "current", "job-stale", "tasks-status.json"), "utf-8"),
791
+ ) as Record<string, unknown>;
792
+ expect(snapshot["state"]).toBe("failed");
793
+ const tasks = snapshot["tasks"] as Record<string, Record<string, unknown>>;
794
+ expect(tasks["research"]?.["state"]).toBe("failed");
795
+
796
+ await handle.stop();
797
+ } finally {
798
+ await rm(root, { recursive: true, force: true });
799
+ }
800
+ });
801
+
802
+ test("a live job in current/ is left running and the handle still resolves", async () => {
803
+ const root = await mkdtemp(join(tmpdir(), "boot-live-"));
804
+ const dataDir = join(root, "pipeline-data");
805
+ try {
806
+ await writeRunningJob(dataDir, "job-live", "2020-01-01T00:00:00.000Z", process.pid);
807
+
808
+ const handle = await startOrchestrator({
809
+ dataDir,
810
+ watcherFactory: makeFakeWatcherFactory(),
811
+ });
812
+
813
+ expect(handle).toBeDefined();
814
+ expect(typeof handle.stop).toBe("function");
815
+
816
+ const snapshot = JSON.parse(
817
+ await readFile(join(dataDir, "current", "job-live", "tasks-status.json"), "utf-8"),
818
+ ) as Record<string, unknown>;
819
+ expect(snapshot["state"]).toBe("running");
820
+ const tasks = snapshot["tasks"] as Record<string, Record<string, unknown>>;
821
+ expect(tasks["research"]?.["state"]).toBe("running");
822
+
823
+ await handle.stop();
824
+ } finally {
825
+ await rm(root, { recursive: true, force: true });
826
+ }
827
+ });
828
+
829
+ test("AC-13: threads staleLeaseTimeoutMs into boot reconciliation so a mid-aged in-flight lease is not abandoned", async () => {
830
+ const root = await mkdtemp(join(tmpdir(), "boot-stale-age-"));
831
+ const dataDir = join(root, "pipeline-data");
832
+ const previous = process.env["PO_STALE_LEASE_TIMEOUT_MS"];
833
+ process.env["PO_STALE_LEASE_TIMEOUT_MS"] = "60000";
834
+ resetConfig();
835
+ try {
836
+ await writeRunningJob(dataDir, "job-inflight", "2020-01-01T00:00:00.000Z", 999999);
837
+ await writeLease(
838
+ dataDir,
839
+ "job-inflight",
840
+ null,
841
+ new Date(Date.now() - 40000).toISOString(),
842
+ );
843
+
844
+ const handle = await startOrchestrator({
845
+ dataDir,
846
+ watcherFactory: makeFakeWatcherFactory(),
847
+ });
848
+
849
+ const snapshot = JSON.parse(
850
+ await readFile(join(dataDir, "current", "job-inflight", "tasks-status.json"), "utf-8"),
851
+ ) as Record<string, unknown>;
852
+ expect(snapshot["state"]).toBe("running");
853
+
854
+ await handle.stop();
855
+ } finally {
856
+ if (previous === undefined) delete process.env["PO_STALE_LEASE_TIMEOUT_MS"];
857
+ else process.env["PO_STALE_LEASE_TIMEOUT_MS"] = previous;
858
+ resetConfig();
859
+ await rm(root, { recursive: true, force: true });
860
+ }
861
+ });
862
+
863
+ test("AC-2: a stale dead_pid lease in runtime/running-jobs/ is pruned at boot", async () => {
864
+ const root = await mkdtemp(join(tmpdir(), "boot-prune-dead-"));
865
+ const dataDir = join(root, "pipeline-data");
866
+ try {
867
+ await writeRunningJob(dataDir, "job-dead", "2020-01-01T00:00:00.000Z", 999999);
868
+ await writeLease(dataDir, "job-dead", 999999, "2020-01-01T00:00:00.000Z");
869
+
870
+ const handle = await startOrchestrator({
871
+ dataDir,
872
+ watcherFactory: makeFakeWatcherFactory(),
873
+ });
874
+
875
+ const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
876
+ expect(existsSync(join(runningJobsDir, "job-dead.json"))).toBe(false);
877
+
878
+ await handle.stop();
879
+ } finally {
880
+ await rm(root, { recursive: true, force: true });
881
+ }
882
+ });
883
+
884
+ test("AC-2: a live-pid lease with matching runner.pid is retained at boot", async () => {
885
+ const root = await mkdtemp(join(tmpdir(), "boot-prune-live-"));
886
+ const dataDir = join(root, "pipeline-data");
887
+ // Use a real disposable child for the "live pid" so re-adoption + handle.stop()
888
+ // (which SIGTERMs the adopted handle) targets the child, never the test runner.
889
+ const child = spawnSleeperChild();
890
+ try {
891
+ await writeRunningJob(dataDir, "job-live", "2020-01-01T00:00:00.000Z", child.pid!);
892
+ await writeLease(dataDir, "job-live", child.pid!, "2020-01-01T00:00:00.000Z");
893
+
894
+ const handle = await startOrchestrator({
895
+ dataDir,
896
+ watcherFactory: makeFakeWatcherFactory(),
897
+ });
898
+
899
+ const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
900
+ expect(existsSync(join(runningJobsDir, "job-live.json"))).toBe(true);
901
+
902
+ await handle.stop();
903
+ } finally {
904
+ await teardownChild(child);
905
+ await rm(root, { recursive: true, force: true });
906
+ }
907
+ });
908
+
909
+ test("AC-7: with PO_MAX_RUNNING_JOBS=foo, startOrchestrator throws synchronously before creating pipeline-data directories", () => {
910
+ const root = mkdtempSync(join(tmpdir(), "boot-bad-env-"));
911
+ const dataDir = join(root, "pipeline-data");
912
+ const previous = process.env["PO_MAX_RUNNING_JOBS"];
913
+ process.env["PO_MAX_RUNNING_JOBS"] = "foo";
914
+ resetConfig();
915
+ try {
916
+ expect(() =>
917
+ startOrchestrator({
918
+ dataDir,
919
+ watcherFactory: makeFakeWatcherFactory(),
920
+ }),
921
+ ).toThrow(/PO_MAX_RUNNING_JOBS/);
922
+
923
+ expect(existsSync(join(dataDir, "pending"))).toBe(false);
924
+ expect(existsSync(join(dataDir, "current"))).toBe(false);
925
+ expect(existsSync(join(dataDir, "complete"))).toBe(false);
926
+ expect(existsSync(join(dataDir, "rejected"))).toBe(false);
927
+ expect(existsSync(join(dataDir, "staging"))).toBe(false);
928
+ } finally {
929
+ if (previous === undefined) delete process.env["PO_MAX_RUNNING_JOBS"];
930
+ else process.env["PO_MAX_RUNNING_JOBS"] = previous;
931
+ resetConfig();
932
+ rmSync(root, { recursive: true, force: true });
933
+ }
934
+ });
935
+ });
936
+
937
+ function spawnSleeperChild(): Subprocess<"ignore", "ignore", "ignore"> {
938
+ return Bun.spawn(["sleep", "30"], {
939
+ stdin: "ignore",
940
+ stdout: "ignore",
941
+ stderr: "ignore",
942
+ });
943
+ }
944
+
945
+ async function teardownChild(child: Subprocess<"ignore", "ignore", "ignore">): Promise<void> {
946
+ try {
947
+ child.kill(9);
948
+ } catch {
949
+ // already gone
950
+ }
951
+ await child.exited.catch(() => undefined);
952
+ }
953
+
954
+ async function writeRunnerPid(dataDir: string, jobId: string, pid: number): Promise<void> {
955
+ const jobDir = join(dataDir, "current", jobId);
956
+ await mkdir(jobDir, { recursive: true });
957
+ await writeFile(join(jobDir, "runner.pid"), String(pid));
958
+ }
959
+
960
+ describe("adoptOrphanedRunners (AC-4, AC-10, AC-11, AC-12)", () => {
961
+ test("AC-4: a live lease whose runner.pid matches is adopted, and kill(15) delivers SIGTERM to the child", async () => {
962
+ const dir = await setupDir("adopt-live-");
963
+ const child = spawnSleeperChild();
964
+ try {
965
+ const jobId = "job-adopt";
966
+ await writeRunnerPid(dir, jobId, child.pid!);
967
+ await writeLease(dir, jobId, child.pid!, new Date().toISOString());
968
+
969
+ const running = new Map<string, ChildHandle>();
970
+ const adopted = await adoptOrphanedRunners({
971
+ dataDir: dir,
972
+ currentDir: join(dir, "current"),
973
+ running,
974
+ });
975
+
976
+ expect(adopted).toEqual([jobId]);
977
+ expect(running.has(jobId)).toBe(true);
978
+
979
+ const handle = running.get(jobId)!;
980
+ handle.kill(15);
981
+
982
+ const result = await Promise.race([
983
+ child.exited.then(() => "exited" as const),
984
+ new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 3000)),
985
+ ]);
986
+ expect(result).toBe("exited");
987
+ } finally {
988
+ await teardownChild(child);
989
+ await rm(dir, { recursive: true, force: true });
990
+ }
991
+ });
992
+
993
+ test("AC-10: a lease with a dead pid is not adopted", async () => {
994
+ const dir = await setupDir("adopt-dead-");
995
+ const child = spawnSleeperChild();
996
+ try {
997
+ const deadPid = 999999;
998
+ await writeRunnerPid(dir, "job-dead", deadPid);
999
+ await writeLease(dir, "job-dead", deadPid, new Date().toISOString());
1000
+
1001
+ const running = new Map<string, ChildHandle>();
1002
+ const adopted = await adoptOrphanedRunners({
1003
+ dataDir: dir,
1004
+ currentDir: join(dir, "current"),
1005
+ running,
1006
+ });
1007
+
1008
+ expect(adopted).toEqual([]);
1009
+ expect(running.size).toBe(0);
1010
+ } finally {
1011
+ await teardownChild(child);
1012
+ await rm(dir, { recursive: true, force: true });
1013
+ }
1014
+ });
1015
+
1016
+ test("AC-10: a live lease with an absent runner.pid is not adopted", async () => {
1017
+ const dir = await setupDir("adopt-nopidfile-");
1018
+ const child = spawnSleeperChild();
1019
+ try {
1020
+ const jobId = "job-nopidfile";
1021
+ await mkdir(join(dir, "current", jobId), { recursive: true });
1022
+ await writeLease(dir, jobId, child.pid!, new Date().toISOString());
1023
+
1024
+ const running = new Map<string, ChildHandle>();
1025
+ const adopted = await adoptOrphanedRunners({
1026
+ dataDir: dir,
1027
+ currentDir: join(dir, "current"),
1028
+ running,
1029
+ });
1030
+
1031
+ expect(adopted).toEqual([]);
1032
+ expect(running.size).toBe(0);
1033
+ } finally {
1034
+ await teardownChild(child);
1035
+ await rm(dir, { recursive: true, force: true });
1036
+ }
1037
+ });
1038
+
1039
+ test("AC-10: a live lease whose runner.pid mismatches is not adopted", async () => {
1040
+ const dir = await setupDir("adopt-mismatch-");
1041
+ const child = spawnSleeperChild();
1042
+ try {
1043
+ const jobId = "job-mismatch";
1044
+ await writeRunnerPid(dir, jobId, child.pid! + 1);
1045
+ await writeLease(dir, jobId, child.pid!, new Date().toISOString());
1046
+
1047
+ const running = new Map<string, ChildHandle>();
1048
+ const adopted = await adoptOrphanedRunners({
1049
+ dataDir: dir,
1050
+ currentDir: join(dir, "current"),
1051
+ running,
1052
+ });
1053
+
1054
+ expect(adopted).toEqual([]);
1055
+ expect(running.size).toBe(0);
1056
+ } finally {
1057
+ await teardownChild(child);
1058
+ await rm(dir, { recursive: true, force: true });
1059
+ }
1060
+ });
1061
+
1062
+ test("AC-11: mutating runner.pid to a different value before kill causes the signal to be skipped", async () => {
1063
+ const dir = await setupDir("adopt-reread-");
1064
+ const child = spawnSleeperChild();
1065
+ try {
1066
+ const jobId = "job-reread";
1067
+ await writeRunnerPid(dir, jobId, child.pid!);
1068
+ await writeLease(dir, jobId, child.pid!, new Date().toISOString());
1069
+
1070
+ const running = new Map<string, ChildHandle>();
1071
+ await adoptOrphanedRunners({
1072
+ dataDir: dir,
1073
+ currentDir: join(dir, "current"),
1074
+ running,
1075
+ });
1076
+
1077
+ // Mutate runner.pid before kill so the re-read guard skips the signal.
1078
+ await writeFile(join(dir, "current", jobId, "runner.pid"), String(child.pid! + 1));
1079
+
1080
+ const handle = running.get(jobId)!;
1081
+ handle.kill(15);
1082
+
1083
+ await new Promise((resolve) => setTimeout(resolve, 300));
1084
+ expect(isProcessAlive(child.pid!)).toBe(true);
1085
+ } finally {
1086
+ await teardownChild(child);
1087
+ await rm(dir, { recursive: true, force: true });
1088
+ }
1089
+ });
1090
+
1091
+ test("AC-12: reading exited starts the poll and resolves with completionType 'signal' after death; getter is lazy (memoized)", async () => {
1092
+ const dir = await setupDir("adopt-exited-");
1093
+ const child = spawnSleeperChild();
1094
+ try {
1095
+ const jobId = "job-exited";
1096
+ await writeRunnerPid(dir, jobId, child.pid!);
1097
+ await writeLease(dir, jobId, child.pid!, new Date().toISOString());
1098
+
1099
+ const running = new Map<string, ChildHandle>();
1100
+ await adoptOrphanedRunners({
1101
+ dataDir: dir,
1102
+ currentDir: join(dir, "current"),
1103
+ running,
1104
+ });
1105
+
1106
+ const handle = running.get(jobId)!;
1107
+
1108
+ // Reading `exited` is what starts the poll; the getter must memoize the promise.
1109
+ const exitedA = handle.exited;
1110
+ const exitedB = handle.exited;
1111
+ expect(exitedA).toBe(exitedB);
1112
+
1113
+ child.kill(9);
1114
+
1115
+ const result = await Promise.race([
1116
+ exitedA,
1117
+ new Promise<{ completionType: string }>((resolve) =>
1118
+ setTimeout(() => resolve({ completionType: "timeout" }), 3000),
1119
+ ),
1120
+ ]);
1121
+ expect(result).toEqual({ code: null, signal: null, completionType: "signal" });
1122
+ } finally {
1123
+ await teardownChild(child);
1124
+ await rm(dir, { recursive: true, force: true });
1125
+ }
1126
+ });
1127
+ });