@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/http-api.md +80 -4
- package/package.json +1 -4
- package/src/api/__tests__/index.test.ts +443 -32
- package/src/api/index.ts +108 -127
- package/src/cli/__tests__/analyze-task.test.ts +40 -7
- package/src/cli/__tests__/index.test.ts +337 -17
- package/src/cli/__tests__/types.test.ts +0 -18
- package/src/cli/__tests__/update-pipeline-json.test.ts +19 -9
- package/src/cli/analyze-task.ts +3 -14
- package/src/cli/index.ts +73 -61
- package/src/cli/types.ts +0 -13
- package/src/cli/update-pipeline-json.ts +3 -14
- package/src/config/__tests__/paths.test.ts +46 -1
- package/src/config/__tests__/pipeline-registry.test.ts +767 -0
- package/src/config/__tests__/sse-events.test.ts +470 -0
- package/src/config/__tests__/statuses.test.ts +41 -0
- package/src/config/paths.ts +11 -2
- package/src/config/pipeline-registry.ts +258 -0
- package/src/config/sse-events.ts +122 -0
- package/src/config/statuses.ts +20 -1
- package/src/core/__tests__/agent-step.test.ts +115 -0
- package/src/core/__tests__/config.test.ts +545 -105
- package/src/core/__tests__/core-boot-resolution.test.ts +181 -0
- package/src/core/__tests__/job-concurrency.test.ts +260 -0
- package/src/core/__tests__/job-submission.test.ts +396 -0
- package/src/core/__tests__/job-view.test.ts +1341 -0
- package/src/core/__tests__/json-file.test.ts +111 -0
- package/src/core/__tests__/lifecycle-policy.test.ts +81 -7
- package/src/core/__tests__/logger.test.ts +22 -0
- package/src/core/__tests__/orchestrator-no-deprecated-exports.test.ts +41 -0
- package/src/core/__tests__/orchestrator.test.ts +615 -2
- package/src/core/__tests__/pipeline-runner.test.ts +946 -9
- package/src/core/__tests__/redact.test.ts +153 -0
- package/src/core/__tests__/runner-liveness.test.ts +910 -0
- package/src/core/__tests__/seed-naming.test.ts +57 -0
- package/src/core/__tests__/single-derivation.test.ts +159 -0
- package/src/core/__tests__/task-runner.test.ts +594 -3
- package/src/core/__tests__/task-telemetry.test.ts +241 -0
- package/src/core/__tests__/workspace-root-resolution-guard.test.ts +62 -0
- package/src/core/agent-step.ts +4 -1
- package/src/core/agent-types.ts +5 -4
- package/src/core/config.ts +134 -222
- package/src/core/control.ts +3 -0
- package/src/core/job-concurrency.ts +56 -20
- package/src/core/job-submission.ts +133 -0
- package/src/core/job-view.ts +473 -0
- package/src/core/json-file.ts +45 -0
- package/src/core/lifecycle-policy.ts +23 -8
- package/src/core/logger.ts +0 -29
- package/src/core/orchestrator.ts +200 -74
- package/src/core/pipeline-runner.ts +85 -53
- package/src/core/redact.ts +40 -0
- package/src/core/runner-liveness.ts +280 -0
- package/src/core/seed-naming.ts +9 -0
- package/src/core/status-writer.ts +27 -33
- package/src/core/task-runner.ts +356 -319
- package/src/core/task-telemetry.ts +107 -0
- package/src/harness/__tests__/subprocess.test.ts +112 -1
- package/src/harness/subprocess.ts +55 -16
- package/src/llm/__tests__/index.test.ts +684 -33
- package/src/llm/index.ts +310 -67
- package/src/providers/__tests__/alibaba.test.ts +67 -6
- package/src/providers/__tests__/anthropic.test.ts +35 -14
- package/src/providers/__tests__/base.test.ts +62 -0
- package/src/providers/__tests__/claude-code.test.ts +99 -14
- package/src/providers/__tests__/deepseek.test.ts +16 -6
- package/src/providers/__tests__/gemini.test.ts +47 -25
- package/src/providers/__tests__/moonshot.test.ts +27 -0
- package/src/providers/__tests__/openai.test.ts +262 -74
- package/src/providers/__tests__/opencode.test.ts +77 -0
- package/src/providers/__tests__/request-timeout-contract.test.ts +226 -0
- package/src/providers/__tests__/stream-accumulator.test.ts +52 -0
- package/src/providers/__tests__/types.test.ts +85 -0
- package/src/providers/__tests__/zero-fill-guard.test.ts +62 -0
- package/src/providers/__tests__/zhipu.test.ts +27 -14
- package/src/providers/alibaba.ts +20 -39
- package/src/providers/anthropic.ts +23 -11
- package/src/providers/base.ts +19 -0
- package/src/providers/claude-code.ts +27 -18
- package/src/providers/deepseek.ts +9 -28
- package/src/providers/gemini.ts +20 -58
- package/src/providers/moonshot.ts +15 -11
- package/src/providers/openai.ts +79 -61
- package/src/providers/opencode.ts +16 -33
- package/src/providers/stream-accumulator.ts +27 -21
- package/src/providers/types.ts +29 -4
- package/src/providers/zhipu.ts +15 -44
- package/src/task-analysis/__tests__/analyzer.test.ts +0 -0
- package/src/task-analysis/__tests__/enrichers-schema-deducer.test.ts +34 -2
- package/src/task-analysis/__tests__/no-ast-imports.test.ts +52 -0
- package/src/task-analysis/__tests__/repository.test.ts +65 -0
- package/src/task-analysis/__tests__/types.test.ts +63 -130
- package/src/task-analysis/analyzer.ts +91 -0
- package/src/task-analysis/enrichers/schema-deducer.ts +13 -2
- package/src/task-analysis/index.ts +2 -36
- package/src/task-analysis/repository.ts +45 -0
- package/src/task-analysis/types.ts +42 -58
- package/src/ui/client/__tests__/api.test.ts +143 -1
- package/src/ui/client/__tests__/bootstrap.test.ts +178 -62
- package/src/ui/client/__tests__/job-adapter.test.ts +203 -2
- package/src/ui/client/__tests__/load-state.test.ts +70 -0
- package/src/ui/client/__tests__/types.test.ts +66 -3
- package/src/ui/client/__tests__/useJobDetailWithUpdates.test.ts +390 -77
- package/src/ui/client/__tests__/useJobList.test.ts +198 -23
- package/src/ui/client/__tests__/useJobListWithUpdates.test.ts +218 -7
- package/src/ui/client/adapters/__tests__/job-adapter.test.ts +186 -0
- package/src/ui/client/adapters/job-adapter.ts +41 -16
- package/src/ui/client/api.ts +38 -15
- package/src/ui/client/bootstrap.ts +19 -14
- package/src/ui/client/hooks/useJobDetailWithUpdates.ts +73 -97
- package/src/ui/client/hooks/useJobList.ts +26 -31
- package/src/ui/client/hooks/useJobListWithUpdates.ts +42 -76
- package/src/ui/client/load-state.ts +20 -0
- package/src/ui/client/reducers/__tests__/job-events.test.ts +568 -0
- package/src/ui/client/reducers/job-events.ts +137 -0
- package/src/ui/client/types.ts +16 -20
- package/src/ui/components/DAGGrid.tsx +6 -1
- package/src/ui/components/JobDetail.tsx +12 -4
- package/src/ui/components/JobTable.tsx +41 -13
- package/src/ui/components/StageTimeline.tsx +8 -20
- package/src/ui/components/TaskAnalysisDisplay.tsx +48 -6
- package/src/ui/components/__tests__/DAGGrid.test.tsx +36 -0
- package/src/ui/components/__tests__/JobDetail.test.tsx +112 -0
- package/src/ui/components/__tests__/JobTable.test.tsx +137 -1
- package/src/ui/components/__tests__/StageTimeline.test.tsx +5 -6
- package/src/ui/components/__tests__/TaskAnalysisDisplay.test.tsx +64 -0
- package/src/ui/components/types.ts +35 -15
- package/src/ui/dist/assets/{index--RH3sAt3.js → index-DN3-zvtP.js} +4324 -263
- package/src/ui/dist/assets/index-DN3-zvtP.js.map +1 -0
- package/src/ui/dist/assets/style-CtZBnjlR.css +2 -0
- package/src/ui/dist/index.html +2 -2
- package/src/ui/pages/PipelineDetail.tsx +60 -4
- package/src/ui/pages/PromptPipelineDashboard.tsx +59 -7
- package/src/ui/pages/__tests__/PromptPipelineDashboard.test.tsx +114 -32
- package/src/ui/pages/__tests__/pages.test.tsx +236 -42
- package/src/ui/server/__tests__/concurrency-endpoint.test.ts +54 -0
- package/src/ui/server/__tests__/config-bridge-node.test.ts +34 -1
- package/src/ui/server/__tests__/config-bridge.test.ts +9 -1
- package/src/ui/server/__tests__/embedded-assets.test.ts +66 -0
- package/src/ui/server/__tests__/file-endpoints.test.ts +183 -5
- package/src/ui/server/__tests__/gate-endpoints.test.ts +55 -0
- package/src/ui/server/__tests__/index.test.ts +63 -0
- package/src/ui/server/__tests__/job-control-endpoints.test.ts +512 -2
- package/src/ui/server/__tests__/job-endpoints.test.ts +158 -2
- package/src/ui/server/__tests__/read-static-path-guard.test.ts +94 -0
- package/src/ui/server/__tests__/router-root-isolation.test.ts +204 -0
- package/src/ui/server/__tests__/router-threading.test.ts +148 -0
- package/src/ui/server/__tests__/router.test.ts +104 -0
- package/src/ui/server/__tests__/sse-broadcast.test.ts +44 -0
- package/src/ui/server/__tests__/sse-enhancer.test.ts +70 -0
- package/src/ui/server/config-bridge-node.ts +8 -26
- package/src/ui/server/config-bridge.ts +3 -4
- package/src/ui/server/embedded-assets-imports.d.ts +44 -0
- package/src/ui/server/embedded-assets.ts +13 -2
- package/src/ui/server/endpoints/__tests__/meta-endpoint.test.ts +1 -0
- package/src/ui/server/endpoints/__tests__/pipeline-analysis-endpoint.test.ts +167 -0
- package/src/ui/server/endpoints/__tests__/schema-file-endpoint.test.ts +104 -0
- package/src/ui/server/endpoints/__tests__/task-analysis-endpoint.test.ts +103 -0
- package/src/ui/server/endpoints/__tests__/upload-endpoints.test.ts +162 -96
- package/src/ui/server/endpoints/concurrency-endpoint.ts +2 -1
- package/src/ui/server/endpoints/create-pipeline-endpoint.ts +14 -29
- package/src/ui/server/endpoints/file-endpoints.ts +15 -10
- package/src/ui/server/endpoints/gate-endpoints.ts +2 -6
- package/src/ui/server/endpoints/job-control-endpoints.ts +129 -141
- package/src/ui/server/endpoints/job-endpoints.ts +7 -0
- package/src/ui/server/endpoints/meta-endpoint.ts +1 -1
- package/src/ui/server/endpoints/pipeline-analysis-endpoint.ts +99 -11
- package/src/ui/server/endpoints/schema-file-endpoint.ts +11 -3
- package/src/ui/server/endpoints/task-analysis-endpoint.ts +21 -5
- package/src/ui/server/endpoints/task-save-endpoint.ts +1 -2
- package/src/ui/server/endpoints/upload-endpoints.ts +5 -40
- package/src/ui/server/index.ts +19 -14
- package/src/ui/server/job-reader.ts +14 -2
- package/src/ui/server/router.ts +33 -10
- package/src/ui/server/sse-broadcast.ts +14 -3
- package/src/ui/server/sse-enhancer.ts +12 -2
- package/src/ui/state/__tests__/schema-loader.test.ts +11 -1
- package/src/ui/state/__tests__/snapshot.test.ts +120 -14
- package/src/ui/state/__tests__/types.test.ts +104 -5
- package/src/ui/state/snapshot.ts +2 -2
- package/src/ui/state/transformers/__tests__/list-transformer.test.ts +85 -2
- package/src/ui/state/transformers/__tests__/status-transformer.test.ts +250 -22
- package/src/ui/state/transformers/list-transformer.ts +13 -3
- package/src/ui/state/transformers/status-transformer.ts +36 -170
- package/src/ui/state/types.ts +15 -48
- package/src/utils/__tests__/path-containment.test.ts +160 -0
- package/src/{ui/server/utils → utils}/path-containment.ts +21 -0
- package/src/task-analysis/__tests__/enrichers-analysis-writer.test.ts +0 -86
- package/src/task-analysis/__tests__/enrichers-artifact-resolver.test.ts +0 -124
- package/src/task-analysis/__tests__/extractors-artifacts.test.ts +0 -133
- package/src/task-analysis/__tests__/extractors-llm-calls.test.ts +0 -46
- package/src/task-analysis/__tests__/extractors-stages.test.ts +0 -52
- package/src/task-analysis/__tests__/index.test.ts +0 -143
- package/src/task-analysis/__tests__/parser.test.ts +0 -41
- package/src/task-analysis/__tests__/utils-ast.test.ts +0 -82
- package/src/task-analysis/enrichers/analysis-writer.ts +0 -75
- package/src/task-analysis/enrichers/artifact-resolver.ts +0 -89
- package/src/task-analysis/extractors/artifacts.ts +0 -143
- package/src/task-analysis/extractors/llm-calls.ts +0 -117
- package/src/task-analysis/extractors/stages.ts +0 -45
- package/src/task-analysis/parser.ts +0 -20
- package/src/task-analysis/utils/ast.ts +0 -45
- package/src/ui/dist/assets/index--RH3sAt3.js.map +0 -1
- package/src/ui/dist/assets/style-CSSKuMOe.css +0 -2
- package/src/ui/embedded-assets.js +0 -12
- package/src/ui/server/__tests__/path-containment.test.ts +0 -54
|
@@ -1,19 +1,25 @@
|
|
|
1
|
-
import { describe, test, expect } from "bun:test";
|
|
1
|
+
import { describe, test, expect, afterEach } from "bun:test";
|
|
2
|
+
import type { Subprocess } from "bun";
|
|
2
3
|
import { mkdtemp, mkdir, writeFile, utimes, rm, readdir, readFile } from "node:fs/promises";
|
|
3
|
-
import { existsSync } from "node:fs";
|
|
4
|
+
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
|
4
5
|
import { tmpdir } from "node:os";
|
|
5
6
|
import { join } from "node:path";
|
|
6
7
|
import {
|
|
8
|
+
adoptOrphanedRunners,
|
|
7
9
|
drainPendingQueue,
|
|
8
10
|
handleChildExit,
|
|
9
11
|
resolveDirs,
|
|
12
|
+
startOrchestrator,
|
|
10
13
|
promoteSeedFile,
|
|
11
14
|
restoreSeedFile,
|
|
12
15
|
} from "../orchestrator";
|
|
16
|
+
import type { ChildHandle } from "../orchestrator";
|
|
13
17
|
import {
|
|
14
18
|
getConcurrencyRuntimePaths,
|
|
15
19
|
getJobConcurrencyStatus,
|
|
16
20
|
} from "../job-concurrency";
|
|
21
|
+
import { isProcessAlive } from "../runner-liveness";
|
|
22
|
+
import { resetConfig } from "../config";
|
|
17
23
|
|
|
18
24
|
async function setupDir(prefix: string): Promise<string> {
|
|
19
25
|
const dir = await mkdtemp(join(tmpdir(), prefix));
|
|
@@ -131,6 +137,7 @@ describe("drainPendingQueue", () => {
|
|
|
131
137
|
dataDir: dir,
|
|
132
138
|
maxConcurrentJobs: 2,
|
|
133
139
|
lockTimeoutMs: 1000,
|
|
140
|
+
staleLeaseTimeoutMs: 1000,
|
|
134
141
|
spawnRunner: fakeSpawnRunner(pids),
|
|
135
142
|
});
|
|
136
143
|
expect(result.promoted).toEqual(["job-a", "job-b"]);
|
|
@@ -154,6 +161,7 @@ describe("drainPendingQueue", () => {
|
|
|
154
161
|
dataDir: dir,
|
|
155
162
|
maxConcurrentJobs: 0,
|
|
156
163
|
lockTimeoutMs: 1000,
|
|
164
|
+
staleLeaseTimeoutMs: 1000,
|
|
157
165
|
spawnRunner: fakeSpawnRunner(pids),
|
|
158
166
|
});
|
|
159
167
|
expect(result.promoted).toEqual([]);
|
|
@@ -181,6 +189,7 @@ describe("drainPendingQueue", () => {
|
|
|
181
189
|
dataDir: dir,
|
|
182
190
|
maxConcurrentJobs: 5,
|
|
183
191
|
lockTimeoutMs: 1000,
|
|
192
|
+
staleLeaseTimeoutMs: 1000,
|
|
184
193
|
spawnRunner: fakeSpawnRunner(pids),
|
|
185
194
|
});
|
|
186
195
|
expect(result.promoted).toEqual(["job-fresh"]);
|
|
@@ -206,6 +215,7 @@ describe("drainPendingQueue", () => {
|
|
|
206
215
|
dataDir: dir,
|
|
207
216
|
maxConcurrentJobs: 5,
|
|
208
217
|
lockTimeoutMs: 1000,
|
|
218
|
+
staleLeaseTimeoutMs: 1000,
|
|
209
219
|
spawnRunner: fakeSpawnRunner(pids),
|
|
210
220
|
});
|
|
211
221
|
expect(result.promoted).toEqual(["job-staged"]);
|
|
@@ -232,6 +242,7 @@ describe("drainPendingQueue", () => {
|
|
|
232
242
|
dataDir: dir,
|
|
233
243
|
maxConcurrentJobs: 5,
|
|
234
244
|
lockTimeoutMs: 1000,
|
|
245
|
+
staleLeaseTimeoutMs: 1000,
|
|
235
246
|
spawnRunner: fakeSpawnRunner(pids),
|
|
236
247
|
});
|
|
237
248
|
expect(result.promoted).toEqual(["job-heal"]);
|
|
@@ -255,6 +266,7 @@ describe("drainPendingQueue", () => {
|
|
|
255
266
|
dataDir: dir,
|
|
256
267
|
maxConcurrentJobs: 5,
|
|
257
268
|
lockTimeoutMs: 1000,
|
|
269
|
+
staleLeaseTimeoutMs: 1000,
|
|
258
270
|
spawnRunner: fakeSpawnRunner(pids),
|
|
259
271
|
});
|
|
260
272
|
expect(result.promoted).toEqual(["job-a", "job-b", "job-c"]);
|
|
@@ -274,6 +286,7 @@ describe("drainPendingQueue", () => {
|
|
|
274
286
|
dataDir: dir,
|
|
275
287
|
maxConcurrentJobs: 2,
|
|
276
288
|
lockTimeoutMs: 1000,
|
|
289
|
+
staleLeaseTimeoutMs: 1000,
|
|
277
290
|
spawnRunner: failingSpawn,
|
|
278
291
|
});
|
|
279
292
|
expect(result.promoted).toEqual([]);
|
|
@@ -300,6 +313,7 @@ describe("drainPendingQueue", () => {
|
|
|
300
313
|
dataDir: dir,
|
|
301
314
|
maxConcurrentJobs: 2,
|
|
302
315
|
lockTimeoutMs: 1000,
|
|
316
|
+
staleLeaseTimeoutMs: 1000,
|
|
303
317
|
spawnRunner: failingSpawn,
|
|
304
318
|
});
|
|
305
319
|
}
|
|
@@ -328,6 +342,7 @@ describe("drainPendingQueue", () => {
|
|
|
328
342
|
dataDir: dir,
|
|
329
343
|
maxConcurrentJobs: 2,
|
|
330
344
|
lockTimeoutMs: 1000,
|
|
345
|
+
staleLeaseTimeoutMs: 1000,
|
|
331
346
|
spawnRunner: failingSpawn,
|
|
332
347
|
});
|
|
333
348
|
}
|
|
@@ -354,6 +369,7 @@ describe("drainPendingQueue", () => {
|
|
|
354
369
|
dataDir: dir,
|
|
355
370
|
maxConcurrentJobs: 2,
|
|
356
371
|
lockTimeoutMs: 1000,
|
|
372
|
+
staleLeaseTimeoutMs: 1000,
|
|
357
373
|
spawnRunner: fakeSpawnRunner(new Map()),
|
|
358
374
|
});
|
|
359
375
|
}
|
|
@@ -377,6 +393,7 @@ describe("drainPendingQueue", () => {
|
|
|
377
393
|
dataDir: dir,
|
|
378
394
|
maxConcurrentJobs: 2,
|
|
379
395
|
lockTimeoutMs: 10,
|
|
396
|
+
staleLeaseTimeoutMs: 10,
|
|
380
397
|
spawnRunner: async () => {
|
|
381
398
|
await mkdir(lockDir, { recursive: true });
|
|
382
399
|
await writeFile(join(lockDir, "owner.json"), JSON.stringify({ pid: process.pid, acquiredAt: new Date().toISOString() }));
|
|
@@ -412,6 +429,7 @@ describe("handleChildExit", () => {
|
|
|
412
429
|
dataDir: dir,
|
|
413
430
|
maxConcurrentJobs: 1,
|
|
414
431
|
lockTimeoutMs: 1000,
|
|
432
|
+
staleLeaseTimeoutMs: 1000,
|
|
415
433
|
spawnRunner: spawn,
|
|
416
434
|
});
|
|
417
435
|
expect(first.promoted).toEqual(["job-a"]);
|
|
@@ -423,6 +441,7 @@ describe("handleChildExit", () => {
|
|
|
423
441
|
dataDir: dir,
|
|
424
442
|
maxConcurrentJobs: 1,
|
|
425
443
|
lockTimeoutMs: 1000,
|
|
444
|
+
staleLeaseTimeoutMs: 1000,
|
|
426
445
|
spawnRunner: spawn,
|
|
427
446
|
}).then((r) => {
|
|
428
447
|
drainResult = r;
|
|
@@ -463,6 +482,146 @@ describe("handleChildExit", () => {
|
|
|
463
482
|
await rm(dir, { recursive: true });
|
|
464
483
|
}
|
|
465
484
|
});
|
|
485
|
+
|
|
486
|
+
test("handleChildExit still attempts reconciliation when slot release fails", async () => {
|
|
487
|
+
const dir = await setupDir("exit-release-fails-");
|
|
488
|
+
try {
|
|
489
|
+
const releaseError = new Error("release failed");
|
|
490
|
+
let drained = false;
|
|
491
|
+
let reconciled = false;
|
|
492
|
+
|
|
493
|
+
await expect(handleChildExit({
|
|
494
|
+
dataDir: dir,
|
|
495
|
+
jobId: "job-release-fails",
|
|
496
|
+
triggerDrain: () => {
|
|
497
|
+
drained = true;
|
|
498
|
+
},
|
|
499
|
+
lockTimeoutMs: 1000,
|
|
500
|
+
releaseSlot: async () => {
|
|
501
|
+
throw releaseError;
|
|
502
|
+
},
|
|
503
|
+
reconcileJob: async () => {
|
|
504
|
+
reconciled = true;
|
|
505
|
+
return { reconciled: false, reason: "unreadable" };
|
|
506
|
+
},
|
|
507
|
+
})).rejects.toThrow("release failed");
|
|
508
|
+
|
|
509
|
+
expect(reconciled).toBe(true);
|
|
510
|
+
expect(drained).toBe(true);
|
|
511
|
+
} finally {
|
|
512
|
+
await rm(dir, { recursive: true });
|
|
513
|
+
}
|
|
514
|
+
});
|
|
515
|
+
|
|
516
|
+
test("handleChildExit reports both cleanup errors when slot release and reconciliation fail", async () => {
|
|
517
|
+
const dir = await setupDir("exit-both-fail-");
|
|
518
|
+
try {
|
|
519
|
+
const releaseError = new Error("release failed");
|
|
520
|
+
const reconcileError = new Error("reconcile failed");
|
|
521
|
+
let drained = false;
|
|
522
|
+
|
|
523
|
+
let thrown: unknown;
|
|
524
|
+
try {
|
|
525
|
+
await handleChildExit({
|
|
526
|
+
dataDir: dir,
|
|
527
|
+
jobId: "job-both-fail",
|
|
528
|
+
triggerDrain: () => {
|
|
529
|
+
drained = true;
|
|
530
|
+
},
|
|
531
|
+
lockTimeoutMs: 1000,
|
|
532
|
+
releaseSlot: async () => {
|
|
533
|
+
throw releaseError;
|
|
534
|
+
},
|
|
535
|
+
reconcileJob: async () => {
|
|
536
|
+
throw reconcileError;
|
|
537
|
+
},
|
|
538
|
+
});
|
|
539
|
+
} catch (err) {
|
|
540
|
+
thrown = err;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
expect(thrown).toBeInstanceOf(AggregateError);
|
|
544
|
+
expect((thrown as AggregateError).message).toBe("child exit cleanup failed");
|
|
545
|
+
expect((thrown as AggregateError).errors).toEqual([releaseError, reconcileError]);
|
|
546
|
+
expect(drained).toBe(true);
|
|
547
|
+
} finally {
|
|
548
|
+
await rm(dir, { recursive: true });
|
|
549
|
+
}
|
|
550
|
+
});
|
|
551
|
+
|
|
552
|
+
test("AC-3: a mid-job runner exit marks the running task and job failed, releases the slot, and triggers a drain", async () => {
|
|
553
|
+
const dir = await setupDir("exit-reconcile-");
|
|
554
|
+
try {
|
|
555
|
+
const jobId = "job-running";
|
|
556
|
+
const jobDir = join(dir, "current", jobId);
|
|
557
|
+
await mkdir(jobDir, { recursive: true });
|
|
558
|
+
await writeFile(
|
|
559
|
+
join(jobDir, "tasks-status.json"),
|
|
560
|
+
JSON.stringify({
|
|
561
|
+
id: jobId,
|
|
562
|
+
state: "running",
|
|
563
|
+
current: "research",
|
|
564
|
+
currentStage: null,
|
|
565
|
+
lastUpdated: "2026-06-19T11:00:00.000Z",
|
|
566
|
+
tasks: { research: { state: "running" } },
|
|
567
|
+
files: { artifacts: [], logs: [], tmp: [] },
|
|
568
|
+
}),
|
|
569
|
+
);
|
|
570
|
+
await writeFile(join(jobDir, "runner.pid"), "999999");
|
|
571
|
+
|
|
572
|
+
let drained = false;
|
|
573
|
+
await handleChildExit({
|
|
574
|
+
dataDir: dir,
|
|
575
|
+
jobId,
|
|
576
|
+
triggerDrain: () => {
|
|
577
|
+
drained = true;
|
|
578
|
+
},
|
|
579
|
+
code: 1,
|
|
580
|
+
signal: null,
|
|
581
|
+
lockTimeoutMs: 1000,
|
|
582
|
+
});
|
|
583
|
+
|
|
584
|
+
expect(drained).toBe(true);
|
|
585
|
+
const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
|
|
586
|
+
expect(existsSync(join(runningJobsDir, `${jobId}.json`))).toBe(false);
|
|
587
|
+
|
|
588
|
+
const snapshot = JSON.parse(await readFile(join(jobDir, "tasks-status.json"), "utf-8")) as Record<string, unknown>;
|
|
589
|
+
expect(snapshot["state"]).toBe("failed");
|
|
590
|
+
const tasks = snapshot["tasks"] as Record<string, Record<string, unknown>>;
|
|
591
|
+
expect(tasks["research"]?.["state"]).toBe("failed");
|
|
592
|
+
expect(tasks["research"]?.["error"]).toBeTruthy();
|
|
593
|
+
const errorContext = tasks["research"]?.["errorContext"] as Record<string, unknown>;
|
|
594
|
+
expect(errorContext?.["kind"]).toBe("runner-interrupted");
|
|
595
|
+
expect(errorContext?.["code"]).toBe(1);
|
|
596
|
+
} finally {
|
|
597
|
+
await rm(dir, { recursive: true });
|
|
598
|
+
}
|
|
599
|
+
});
|
|
600
|
+
|
|
601
|
+
test("AC-10: a job already moved to complete/ (no current/ status) is untouched, and the drain still fires", async () => {
|
|
602
|
+
const dir = await setupDir("exit-complete-");
|
|
603
|
+
try {
|
|
604
|
+
const jobId = "job-gone";
|
|
605
|
+
const statusPath = join(dir, "current", jobId, "tasks-status.json");
|
|
606
|
+
|
|
607
|
+
let drainCount = 0;
|
|
608
|
+
await handleChildExit({
|
|
609
|
+
dataDir: dir,
|
|
610
|
+
jobId,
|
|
611
|
+
triggerDrain: () => {
|
|
612
|
+
drainCount++;
|
|
613
|
+
},
|
|
614
|
+
code: 0,
|
|
615
|
+
signal: null,
|
|
616
|
+
lockTimeoutMs: 1000,
|
|
617
|
+
});
|
|
618
|
+
|
|
619
|
+
expect(drainCount).toBe(1);
|
|
620
|
+
expect(existsSync(statusPath)).toBe(false);
|
|
621
|
+
} finally {
|
|
622
|
+
await rm(dir, { recursive: true });
|
|
623
|
+
}
|
|
624
|
+
});
|
|
466
625
|
});
|
|
467
626
|
|
|
468
627
|
describe("drainPendingQueue concurrency", () => {
|
|
@@ -481,12 +640,14 @@ describe("drainPendingQueue concurrency", () => {
|
|
|
481
640
|
dataDir: dir,
|
|
482
641
|
maxConcurrentJobs: 2,
|
|
483
642
|
lockTimeoutMs: 1000,
|
|
643
|
+
staleLeaseTimeoutMs: 1000,
|
|
484
644
|
spawnRunner: spawn,
|
|
485
645
|
}),
|
|
486
646
|
drainPendingQueue({
|
|
487
647
|
dataDir: dir,
|
|
488
648
|
maxConcurrentJobs: 2,
|
|
489
649
|
lockTimeoutMs: 1000,
|
|
650
|
+
staleLeaseTimeoutMs: 1000,
|
|
490
651
|
spawnRunner: spawn,
|
|
491
652
|
}),
|
|
492
653
|
]);
|
|
@@ -512,3 +673,455 @@ describe("drainPendingQueue concurrency", () => {
|
|
|
512
673
|
}
|
|
513
674
|
});
|
|
514
675
|
});
|
|
676
|
+
|
|
677
|
+
describe("drainPendingQueue stale-lease age (AC-5)", () => {
|
|
678
|
+
test("does not prune a fresh null-pid in-flight lease when lockTimeoutMs is short but staleLeaseTimeoutMs is long", async () => {
|
|
679
|
+
const dir = await setupDir("drain-stale-age-");
|
|
680
|
+
try {
|
|
681
|
+
await writeLease(
|
|
682
|
+
dir,
|
|
683
|
+
"job-held",
|
|
684
|
+
null,
|
|
685
|
+
new Date(Date.now() - 1000).toISOString(),
|
|
686
|
+
);
|
|
687
|
+
await writeSeed(dir, "job-queued", { pipeline: "p" }, 1700000000);
|
|
688
|
+
|
|
689
|
+
const result = await drainPendingQueue({
|
|
690
|
+
dataDir: dir,
|
|
691
|
+
maxConcurrentJobs: 1,
|
|
692
|
+
lockTimeoutMs: 50,
|
|
693
|
+
staleLeaseTimeoutMs: 30000,
|
|
694
|
+
spawnRunner: fakeSpawnRunner(new Map()),
|
|
695
|
+
});
|
|
696
|
+
|
|
697
|
+
expect(result.promoted).toEqual([]);
|
|
698
|
+
const { runningJobsDir } = getConcurrencyRuntimePaths(dir);
|
|
699
|
+
expect(existsSync(join(runningJobsDir, "job-held.json"))).toBe(true);
|
|
700
|
+
} finally {
|
|
701
|
+
await rm(dir, { recursive: true });
|
|
702
|
+
}
|
|
703
|
+
});
|
|
704
|
+
});
|
|
705
|
+
|
|
706
|
+
interface FakeWatcher {
|
|
707
|
+
on(event: "add", cb: (path: string) => void): FakeWatcher;
|
|
708
|
+
on(event: "ready", cb: () => void): FakeWatcher;
|
|
709
|
+
on(event: "error", cb: (err: Error) => void): FakeWatcher;
|
|
710
|
+
close(): Promise<void>;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
function makeFakeWatcherFactory() {
|
|
714
|
+
return (_path: string, _options: Record<string, unknown>): FakeWatcher => {
|
|
715
|
+
const watcher = {
|
|
716
|
+
on(event: "add" | "ready" | "error", cb: () => void): FakeWatcher {
|
|
717
|
+
if (event === "ready") queueMicrotask(() => cb());
|
|
718
|
+
return watcher;
|
|
719
|
+
},
|
|
720
|
+
close(): Promise<void> {
|
|
721
|
+
return Promise.resolve();
|
|
722
|
+
},
|
|
723
|
+
} as unknown as FakeWatcher;
|
|
724
|
+
return watcher;
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
async function writeRunningJob(
|
|
729
|
+
dataDir: string,
|
|
730
|
+
jobId: string,
|
|
731
|
+
lastUpdated: string,
|
|
732
|
+
runnerPid: number,
|
|
733
|
+
): Promise<void> {
|
|
734
|
+
const jobDir = join(dataDir, "current", jobId);
|
|
735
|
+
await mkdir(jobDir, { recursive: true });
|
|
736
|
+
await writeFile(
|
|
737
|
+
join(jobDir, "tasks-status.json"),
|
|
738
|
+
JSON.stringify({
|
|
739
|
+
id: jobId,
|
|
740
|
+
state: "running",
|
|
741
|
+
current: "research",
|
|
742
|
+
currentStage: null,
|
|
743
|
+
lastUpdated,
|
|
744
|
+
tasks: { research: { state: "running" } },
|
|
745
|
+
files: { artifacts: [], logs: [], tmp: [] },
|
|
746
|
+
}),
|
|
747
|
+
);
|
|
748
|
+
await writeFile(join(jobDir, "runner.pid"), String(runnerPid));
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
async function writeLease(
|
|
752
|
+
dataDir: string,
|
|
753
|
+
jobId: string,
|
|
754
|
+
pid: number | null,
|
|
755
|
+
acquiredAt: string,
|
|
756
|
+
): Promise<string> {
|
|
757
|
+
const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
|
|
758
|
+
await mkdir(runningJobsDir, { recursive: true });
|
|
759
|
+
const slotPath = join(runningJobsDir, `${jobId}.json`);
|
|
760
|
+
await writeFile(slotPath, JSON.stringify({
|
|
761
|
+
jobId,
|
|
762
|
+
pid,
|
|
763
|
+
acquiredAt,
|
|
764
|
+
source: "orchestrator",
|
|
765
|
+
slotPath,
|
|
766
|
+
}));
|
|
767
|
+
return slotPath;
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
describe("startOrchestrator boot sweep (AC-16)", () => {
|
|
771
|
+
afterEach(() => {
|
|
772
|
+
resetConfig();
|
|
773
|
+
});
|
|
774
|
+
|
|
775
|
+
test("a stale orphaned job in current/ is marked failed and the handle still resolves", async () => {
|
|
776
|
+
const root = await mkdtemp(join(tmpdir(), "boot-stale-"));
|
|
777
|
+
const dataDir = join(root, "pipeline-data");
|
|
778
|
+
try {
|
|
779
|
+
await writeRunningJob(dataDir, "job-stale", "2020-01-01T00:00:00.000Z", 999999);
|
|
780
|
+
|
|
781
|
+
const handle = await startOrchestrator({
|
|
782
|
+
dataDir,
|
|
783
|
+
watcherFactory: makeFakeWatcherFactory(),
|
|
784
|
+
});
|
|
785
|
+
|
|
786
|
+
expect(handle).toBeDefined();
|
|
787
|
+
expect(typeof handle.stop).toBe("function");
|
|
788
|
+
|
|
789
|
+
const snapshot = JSON.parse(
|
|
790
|
+
await readFile(join(dataDir, "current", "job-stale", "tasks-status.json"), "utf-8"),
|
|
791
|
+
) as Record<string, unknown>;
|
|
792
|
+
expect(snapshot["state"]).toBe("failed");
|
|
793
|
+
const tasks = snapshot["tasks"] as Record<string, Record<string, unknown>>;
|
|
794
|
+
expect(tasks["research"]?.["state"]).toBe("failed");
|
|
795
|
+
|
|
796
|
+
await handle.stop();
|
|
797
|
+
} finally {
|
|
798
|
+
await rm(root, { recursive: true, force: true });
|
|
799
|
+
}
|
|
800
|
+
});
|
|
801
|
+
|
|
802
|
+
test("a live job in current/ is left running and the handle still resolves", async () => {
|
|
803
|
+
const root = await mkdtemp(join(tmpdir(), "boot-live-"));
|
|
804
|
+
const dataDir = join(root, "pipeline-data");
|
|
805
|
+
try {
|
|
806
|
+
await writeRunningJob(dataDir, "job-live", "2020-01-01T00:00:00.000Z", process.pid);
|
|
807
|
+
|
|
808
|
+
const handle = await startOrchestrator({
|
|
809
|
+
dataDir,
|
|
810
|
+
watcherFactory: makeFakeWatcherFactory(),
|
|
811
|
+
});
|
|
812
|
+
|
|
813
|
+
expect(handle).toBeDefined();
|
|
814
|
+
expect(typeof handle.stop).toBe("function");
|
|
815
|
+
|
|
816
|
+
const snapshot = JSON.parse(
|
|
817
|
+
await readFile(join(dataDir, "current", "job-live", "tasks-status.json"), "utf-8"),
|
|
818
|
+
) as Record<string, unknown>;
|
|
819
|
+
expect(snapshot["state"]).toBe("running");
|
|
820
|
+
const tasks = snapshot["tasks"] as Record<string, Record<string, unknown>>;
|
|
821
|
+
expect(tasks["research"]?.["state"]).toBe("running");
|
|
822
|
+
|
|
823
|
+
await handle.stop();
|
|
824
|
+
} finally {
|
|
825
|
+
await rm(root, { recursive: true, force: true });
|
|
826
|
+
}
|
|
827
|
+
});
|
|
828
|
+
|
|
829
|
+
test("AC-13: threads staleLeaseTimeoutMs into boot reconciliation so a mid-aged in-flight lease is not abandoned", async () => {
|
|
830
|
+
const root = await mkdtemp(join(tmpdir(), "boot-stale-age-"));
|
|
831
|
+
const dataDir = join(root, "pipeline-data");
|
|
832
|
+
const previous = process.env["PO_STALE_LEASE_TIMEOUT_MS"];
|
|
833
|
+
process.env["PO_STALE_LEASE_TIMEOUT_MS"] = "60000";
|
|
834
|
+
resetConfig();
|
|
835
|
+
try {
|
|
836
|
+
await writeRunningJob(dataDir, "job-inflight", "2020-01-01T00:00:00.000Z", 999999);
|
|
837
|
+
await writeLease(
|
|
838
|
+
dataDir,
|
|
839
|
+
"job-inflight",
|
|
840
|
+
null,
|
|
841
|
+
new Date(Date.now() - 40000).toISOString(),
|
|
842
|
+
);
|
|
843
|
+
|
|
844
|
+
const handle = await startOrchestrator({
|
|
845
|
+
dataDir,
|
|
846
|
+
watcherFactory: makeFakeWatcherFactory(),
|
|
847
|
+
});
|
|
848
|
+
|
|
849
|
+
const snapshot = JSON.parse(
|
|
850
|
+
await readFile(join(dataDir, "current", "job-inflight", "tasks-status.json"), "utf-8"),
|
|
851
|
+
) as Record<string, unknown>;
|
|
852
|
+
expect(snapshot["state"]).toBe("running");
|
|
853
|
+
|
|
854
|
+
await handle.stop();
|
|
855
|
+
} finally {
|
|
856
|
+
if (previous === undefined) delete process.env["PO_STALE_LEASE_TIMEOUT_MS"];
|
|
857
|
+
else process.env["PO_STALE_LEASE_TIMEOUT_MS"] = previous;
|
|
858
|
+
resetConfig();
|
|
859
|
+
await rm(root, { recursive: true, force: true });
|
|
860
|
+
}
|
|
861
|
+
});
|
|
862
|
+
|
|
863
|
+
test("AC-2: a stale dead_pid lease in runtime/running-jobs/ is pruned at boot", async () => {
|
|
864
|
+
const root = await mkdtemp(join(tmpdir(), "boot-prune-dead-"));
|
|
865
|
+
const dataDir = join(root, "pipeline-data");
|
|
866
|
+
try {
|
|
867
|
+
await writeRunningJob(dataDir, "job-dead", "2020-01-01T00:00:00.000Z", 999999);
|
|
868
|
+
await writeLease(dataDir, "job-dead", 999999, "2020-01-01T00:00:00.000Z");
|
|
869
|
+
|
|
870
|
+
const handle = await startOrchestrator({
|
|
871
|
+
dataDir,
|
|
872
|
+
watcherFactory: makeFakeWatcherFactory(),
|
|
873
|
+
});
|
|
874
|
+
|
|
875
|
+
const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
|
|
876
|
+
expect(existsSync(join(runningJobsDir, "job-dead.json"))).toBe(false);
|
|
877
|
+
|
|
878
|
+
await handle.stop();
|
|
879
|
+
} finally {
|
|
880
|
+
await rm(root, { recursive: true, force: true });
|
|
881
|
+
}
|
|
882
|
+
});
|
|
883
|
+
|
|
884
|
+
test("AC-2: a live-pid lease with matching runner.pid is retained at boot", async () => {
|
|
885
|
+
const root = await mkdtemp(join(tmpdir(), "boot-prune-live-"));
|
|
886
|
+
const dataDir = join(root, "pipeline-data");
|
|
887
|
+
// Use a real disposable child for the "live pid" so re-adoption + handle.stop()
|
|
888
|
+
// (which SIGTERMs the adopted handle) targets the child, never the test runner.
|
|
889
|
+
const child = spawnSleeperChild();
|
|
890
|
+
try {
|
|
891
|
+
await writeRunningJob(dataDir, "job-live", "2020-01-01T00:00:00.000Z", child.pid!);
|
|
892
|
+
await writeLease(dataDir, "job-live", child.pid!, "2020-01-01T00:00:00.000Z");
|
|
893
|
+
|
|
894
|
+
const handle = await startOrchestrator({
|
|
895
|
+
dataDir,
|
|
896
|
+
watcherFactory: makeFakeWatcherFactory(),
|
|
897
|
+
});
|
|
898
|
+
|
|
899
|
+
const { runningJobsDir } = getConcurrencyRuntimePaths(dataDir);
|
|
900
|
+
expect(existsSync(join(runningJobsDir, "job-live.json"))).toBe(true);
|
|
901
|
+
|
|
902
|
+
await handle.stop();
|
|
903
|
+
} finally {
|
|
904
|
+
await teardownChild(child);
|
|
905
|
+
await rm(root, { recursive: true, force: true });
|
|
906
|
+
}
|
|
907
|
+
});
|
|
908
|
+
|
|
909
|
+
test("AC-7: with PO_MAX_RUNNING_JOBS=foo, startOrchestrator throws synchronously before creating pipeline-data directories", () => {
|
|
910
|
+
const root = mkdtempSync(join(tmpdir(), "boot-bad-env-"));
|
|
911
|
+
const dataDir = join(root, "pipeline-data");
|
|
912
|
+
const previous = process.env["PO_MAX_RUNNING_JOBS"];
|
|
913
|
+
process.env["PO_MAX_RUNNING_JOBS"] = "foo";
|
|
914
|
+
resetConfig();
|
|
915
|
+
try {
|
|
916
|
+
expect(() =>
|
|
917
|
+
startOrchestrator({
|
|
918
|
+
dataDir,
|
|
919
|
+
watcherFactory: makeFakeWatcherFactory(),
|
|
920
|
+
}),
|
|
921
|
+
).toThrow(/PO_MAX_RUNNING_JOBS/);
|
|
922
|
+
|
|
923
|
+
expect(existsSync(join(dataDir, "pending"))).toBe(false);
|
|
924
|
+
expect(existsSync(join(dataDir, "current"))).toBe(false);
|
|
925
|
+
expect(existsSync(join(dataDir, "complete"))).toBe(false);
|
|
926
|
+
expect(existsSync(join(dataDir, "rejected"))).toBe(false);
|
|
927
|
+
expect(existsSync(join(dataDir, "staging"))).toBe(false);
|
|
928
|
+
} finally {
|
|
929
|
+
if (previous === undefined) delete process.env["PO_MAX_RUNNING_JOBS"];
|
|
930
|
+
else process.env["PO_MAX_RUNNING_JOBS"] = previous;
|
|
931
|
+
resetConfig();
|
|
932
|
+
rmSync(root, { recursive: true, force: true });
|
|
933
|
+
}
|
|
934
|
+
});
|
|
935
|
+
});
|
|
936
|
+
|
|
937
|
+
function spawnSleeperChild(): Subprocess<"ignore", "ignore", "ignore"> {
|
|
938
|
+
return Bun.spawn(["sleep", "30"], {
|
|
939
|
+
stdin: "ignore",
|
|
940
|
+
stdout: "ignore",
|
|
941
|
+
stderr: "ignore",
|
|
942
|
+
});
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
async function teardownChild(child: Subprocess<"ignore", "ignore", "ignore">): Promise<void> {
|
|
946
|
+
try {
|
|
947
|
+
child.kill(9);
|
|
948
|
+
} catch {
|
|
949
|
+
// already gone
|
|
950
|
+
}
|
|
951
|
+
await child.exited.catch(() => undefined);
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
async function writeRunnerPid(dataDir: string, jobId: string, pid: number): Promise<void> {
|
|
955
|
+
const jobDir = join(dataDir, "current", jobId);
|
|
956
|
+
await mkdir(jobDir, { recursive: true });
|
|
957
|
+
await writeFile(join(jobDir, "runner.pid"), String(pid));
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
describe("adoptOrphanedRunners (AC-4, AC-10, AC-11, AC-12)", () => {
|
|
961
|
+
test("AC-4: a live lease whose runner.pid matches is adopted, and kill(15) delivers SIGTERM to the child", async () => {
|
|
962
|
+
const dir = await setupDir("adopt-live-");
|
|
963
|
+
const child = spawnSleeperChild();
|
|
964
|
+
try {
|
|
965
|
+
const jobId = "job-adopt";
|
|
966
|
+
await writeRunnerPid(dir, jobId, child.pid!);
|
|
967
|
+
await writeLease(dir, jobId, child.pid!, new Date().toISOString());
|
|
968
|
+
|
|
969
|
+
const running = new Map<string, ChildHandle>();
|
|
970
|
+
const adopted = await adoptOrphanedRunners({
|
|
971
|
+
dataDir: dir,
|
|
972
|
+
currentDir: join(dir, "current"),
|
|
973
|
+
running,
|
|
974
|
+
});
|
|
975
|
+
|
|
976
|
+
expect(adopted).toEqual([jobId]);
|
|
977
|
+
expect(running.has(jobId)).toBe(true);
|
|
978
|
+
|
|
979
|
+
const handle = running.get(jobId)!;
|
|
980
|
+
handle.kill(15);
|
|
981
|
+
|
|
982
|
+
const result = await Promise.race([
|
|
983
|
+
child.exited.then(() => "exited" as const),
|
|
984
|
+
new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 3000)),
|
|
985
|
+
]);
|
|
986
|
+
expect(result).toBe("exited");
|
|
987
|
+
} finally {
|
|
988
|
+
await teardownChild(child);
|
|
989
|
+
await rm(dir, { recursive: true, force: true });
|
|
990
|
+
}
|
|
991
|
+
});
|
|
992
|
+
|
|
993
|
+
test("AC-10: a lease with a dead pid is not adopted", async () => {
|
|
994
|
+
const dir = await setupDir("adopt-dead-");
|
|
995
|
+
const child = spawnSleeperChild();
|
|
996
|
+
try {
|
|
997
|
+
const deadPid = 999999;
|
|
998
|
+
await writeRunnerPid(dir, "job-dead", deadPid);
|
|
999
|
+
await writeLease(dir, "job-dead", deadPid, new Date().toISOString());
|
|
1000
|
+
|
|
1001
|
+
const running = new Map<string, ChildHandle>();
|
|
1002
|
+
const adopted = await adoptOrphanedRunners({
|
|
1003
|
+
dataDir: dir,
|
|
1004
|
+
currentDir: join(dir, "current"),
|
|
1005
|
+
running,
|
|
1006
|
+
});
|
|
1007
|
+
|
|
1008
|
+
expect(adopted).toEqual([]);
|
|
1009
|
+
expect(running.size).toBe(0);
|
|
1010
|
+
} finally {
|
|
1011
|
+
await teardownChild(child);
|
|
1012
|
+
await rm(dir, { recursive: true, force: true });
|
|
1013
|
+
}
|
|
1014
|
+
});
|
|
1015
|
+
|
|
1016
|
+
test("AC-10: a live lease with an absent runner.pid is not adopted", async () => {
|
|
1017
|
+
const dir = await setupDir("adopt-nopidfile-");
|
|
1018
|
+
const child = spawnSleeperChild();
|
|
1019
|
+
try {
|
|
1020
|
+
const jobId = "job-nopidfile";
|
|
1021
|
+
await mkdir(join(dir, "current", jobId), { recursive: true });
|
|
1022
|
+
await writeLease(dir, jobId, child.pid!, new Date().toISOString());
|
|
1023
|
+
|
|
1024
|
+
const running = new Map<string, ChildHandle>();
|
|
1025
|
+
const adopted = await adoptOrphanedRunners({
|
|
1026
|
+
dataDir: dir,
|
|
1027
|
+
currentDir: join(dir, "current"),
|
|
1028
|
+
running,
|
|
1029
|
+
});
|
|
1030
|
+
|
|
1031
|
+
expect(adopted).toEqual([]);
|
|
1032
|
+
expect(running.size).toBe(0);
|
|
1033
|
+
} finally {
|
|
1034
|
+
await teardownChild(child);
|
|
1035
|
+
await rm(dir, { recursive: true, force: true });
|
|
1036
|
+
}
|
|
1037
|
+
});
|
|
1038
|
+
|
|
1039
|
+
test("AC-10: a live lease whose runner.pid mismatches is not adopted", async () => {
|
|
1040
|
+
const dir = await setupDir("adopt-mismatch-");
|
|
1041
|
+
const child = spawnSleeperChild();
|
|
1042
|
+
try {
|
|
1043
|
+
const jobId = "job-mismatch";
|
|
1044
|
+
await writeRunnerPid(dir, jobId, child.pid! + 1);
|
|
1045
|
+
await writeLease(dir, jobId, child.pid!, new Date().toISOString());
|
|
1046
|
+
|
|
1047
|
+
const running = new Map<string, ChildHandle>();
|
|
1048
|
+
const adopted = await adoptOrphanedRunners({
|
|
1049
|
+
dataDir: dir,
|
|
1050
|
+
currentDir: join(dir, "current"),
|
|
1051
|
+
running,
|
|
1052
|
+
});
|
|
1053
|
+
|
|
1054
|
+
expect(adopted).toEqual([]);
|
|
1055
|
+
expect(running.size).toBe(0);
|
|
1056
|
+
} finally {
|
|
1057
|
+
await teardownChild(child);
|
|
1058
|
+
await rm(dir, { recursive: true, force: true });
|
|
1059
|
+
}
|
|
1060
|
+
});
|
|
1061
|
+
|
|
1062
|
+
test("AC-11: mutating runner.pid to a different value before kill causes the signal to be skipped", async () => {
|
|
1063
|
+
const dir = await setupDir("adopt-reread-");
|
|
1064
|
+
const child = spawnSleeperChild();
|
|
1065
|
+
try {
|
|
1066
|
+
const jobId = "job-reread";
|
|
1067
|
+
await writeRunnerPid(dir, jobId, child.pid!);
|
|
1068
|
+
await writeLease(dir, jobId, child.pid!, new Date().toISOString());
|
|
1069
|
+
|
|
1070
|
+
const running = new Map<string, ChildHandle>();
|
|
1071
|
+
await adoptOrphanedRunners({
|
|
1072
|
+
dataDir: dir,
|
|
1073
|
+
currentDir: join(dir, "current"),
|
|
1074
|
+
running,
|
|
1075
|
+
});
|
|
1076
|
+
|
|
1077
|
+
// Mutate runner.pid before kill so the re-read guard skips the signal.
|
|
1078
|
+
await writeFile(join(dir, "current", jobId, "runner.pid"), String(child.pid! + 1));
|
|
1079
|
+
|
|
1080
|
+
const handle = running.get(jobId)!;
|
|
1081
|
+
handle.kill(15);
|
|
1082
|
+
|
|
1083
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
1084
|
+
expect(isProcessAlive(child.pid!)).toBe(true);
|
|
1085
|
+
} finally {
|
|
1086
|
+
await teardownChild(child);
|
|
1087
|
+
await rm(dir, { recursive: true, force: true });
|
|
1088
|
+
}
|
|
1089
|
+
});
|
|
1090
|
+
|
|
1091
|
+
test("AC-12: reading exited starts the poll and resolves with completionType 'signal' after death; getter is lazy (memoized)", async () => {
|
|
1092
|
+
const dir = await setupDir("adopt-exited-");
|
|
1093
|
+
const child = spawnSleeperChild();
|
|
1094
|
+
try {
|
|
1095
|
+
const jobId = "job-exited";
|
|
1096
|
+
await writeRunnerPid(dir, jobId, child.pid!);
|
|
1097
|
+
await writeLease(dir, jobId, child.pid!, new Date().toISOString());
|
|
1098
|
+
|
|
1099
|
+
const running = new Map<string, ChildHandle>();
|
|
1100
|
+
await adoptOrphanedRunners({
|
|
1101
|
+
dataDir: dir,
|
|
1102
|
+
currentDir: join(dir, "current"),
|
|
1103
|
+
running,
|
|
1104
|
+
});
|
|
1105
|
+
|
|
1106
|
+
const handle = running.get(jobId)!;
|
|
1107
|
+
|
|
1108
|
+
// Reading `exited` is what starts the poll; the getter must memoize the promise.
|
|
1109
|
+
const exitedA = handle.exited;
|
|
1110
|
+
const exitedB = handle.exited;
|
|
1111
|
+
expect(exitedA).toBe(exitedB);
|
|
1112
|
+
|
|
1113
|
+
child.kill(9);
|
|
1114
|
+
|
|
1115
|
+
const result = await Promise.race([
|
|
1116
|
+
exitedA,
|
|
1117
|
+
new Promise<{ completionType: string }>((resolve) =>
|
|
1118
|
+
setTimeout(() => resolve({ completionType: "timeout" }), 3000),
|
|
1119
|
+
),
|
|
1120
|
+
]);
|
|
1121
|
+
expect(result).toEqual({ code: null, signal: null, completionType: "signal" });
|
|
1122
|
+
} finally {
|
|
1123
|
+
await teardownChild(child);
|
|
1124
|
+
await rm(dir, { recursive: true, force: true });
|
|
1125
|
+
}
|
|
1126
|
+
});
|
|
1127
|
+
});
|