@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/http-api.md +66 -0
- package/package.json +1 -1
- package/src/api/__tests__/index.test.ts +311 -3
- package/src/api/index.ts +94 -45
- package/src/config/__tests__/statuses.test.ts +41 -0
- package/src/config/statuses.ts +20 -1
- package/src/core/__tests__/config.test.ts +31 -1
- package/src/core/__tests__/job-concurrency.test.ts +60 -0
- package/src/core/__tests__/job-view.test.ts +859 -0
- package/src/core/__tests__/orchestrator.test.ts +242 -0
- package/src/core/__tests__/runner-liveness.test.ts +865 -0
- package/src/core/__tests__/single-derivation.test.ts +159 -0
- package/src/core/config.ts +19 -0
- package/src/core/job-concurrency.ts +6 -1
- package/src/core/job-view.ts +329 -0
- package/src/core/orchestrator.ts +78 -6
- package/src/core/runner-liveness.ts +276 -0
- package/src/core/status-writer.ts +24 -5
- package/src/ui/client/__tests__/job-adapter.test.ts +78 -0
- package/src/ui/client/adapters/job-adapter.ts +10 -0
- package/src/ui/client/types.ts +2 -0
- package/src/ui/components/JobTable.tsx +29 -12
- package/src/ui/components/__tests__/JobTable.test.tsx +54 -1
- package/src/ui/components/types.ts +2 -0
- package/src/ui/dist/assets/{index--RH3sAt3.js → index-L6cvsCAx.js} +25 -12
- package/src/ui/dist/assets/{index--RH3sAt3.js.map → index-L6cvsCAx.js.map} +1 -1
- package/src/ui/dist/index.html +1 -1
- package/src/ui/embedded-assets.js +6 -6
- package/src/ui/server/__tests__/config-bridge.test.ts +9 -1
- package/src/ui/server/__tests__/job-control-endpoints.test.ts +57 -1
- package/src/ui/server/__tests__/job-endpoints.test.ts +115 -1
- package/src/ui/server/__tests__/sse-enhancer.test.ts +31 -0
- package/src/ui/server/config-bridge.ts +3 -4
- package/src/ui/server/endpoints/__tests__/meta-endpoint.test.ts +1 -0
- package/src/ui/server/endpoints/gate-endpoints.ts +2 -6
- package/src/ui/server/endpoints/job-control-endpoints.ts +7 -68
- package/src/ui/server/endpoints/job-endpoints.ts +6 -0
- package/src/ui/server/endpoints/meta-endpoint.ts +1 -1
- package/src/ui/state/__tests__/snapshot.test.ts +51 -0
- package/src/ui/state/__tests__/types.test.ts +98 -5
- package/src/ui/state/snapshot.ts +1 -1
- package/src/ui/state/transformers/__tests__/list-transformer.test.ts +43 -2
- package/src/ui/state/transformers/__tests__/status-transformer.test.ts +208 -22
- package/src/ui/state/transformers/list-transformer.ts +7 -3
- package/src/ui/state/transformers/status-transformer.ts +36 -170
- package/src/ui/state/types.ts +9 -47
package/src/ui/dist/index.html
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
/>
|
|
12
12
|
<title>Prompt Pipeline Dashboard</title>
|
|
13
13
|
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
|
14
|
-
<script type="module" crossorigin src="/assets/index
|
|
14
|
+
<script type="module" crossorigin src="/assets/index-L6cvsCAx.js"></script>
|
|
15
15
|
<link rel="stylesheet" crossorigin href="/assets/style-CSSKuMOe.css">
|
|
16
16
|
</head>
|
|
17
17
|
<body>
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
// Auto-generated by scripts/generate-embedded-assets.js — do not edit
|
|
2
2
|
import _asset0 from "./dist/index.html" with { type: "file" };
|
|
3
|
-
import _asset1 from "./dist/assets/
|
|
4
|
-
import _asset2 from "./dist/assets/
|
|
5
|
-
import _asset3 from "./dist/assets/index
|
|
3
|
+
import _asset1 from "./dist/assets/index-L6cvsCAx.js.map" with { type: "file" };
|
|
4
|
+
import _asset2 from "./dist/assets/style-CSSKuMOe.css" with { type: "file" };
|
|
5
|
+
import _asset3 from "./dist/assets/index-L6cvsCAx.js" with { type: "file" };
|
|
6
6
|
|
|
7
7
|
export const embeddedAssets = {
|
|
8
8
|
"/index.html": { path: _asset0, mime: "text/html" },
|
|
9
|
-
"/assets/
|
|
10
|
-
"/assets/
|
|
11
|
-
"/assets/index
|
|
9
|
+
"/assets/index-L6cvsCAx.js.map": { path: _asset1, mime: "application/json" },
|
|
10
|
+
"/assets/style-CSSKuMOe.css": { path: _asset2, mime: "text/css" },
|
|
11
|
+
"/assets/index-L6cvsCAx.js": { path: _asset3, mime: "application/javascript" }
|
|
12
12
|
};
|
|
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";
|
|
|
3
3
|
import {
|
|
4
4
|
Constants,
|
|
5
5
|
createErrorResponse,
|
|
6
|
+
determineJobStatus,
|
|
6
7
|
getStatusPriority,
|
|
7
8
|
validateJobId,
|
|
8
9
|
validateTaskState,
|
|
@@ -25,7 +26,7 @@ describe("config-bridge", () => {
|
|
|
25
26
|
|
|
26
27
|
it("returns status priorities", () => {
|
|
27
28
|
expect(getStatusPriority("running")).toBe(4);
|
|
28
|
-
expect(getStatusPriority("
|
|
29
|
+
expect(getStatusPriority("failed")).toBe(3);
|
|
29
30
|
expect(getStatusPriority("pending")).toBe(2);
|
|
30
31
|
expect(getStatusPriority("complete")).toBe(1);
|
|
31
32
|
expect(getStatusPriority("unknown")).toBe(0);
|
|
@@ -38,4 +39,11 @@ describe("config-bridge", () => {
|
|
|
38
39
|
message: "missing",
|
|
39
40
|
});
|
|
40
41
|
});
|
|
42
|
+
|
|
43
|
+
it("determines the canonical job status from task states", () => {
|
|
44
|
+
expect(determineJobStatus({ a: { state: "failed" } })).toBe("failed");
|
|
45
|
+
expect(determineJobStatus({ a: { state: "done" }, b: { state: "done" } })).toBe("complete");
|
|
46
|
+
expect(determineJobStatus({ a: { state: "running" } })).toBe("running");
|
|
47
|
+
expect(determineJobStatus({ a: { state: "pending" } })).toBe("pending");
|
|
48
|
+
});
|
|
41
49
|
});
|
|
@@ -7,11 +7,11 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
|
|
7
7
|
|
|
8
8
|
import { initPATHS, resetPATHS } from "../config-bridge-node";
|
|
9
9
|
import {
|
|
10
|
-
classifyStaleRunningStatus,
|
|
11
10
|
handleJobRestart,
|
|
12
11
|
handleJobStop,
|
|
13
12
|
handleTaskStart,
|
|
14
13
|
} from "../endpoints/job-control-endpoints";
|
|
14
|
+
import { classifyStaleRunningStatus } from "../../../core/runner-liveness";
|
|
15
15
|
import { resetConfig } from "../../../core/config";
|
|
16
16
|
import {
|
|
17
17
|
getConcurrencyRuntimePaths,
|
|
@@ -122,6 +122,7 @@ afterEach(async () => {
|
|
|
122
122
|
vi.restoreAllMocks();
|
|
123
123
|
resetPATHS();
|
|
124
124
|
delete process.env["PO_MAX_RUNNING_JOBS"];
|
|
125
|
+
delete process.env["PO_STALE_RUNNING_GRACE_MS"];
|
|
125
126
|
resetConfig();
|
|
126
127
|
for (const proc of childProcs.splice(0)) {
|
|
127
128
|
try { proc.kill(); } catch {}
|
|
@@ -566,6 +567,33 @@ describe("handleJobRestart", () => {
|
|
|
566
567
|
expect(body["code"]).toBe("job_running");
|
|
567
568
|
});
|
|
568
569
|
|
|
570
|
+
it("uses configured staleRunningGraceMs when deciding whether restart can recover a running task", async () => {
|
|
571
|
+
const root = await makeTempRoot();
|
|
572
|
+
initPATHS(root);
|
|
573
|
+
process.env["PO_STALE_RUNNING_GRACE_MS"] = "60000";
|
|
574
|
+
resetConfig();
|
|
575
|
+
|
|
576
|
+
vi.spyOn(Date, "now").mockReturnValue(Date.parse("2026-04-01T10:00:45.000Z"));
|
|
577
|
+
await setupJob(root, "restart-configured-grace", {
|
|
578
|
+
id: "restart-configured-grace",
|
|
579
|
+
state: "running",
|
|
580
|
+
current: "research",
|
|
581
|
+
currentStage: "prompt",
|
|
582
|
+
lastUpdated: "2026-04-01T10:00:00.000Z",
|
|
583
|
+
tasks: {
|
|
584
|
+
research: { state: "running", currentStage: "prompt" },
|
|
585
|
+
},
|
|
586
|
+
files: { artifacts: [], logs: [], tmp: [] },
|
|
587
|
+
}, 999999);
|
|
588
|
+
|
|
589
|
+
const req = new Request("http://localhost/api/jobs/restart-configured-grace/restart", { method: "POST" });
|
|
590
|
+
const res = await handleJobRestart(req, "restart-configured-grace", root);
|
|
591
|
+
const body = await res.json() as Record<string, unknown>;
|
|
592
|
+
|
|
593
|
+
expect(res.status).toBe(409);
|
|
594
|
+
expect(body["code"]).toBe("job_running");
|
|
595
|
+
});
|
|
596
|
+
|
|
569
597
|
it("proceeds when task-derived running status is stale and the runner is gone", async () => {
|
|
570
598
|
const root = await makeTempRoot();
|
|
571
599
|
initPATHS(root);
|
|
@@ -939,6 +967,34 @@ describe("handleTaskStart", () => {
|
|
|
939
967
|
expect(body["code"]).toBe("job_running");
|
|
940
968
|
});
|
|
941
969
|
|
|
970
|
+
it("uses configured staleRunningGraceMs when deciding whether task start can recover a running task", async () => {
|
|
971
|
+
const root = await makeTempRoot();
|
|
972
|
+
initPATHS(root);
|
|
973
|
+
process.env["PO_STALE_RUNNING_GRACE_MS"] = "60000";
|
|
974
|
+
resetConfig();
|
|
975
|
+
|
|
976
|
+
vi.spyOn(Date, "now").mockReturnValue(Date.parse("2026-04-01T10:00:45.000Z"));
|
|
977
|
+
await setupJob(root, "task-start-configured-grace", {
|
|
978
|
+
id: "task-start-configured-grace",
|
|
979
|
+
state: "running",
|
|
980
|
+
current: "research",
|
|
981
|
+
currentStage: "prompt",
|
|
982
|
+
lastUpdated: "2026-04-01T10:00:00.000Z",
|
|
983
|
+
tasks: {
|
|
984
|
+
research: { state: "running", currentStage: "prompt" },
|
|
985
|
+
analysis: { state: "pending", currentStage: null },
|
|
986
|
+
},
|
|
987
|
+
files: { artifacts: [], logs: [], tmp: [] },
|
|
988
|
+
}, 999999);
|
|
989
|
+
|
|
990
|
+
const req = new Request("http://localhost/api/jobs/task-start-configured-grace/tasks/analysis/start", { method: "POST" });
|
|
991
|
+
const res = await handleTaskStart(req, "task-start-configured-grace", "analysis", root);
|
|
992
|
+
const body = await res.json() as Record<string, unknown>;
|
|
993
|
+
|
|
994
|
+
expect(res.status).toBe(409);
|
|
995
|
+
expect(body["code"]).toBe("job_running");
|
|
996
|
+
});
|
|
997
|
+
|
|
942
998
|
it("returns 202 when stale-running status is the only blocker for an eligible pending task", async () => {
|
|
943
999
|
const root = await makeTempRoot();
|
|
944
1000
|
initPATHS(root);
|
|
@@ -5,7 +5,7 @@ import path from "node:path";
|
|
|
5
5
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
6
6
|
|
|
7
7
|
import { resetConfig } from "../../../core/config";
|
|
8
|
-
import { handleJobDetail } from "../endpoints/job-endpoints";
|
|
8
|
+
import { handleJobDetail, handleJobList } from "../endpoints/job-endpoints";
|
|
9
9
|
import { initPATHS, resetPATHS } from "../config-bridge-node";
|
|
10
10
|
|
|
11
11
|
const tempRoots: string[] = [];
|
|
@@ -122,3 +122,117 @@ describe("job endpoints pipeline config loading", () => {
|
|
|
122
122
|
expect(body.data.pipelineConfig?.["tasks"]).toEqual([{ name: "shared-task" }]);
|
|
123
123
|
});
|
|
124
124
|
});
|
|
125
|
+
|
|
126
|
+
describe("handleJobDetail corrupt status", () => {
|
|
127
|
+
it("returns 409 STATUS_CORRUPT with the corrupt tasks-status.json path", async () => {
|
|
128
|
+
const root = await makeTempRoot();
|
|
129
|
+
process.env["PO_ROOT"] = root;
|
|
130
|
+
resetConfig();
|
|
131
|
+
initPATHS(root);
|
|
132
|
+
await writePipelineRegistry(root);
|
|
133
|
+
const jobDir = await writeJob(root, "current", "job-corrupt");
|
|
134
|
+
await writeFile(path.join(jobDir, "tasks-status.json"), "{ not valid json");
|
|
135
|
+
|
|
136
|
+
const res = await handleJobDetail("job-corrupt");
|
|
137
|
+
const body = (await res.json()) as { code: string; path?: string };
|
|
138
|
+
|
|
139
|
+
expect(res.status).toBe(409);
|
|
140
|
+
expect(body.code).toBe("STATUS_CORRUPT");
|
|
141
|
+
expect(body.path).toBe(path.join(jobDir, "tasks-status.json"));
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
describe("canonical status in HTTP responses", () => {
|
|
146
|
+
it("handleJobDetail returns canonical status:'failed' for a failed-state fixture", async () => {
|
|
147
|
+
const root = await makeTempRoot();
|
|
148
|
+
process.env["PO_ROOT"] = root;
|
|
149
|
+
resetConfig();
|
|
150
|
+
initPATHS(root);
|
|
151
|
+
await writePipelineRegistry(root);
|
|
152
|
+
const jobDir = await writeJob(root, "current", "job-failed");
|
|
153
|
+
await writeFile(
|
|
154
|
+
path.join(jobDir, "tasks-status.json"),
|
|
155
|
+
JSON.stringify({
|
|
156
|
+
id: "job-failed",
|
|
157
|
+
name: "job-failed",
|
|
158
|
+
pipeline: "demo",
|
|
159
|
+
createdAt: "2026-06-12T12:00:00.000Z",
|
|
160
|
+
state: "failed",
|
|
161
|
+
tasks: { a: { state: "failed" } },
|
|
162
|
+
}),
|
|
163
|
+
);
|
|
164
|
+
|
|
165
|
+
const res = await handleJobDetail("job-failed");
|
|
166
|
+
const body = (await res.json()) as { data: { status: string } };
|
|
167
|
+
|
|
168
|
+
expect(res.status).toBe(200);
|
|
169
|
+
expect(body.data.status).toBe("failed");
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it("handleJobDetail returns canonical status:'complete' for a complete-state fixture", async () => {
|
|
173
|
+
const root = await makeTempRoot();
|
|
174
|
+
process.env["PO_ROOT"] = root;
|
|
175
|
+
resetConfig();
|
|
176
|
+
initPATHS(root);
|
|
177
|
+
await writePipelineRegistry(root);
|
|
178
|
+
const jobDir = await writeJob(root, "current", "job-complete");
|
|
179
|
+
await writeFile(
|
|
180
|
+
path.join(jobDir, "tasks-status.json"),
|
|
181
|
+
JSON.stringify({
|
|
182
|
+
id: "job-complete",
|
|
183
|
+
name: "job-complete",
|
|
184
|
+
pipeline: "demo",
|
|
185
|
+
createdAt: "2026-06-12T12:00:00.000Z",
|
|
186
|
+
state: "done",
|
|
187
|
+
tasks: { a: { state: "done" } },
|
|
188
|
+
}),
|
|
189
|
+
);
|
|
190
|
+
|
|
191
|
+
const res = await handleJobDetail("job-complete");
|
|
192
|
+
const body = (await res.json()) as { data: { status: string } };
|
|
193
|
+
|
|
194
|
+
expect(res.status).toBe(200);
|
|
195
|
+
expect(body.data.status).toBe("complete");
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it("handleJobList returns canonical status for every job in the listing", async () => {
|
|
199
|
+
const root = await makeTempRoot();
|
|
200
|
+
process.env["PO_ROOT"] = root;
|
|
201
|
+
resetConfig();
|
|
202
|
+
initPATHS(root);
|
|
203
|
+
await writePipelineRegistry(root);
|
|
204
|
+
const failedDir = await writeJob(root, "current", "job-list-failed");
|
|
205
|
+
await writeFile(
|
|
206
|
+
path.join(failedDir, "tasks-status.json"),
|
|
207
|
+
JSON.stringify({
|
|
208
|
+
id: "job-list-failed",
|
|
209
|
+
name: "job-list-failed",
|
|
210
|
+
pipeline: "demo",
|
|
211
|
+
createdAt: "2026-06-12T12:00:00.000Z",
|
|
212
|
+
state: "failed",
|
|
213
|
+
tasks: { a: { state: "failed" } },
|
|
214
|
+
}),
|
|
215
|
+
);
|
|
216
|
+
const completeDir = await writeJob(root, "complete", "job-list-complete");
|
|
217
|
+
await writeFile(
|
|
218
|
+
path.join(completeDir, "tasks-status.json"),
|
|
219
|
+
JSON.stringify({
|
|
220
|
+
id: "job-list-complete",
|
|
221
|
+
name: "job-list-complete",
|
|
222
|
+
pipeline: "demo",
|
|
223
|
+
createdAt: "2026-06-12T12:00:00.000Z",
|
|
224
|
+
state: "done",
|
|
225
|
+
tasks: { a: { state: "done" } },
|
|
226
|
+
}),
|
|
227
|
+
);
|
|
228
|
+
|
|
229
|
+
const res = await handleJobList();
|
|
230
|
+
const body = (await res.json()) as { data: Array<{ jobId: string; status: string }> };
|
|
231
|
+
|
|
232
|
+
expect(res.status).toBe(200);
|
|
233
|
+
expect(body.data).toHaveLength(2);
|
|
234
|
+
const byId = new Map(body.data.map((job) => [job.jobId, job.status]));
|
|
235
|
+
expect(byId.get("job-list-failed")).toBe("failed");
|
|
236
|
+
expect(byId.get("job-list-complete")).toBe("complete");
|
|
237
|
+
});
|
|
238
|
+
});
|
|
@@ -53,4 +53,35 @@ describe("sse-enhancer", () => {
|
|
|
53
53
|
enhancer.cleanup();
|
|
54
54
|
expect(enhancer.getPendingCount()).toBe(0);
|
|
55
55
|
});
|
|
56
|
+
|
|
57
|
+
it("broadcasts canonical failed status when the persisted state is 'failed'", async () => {
|
|
58
|
+
const readJobFn = vi.fn(async () => ({
|
|
59
|
+
ok: true as const,
|
|
60
|
+
jobId: "j1",
|
|
61
|
+
location: "current",
|
|
62
|
+
path: "/tmp/j1/tasks-status.json",
|
|
63
|
+
data: {
|
|
64
|
+
title: "Job 1",
|
|
65
|
+
state: "failed",
|
|
66
|
+
createdAt: "2024-01-01T00:00:00.000Z",
|
|
67
|
+
updatedAt: "2024-01-01T00:00:00.000Z",
|
|
68
|
+
tasks: { a: { state: "failed", files: {} } },
|
|
69
|
+
},
|
|
70
|
+
}));
|
|
71
|
+
const broadcast = vi.fn();
|
|
72
|
+
const enhancer = createSSEEnhancer({
|
|
73
|
+
readJobFn,
|
|
74
|
+
sseRegistry: { addClient() {}, removeClient() {}, broadcast, getClientCount() { return 0; }, closeAll() {} },
|
|
75
|
+
debounceMs: 100,
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
enhancer.handleJobChange({ jobId: "j1" });
|
|
79
|
+
vi.advanceTimersByTime(100);
|
|
80
|
+
await Promise.resolve();
|
|
81
|
+
|
|
82
|
+
expect(broadcast).toHaveBeenCalledTimes(1);
|
|
83
|
+
const [, payload] = broadcast.mock.calls[0] as [string, { status: string }];
|
|
84
|
+
expect(broadcast.mock.calls[0]?.[0]).toBe("job:created");
|
|
85
|
+
expect(payload.status).toBe("failed");
|
|
86
|
+
});
|
|
56
87
|
});
|
|
@@ -16,7 +16,7 @@ export const Constants = {
|
|
|
16
16
|
JOB_ID_REGEX: /^[A-Za-z0-9-_]+$/,
|
|
17
17
|
TASK_STATES: Object.freeze(Object.values(TaskState)),
|
|
18
18
|
JOB_LOCATIONS: Object.freeze(Object.values(JobLocation)),
|
|
19
|
-
STATUS_ORDER: Object.freeze(["running", "
|
|
19
|
+
STATUS_ORDER: Object.freeze(["running", "failed", "pending", "complete"]),
|
|
20
20
|
FILE_LIMITS: Object.freeze({ MAX_FILE_SIZE: 5 * 1024 * 1024 }),
|
|
21
21
|
RETRY_CONFIG: Object.freeze({ MAX_ATTEMPTS: 3, DELAY_MS: 10 }),
|
|
22
22
|
SSE_CONFIG: Object.freeze({ DEBOUNCE_MS: 200 }),
|
|
@@ -32,7 +32,7 @@ export const Constants = {
|
|
|
32
32
|
|
|
33
33
|
const STATUS_PRIORITY = new Map<string, number>([
|
|
34
34
|
["running", 4],
|
|
35
|
-
["
|
|
35
|
+
["failed", 3],
|
|
36
36
|
["pending", 2],
|
|
37
37
|
["complete", 1],
|
|
38
38
|
]);
|
|
@@ -53,8 +53,7 @@ export function determineJobStatus(tasks: Record<string, { state: string }>): st
|
|
|
53
53
|
const normalizedTasks = Object.values(tasks).map((task) => ({
|
|
54
54
|
state: task.state,
|
|
55
55
|
}));
|
|
56
|
-
|
|
57
|
-
return status === "failed" ? "error" : status;
|
|
56
|
+
return deriveJobStatusFromTasks(normalizedTasks);
|
|
58
57
|
}
|
|
59
58
|
|
|
60
59
|
export function createErrorResponse(
|
|
@@ -11,6 +11,7 @@ describe("handleMeta", () => {
|
|
|
11
11
|
const body = await res.json() as { ok: boolean; data: Record<string, unknown> };
|
|
12
12
|
expect(body.ok).toBe(true);
|
|
13
13
|
expect(body.data["protocolVersion"]).toBe(PROTOCOL_VERSION);
|
|
14
|
+
expect(body.data["protocolVersion"]).toBe(2);
|
|
14
15
|
});
|
|
15
16
|
|
|
16
17
|
it("serves the exact package.json version (drift guard)", async () => {
|
|
@@ -9,12 +9,8 @@ import { releaseJobSlot } from "../../../core/job-concurrency";
|
|
|
9
9
|
import { appendRunEvent } from "../../../core/run-events";
|
|
10
10
|
import { readJobStatus, writeJobStatus, StatusCorruptError } from "../../../core/status-writer";
|
|
11
11
|
import { assertStatusParseable, statusCorruptResponse } from "../utils/status-corruption";
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
isProcessAlive,
|
|
15
|
-
readRunnerPid,
|
|
16
|
-
spawnWithSlot,
|
|
17
|
-
} from "./job-control-endpoints";
|
|
12
|
+
import { acquireConcurrencySlot, spawnWithSlot } from "./job-control-endpoints";
|
|
13
|
+
import { isProcessAlive, readRunnerPid } from "../../../core/runner-liveness";
|
|
18
14
|
|
|
19
15
|
type GateAction = "approve" | "reject";
|
|
20
16
|
|
|
@@ -5,7 +5,6 @@ import { createErrorResponse } from "../config-bridge";
|
|
|
5
5
|
import { Constants } from "../config-bridge-node";
|
|
6
6
|
import { sendJson } from "../utils/http-utils";
|
|
7
7
|
import { getJobDirectoryPath, getPipelineDataDir } from "../../../config/paths";
|
|
8
|
-
import { deriveJobStatusFromTasks } from "../../../config/statuses";
|
|
9
8
|
import { getOrchestratorConfig, getPipelineConfig } from "../../../core/config";
|
|
10
9
|
import {
|
|
11
10
|
releaseJobSlot,
|
|
@@ -13,6 +12,11 @@ import {
|
|
|
13
12
|
updateJobSlotPid,
|
|
14
13
|
type JobSlotLease,
|
|
15
14
|
} from "../../../core/job-concurrency";
|
|
15
|
+
import {
|
|
16
|
+
classifyStaleRunningStatus,
|
|
17
|
+
isProcessAlive,
|
|
18
|
+
readRunnerPid,
|
|
19
|
+
} from "../../../core/runner-liveness";
|
|
16
20
|
import {
|
|
17
21
|
readJobStatus,
|
|
18
22
|
resetJobFromTask,
|
|
@@ -29,19 +33,6 @@ import {
|
|
|
29
33
|
import { readFileWithRetry } from "../file-reader";
|
|
30
34
|
|
|
31
35
|
const RUNNER_PATH = path.resolve(import.meta.dir, "../../../core/pipeline-runner.ts");
|
|
32
|
-
const STALE_RUNNING_GRACE_MS = 30_000;
|
|
33
|
-
|
|
34
|
-
interface StaleRunningInput {
|
|
35
|
-
status: StatusSnapshot;
|
|
36
|
-
pid: number | null;
|
|
37
|
-
pidAlive: boolean;
|
|
38
|
-
nowMs: number;
|
|
39
|
-
staleAfterMs: number;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
type StaleRunningDecision =
|
|
43
|
-
| { stale: false; reason: "not_running" | "live_pid" | "fresh_status" | "missing_last_updated" }
|
|
44
|
-
| { stale: true; runningTaskId: string | null; ageMs: number };
|
|
45
36
|
|
|
46
37
|
const restartingJobs = new Set<string>();
|
|
47
38
|
const stoppingJobs = new Set<string>();
|
|
@@ -174,25 +165,6 @@ export async function spawnWithSlot(
|
|
|
174
165
|
}
|
|
175
166
|
}
|
|
176
167
|
|
|
177
|
-
export async function readRunnerPid(jobDir: string): Promise<number | null> {
|
|
178
|
-
try {
|
|
179
|
-
const content = await Bun.file(path.join(jobDir, "runner.pid")).text();
|
|
180
|
-
const pid = parseInt(content.trim(), 10);
|
|
181
|
-
return Number.isNaN(pid) ? null : pid;
|
|
182
|
-
} catch {
|
|
183
|
-
return null;
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
export function isProcessAlive(pid: number): boolean {
|
|
188
|
-
try {
|
|
189
|
-
process.kill(pid, 0);
|
|
190
|
-
return true;
|
|
191
|
-
} catch {
|
|
192
|
-
return false;
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
|
|
196
168
|
const KILL_GRACE_MS = 1500;
|
|
197
169
|
const KILL_POLL_MS = 100;
|
|
198
170
|
|
|
@@ -208,39 +180,6 @@ function isDependencySatisfied(state: unknown): boolean {
|
|
|
208
180
|
return state === "done";
|
|
209
181
|
}
|
|
210
182
|
|
|
211
|
-
function getTaskDerivedStatus(snapshot: StatusSnapshot): string {
|
|
212
|
-
return deriveJobStatusFromTasks(
|
|
213
|
-
Object.values(snapshot.tasks).map((task) => ({ state: task.state })),
|
|
214
|
-
);
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
function findRunningTaskId(snapshot: StatusSnapshot): string | null {
|
|
218
|
-
if (snapshot.current && snapshot.tasks[snapshot.current]?.state === "running") {
|
|
219
|
-
return snapshot.current;
|
|
220
|
-
}
|
|
221
|
-
return Object.keys(snapshot.tasks).find((taskId) => snapshot.tasks[taskId]?.state === "running") ?? null;
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
export function classifyStaleRunningStatus(input: StaleRunningInput): StaleRunningDecision {
|
|
225
|
-
if (getTaskDerivedStatus(input.status) !== "running") {
|
|
226
|
-
return { stale: false, reason: "not_running" };
|
|
227
|
-
}
|
|
228
|
-
if (input.pid !== null && input.pidAlive) {
|
|
229
|
-
return { stale: false, reason: "live_pid" };
|
|
230
|
-
}
|
|
231
|
-
const updatedMs = Date.parse(input.status.lastUpdated);
|
|
232
|
-
if (!Number.isFinite(updatedMs)) {
|
|
233
|
-
// No live PID and no parseable timestamp: the runner crashed before writing a
|
|
234
|
-
// valid lastUpdated. There is no evidence of liveness, so treat as stale.
|
|
235
|
-
return { stale: true, runningTaskId: findRunningTaskId(input.status), ageMs: 0 };
|
|
236
|
-
}
|
|
237
|
-
const ageMs = input.nowMs - updatedMs;
|
|
238
|
-
if (ageMs < input.staleAfterMs) {
|
|
239
|
-
return { stale: false, reason: "fresh_status" };
|
|
240
|
-
}
|
|
241
|
-
return { stale: true, runningTaskId: findRunningTaskId(input.status), ageMs };
|
|
242
|
-
}
|
|
243
|
-
|
|
244
183
|
function isTaskLikelyInProgress(task: Record<string, unknown>): boolean {
|
|
245
184
|
const state = task["state"];
|
|
246
185
|
const nonTerminal = isNonTerminalTaskState(state);
|
|
@@ -552,7 +491,7 @@ export async function handleJobRestart(
|
|
|
552
491
|
pid,
|
|
553
492
|
pidAlive,
|
|
554
493
|
nowMs: Date.now(),
|
|
555
|
-
staleAfterMs:
|
|
494
|
+
staleAfterMs: getOrchestratorConfig().staleRunningGraceMs,
|
|
556
495
|
});
|
|
557
496
|
// Note: "live_pid" can never be returned here because pidAlive is guaranteed false
|
|
558
497
|
// by the early-return guard above; only "not_running", "fresh_status", or stale=true
|
|
@@ -764,7 +703,7 @@ export async function handleTaskStart(
|
|
|
764
703
|
pid,
|
|
765
704
|
pidAlive,
|
|
766
705
|
nowMs: Date.now(),
|
|
767
|
-
staleAfterMs:
|
|
706
|
+
staleAfterMs: getOrchestratorConfig().staleRunningGraceMs,
|
|
768
707
|
});
|
|
769
708
|
if (!staleDecision.stale && staleDecision.reason !== "not_running") {
|
|
770
709
|
return sendJson(409, createErrorResponse("job_running", "Job is currently running (task-level running)"));
|
|
@@ -60,6 +60,12 @@ export async function handleJobDetail(jobId: string): Promise<Response> {
|
|
|
60
60
|
|
|
61
61
|
const result = await readJob(jobId);
|
|
62
62
|
if (!result.ok) {
|
|
63
|
+
if (result.code === Constants.ERROR_CODES.INVALID_JSON) {
|
|
64
|
+
return sendJson(
|
|
65
|
+
409,
|
|
66
|
+
createErrorResponse(Constants.ERROR_CODES.STATUS_CORRUPT, result.message, result.path),
|
|
67
|
+
);
|
|
68
|
+
}
|
|
63
69
|
const status = result.code === Constants.ERROR_CODES.JOB_NOT_FOUND ? 404 : 400;
|
|
64
70
|
return sendJson(status, result);
|
|
65
71
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { describe, expect, it, vi } from "vitest";
|
|
2
2
|
|
|
3
3
|
import { buildSnapshotFromFilesystem, composeStateSnapshot } from "../snapshot";
|
|
4
|
+
import type { CanonicalJob, JobReadResult } from "../types";
|
|
4
5
|
|
|
5
6
|
describe("snapshot", () => {
|
|
6
7
|
it("composes default snapshots and extracts ids from variant fields", () => {
|
|
@@ -81,4 +82,54 @@ describe("snapshot", () => {
|
|
|
81
82
|
}),
|
|
82
83
|
).rejects.toThrow(/unavailable/);
|
|
83
84
|
});
|
|
85
|
+
|
|
86
|
+
it("sorts snapshot jobs with failed (former error) priority first", async () => {
|
|
87
|
+
const statusByJobId: Record<string, CanonicalJob["status"]> = {
|
|
88
|
+
"job-failed": "failed",
|
|
89
|
+
"job-running": "running",
|
|
90
|
+
"job-complete": "complete",
|
|
91
|
+
"job-pending": "pending",
|
|
92
|
+
};
|
|
93
|
+
const transformMultipleJobs = vi.fn((results: JobReadResult[]): CanonicalJob[] =>
|
|
94
|
+
results.map((result) => ({
|
|
95
|
+
id: result.jobId,
|
|
96
|
+
jobId: result.jobId,
|
|
97
|
+
name: result.jobId,
|
|
98
|
+
title: result.jobId,
|
|
99
|
+
status: statusByJobId[result.jobId] ?? "pending",
|
|
100
|
+
progress: 0,
|
|
101
|
+
createdAt: "2024-01-01T00:00:00.000Z",
|
|
102
|
+
updatedAt: "2024-01-01T00:00:00.000Z",
|
|
103
|
+
location: result.location,
|
|
104
|
+
tasks: {},
|
|
105
|
+
files: {},
|
|
106
|
+
costs: {},
|
|
107
|
+
})),
|
|
108
|
+
);
|
|
109
|
+
|
|
110
|
+
const snapshot = await buildSnapshotFromFilesystem({
|
|
111
|
+
listAllJobs: () => ({
|
|
112
|
+
current: ["job-pending", "job-running", "job-failed", "job-complete"],
|
|
113
|
+
complete: [],
|
|
114
|
+
}),
|
|
115
|
+
readJob: async (jobId, location) => ({ ok: true, jobId, location, data: {} }),
|
|
116
|
+
transformMultipleJobs,
|
|
117
|
+
now: () => new Date("2024-03-01T00:00:00.000Z"),
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
expect(transformMultipleJobs).toHaveBeenCalledTimes(1);
|
|
121
|
+
expect(snapshot.jobs.map((job) => job.jobId)).toEqual([
|
|
122
|
+
"job-failed",
|
|
123
|
+
"job-running",
|
|
124
|
+
"job-complete",
|
|
125
|
+
"job-pending",
|
|
126
|
+
]);
|
|
127
|
+
expect(snapshot.jobs.map((job) => job.status)).toEqual([
|
|
128
|
+
"failed",
|
|
129
|
+
"running",
|
|
130
|
+
"complete",
|
|
131
|
+
"pending",
|
|
132
|
+
]);
|
|
133
|
+
expect(snapshot.jobs[0]?.jobId).toBe("job-failed");
|
|
134
|
+
});
|
|
84
135
|
});
|