@ryanfw/prompt-orchestration-pipeline 1.2.7 → 1.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/config/__tests__/models.test.ts +31 -1
- package/src/config/models.ts +81 -35
- package/src/config/paths.ts +13 -8
- package/src/core/__tests__/config.test.ts +121 -0
- package/src/core/__tests__/job-concurrency.test.ts +554 -0
- package/src/core/__tests__/orchestrator.test.ts +353 -0
- package/src/core/__tests__/pipeline-runner.test.ts +430 -2
- package/src/core/__tests__/task-runner.test.ts +1 -2
- package/src/core/config.ts +48 -1
- package/src/core/job-concurrency.ts +462 -0
- package/src/core/orchestrator.ts +370 -57
- package/src/core/pipeline-runner.ts +79 -15
- package/src/core/status-writer.ts +4 -0
- package/src/core/task-runner.ts +1 -1
- package/src/providers/__tests__/base.test.ts +1 -1
- package/src/ui/client/__tests__/api.test.ts +101 -1
- package/src/ui/client/__tests__/job-adapter.test.ts +12 -0
- package/src/ui/client/__tests__/useConcurrencyStatus.test.ts +126 -0
- package/src/ui/client/adapters/job-adapter.ts +1 -0
- package/src/ui/client/api.ts +77 -7
- package/src/ui/client/hooks/useConcurrencyStatus.ts +102 -0
- package/src/ui/client/types.ts +34 -1
- package/src/ui/components/DAGGrid.tsx +11 -1
- package/src/ui/components/JobDetail.tsx +2 -1
- package/src/ui/components/__tests__/DAGGrid.test.tsx +92 -0
- package/src/ui/components/__tests__/JobDetail.test.tsx +62 -0
- package/src/ui/components/types.ts +2 -0
- package/src/ui/dist/assets/{index-SKy2shWc.js → index-BnAqY4_n.js} +336 -52
- package/src/ui/dist/assets/index-BnAqY4_n.js.map +1 -0
- package/src/ui/dist/assets/style-BKG0bHu-.css +2 -0
- package/src/ui/dist/index.html +2 -2
- package/src/ui/embedded-assets.js +6 -6
- package/src/ui/pages/PromptPipelineDashboard.tsx +186 -4
- package/src/ui/pages/__tests__/PromptPipelineDashboard.test.tsx +272 -1
- package/src/ui/server/__tests__/concurrency-endpoint.test.ts +190 -0
- package/src/ui/server/__tests__/index.test.ts +92 -3
- package/src/ui/server/__tests__/job-control-endpoints.test.ts +660 -3
- package/src/ui/server/endpoints/concurrency-endpoint.ts +72 -0
- package/src/ui/server/endpoints/job-control-endpoints.ts +248 -37
- package/src/ui/server/index.ts +21 -2
- package/src/ui/server/router.ts +2 -0
- package/src/ui/state/__tests__/watcher.test.ts +31 -0
- package/src/ui/state/transformers/__tests__/status-transformer.test.ts +15 -0
- package/src/ui/state/transformers/status-transformer.ts +1 -0
- package/src/ui/state/types.ts +3 -0
- package/src/ui/state/watcher.ts +9 -1
- package/src/utils/__tests__/dag.test.ts +35 -0
- package/src/utils/dag.ts +1 -0
- package/src/ui/dist/assets/index-SKy2shWc.js.map +0 -1
- package/src/ui/dist/assets/style-DA1Ma4YS.css +0 -2
|
@@ -14,6 +14,7 @@ export interface TaskEntry {
|
|
|
14
14
|
failedStage?: string;
|
|
15
15
|
error?: string;
|
|
16
16
|
attempts?: number;
|
|
17
|
+
restartCount?: number;
|
|
17
18
|
refinementAttempts?: number;
|
|
18
19
|
tokenUsage?: unknown[];
|
|
19
20
|
startedAt?: string;
|
|
@@ -227,6 +228,7 @@ export function resetJobFromTask(jobDir: string, fromTask: string, options?: Res
|
|
|
227
228
|
delete task.failedStage;
|
|
228
229
|
delete task.error;
|
|
229
230
|
task.attempts = 0;
|
|
231
|
+
task.restartCount = 0;
|
|
230
232
|
task.refinementAttempts = 0;
|
|
231
233
|
if (options?.clearTokenUsage !== false) {
|
|
232
234
|
task.tokenUsage = [];
|
|
@@ -253,6 +255,7 @@ export function resetJobToCleanSlate(jobDir: string, options?: ResetOptions): Pr
|
|
|
253
255
|
delete task.failedStage;
|
|
254
256
|
delete task.error;
|
|
255
257
|
task.attempts = 0;
|
|
258
|
+
task.restartCount = 0;
|
|
256
259
|
task.refinementAttempts = 0;
|
|
257
260
|
if (options?.clearTokenUsage !== false) {
|
|
258
261
|
task.tokenUsage = [];
|
|
@@ -280,6 +283,7 @@ export function resetSingleTask(jobDir: string, taskId: string, options?: ResetO
|
|
|
280
283
|
delete task.failedStage;
|
|
281
284
|
delete task.error;
|
|
282
285
|
task.attempts = 0;
|
|
286
|
+
task.restartCount = 0;
|
|
283
287
|
task.refinementAttempts = 0;
|
|
284
288
|
if (options?.clearTokenUsage !== false) {
|
|
285
289
|
task.tokenUsage = [];
|
package/src/core/task-runner.ts
CHANGED
|
@@ -795,7 +795,7 @@ export async function runPipeline(
|
|
|
795
795
|
|
|
796
796
|
// Write done status (best-effort)
|
|
797
797
|
try {
|
|
798
|
-
const lastStage = KNOWN_STAGES
|
|
798
|
+
const lastStage = KNOWN_STAGES.at(-1)!;
|
|
799
799
|
const doneProgress = computeDeterministicProgress(
|
|
800
800
|
pipelineTasks ?? [taskName],
|
|
801
801
|
taskName,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
2
2
|
|
|
3
|
-
import { restartJob, stopJob } from "../api";
|
|
3
|
+
import { fetchConcurrencyStatus, restartJob, startTask, stopJob } from "../api";
|
|
4
|
+
import type { JobConcurrencyApiStatus } from "../types";
|
|
4
5
|
|
|
5
6
|
const fetchMock = vi.fn<typeof fetch>();
|
|
6
7
|
const originalFetch = globalThis.fetch;
|
|
@@ -43,6 +44,46 @@ describe("ui client api", () => {
|
|
|
43
44
|
});
|
|
44
45
|
});
|
|
45
46
|
|
|
47
|
+
it("preserves concurrency_limit_reached on restart and surfaces capacity message", async () => {
|
|
48
|
+
fetchMock.mockResolvedValue(
|
|
49
|
+
new Response(
|
|
50
|
+
JSON.stringify({
|
|
51
|
+
ok: false,
|
|
52
|
+
code: "concurrency_limit_reached",
|
|
53
|
+
message: "concurrency limit reached",
|
|
54
|
+
}),
|
|
55
|
+
{ status: 409 },
|
|
56
|
+
),
|
|
57
|
+
);
|
|
58
|
+
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
|
59
|
+
|
|
60
|
+
await expect(restartJob("job-1")).rejects.toMatchObject({
|
|
61
|
+
code: "concurrency_limit_reached",
|
|
62
|
+
status: 409,
|
|
63
|
+
message: "Capacity reached. Wait for a job to finish, then try again.",
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it("preserves concurrency_limit_reached on task start and surfaces capacity message", async () => {
|
|
68
|
+
fetchMock.mockResolvedValue(
|
|
69
|
+
new Response(
|
|
70
|
+
JSON.stringify({
|
|
71
|
+
ok: false,
|
|
72
|
+
code: "concurrency_limit_reached",
|
|
73
|
+
message: "concurrency limit reached",
|
|
74
|
+
}),
|
|
75
|
+
{ status: 409 },
|
|
76
|
+
),
|
|
77
|
+
);
|
|
78
|
+
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
|
79
|
+
|
|
80
|
+
await expect(startTask("job-1", "task-1")).rejects.toMatchObject({
|
|
81
|
+
code: "concurrency_limit_reached",
|
|
82
|
+
status: 409,
|
|
83
|
+
message: "Capacity reached. Wait for a job to finish, then try again.",
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
46
87
|
it("returns ok responses for stop", async () => {
|
|
47
88
|
fetchMock.mockResolvedValue(new Response(JSON.stringify({ ok: true }), { status: 200 }));
|
|
48
89
|
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
|
@@ -59,4 +100,63 @@ describe("ui client api", () => {
|
|
|
59
100
|
message: "offline",
|
|
60
101
|
});
|
|
61
102
|
});
|
|
103
|
+
|
|
104
|
+
describe("fetchConcurrencyStatus", () => {
|
|
105
|
+
const sampleStatus: JobConcurrencyApiStatus = {
|
|
106
|
+
limit: 3,
|
|
107
|
+
runningCount: 1,
|
|
108
|
+
availableSlots: 2,
|
|
109
|
+
queuedCount: 1,
|
|
110
|
+
activeJobs: [
|
|
111
|
+
{ jobId: "job-1", pid: 1234, acquiredAt: "2024-01-01T00:00:00Z", source: "orchestrator" },
|
|
112
|
+
],
|
|
113
|
+
queuedJobs: [
|
|
114
|
+
{ jobId: "job-2", queuedAt: "2024-01-01T00:00:01Z", name: "second", pipeline: "demo" },
|
|
115
|
+
],
|
|
116
|
+
staleSlots: [],
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
it("parses a successful response and unwraps data", async () => {
|
|
120
|
+
fetchMock.mockResolvedValue(
|
|
121
|
+
new Response(JSON.stringify({ ok: true, data: sampleStatus }), { status: 200 }),
|
|
122
|
+
);
|
|
123
|
+
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
|
124
|
+
|
|
125
|
+
await expect(fetchConcurrencyStatus()).resolves.toEqual(sampleStatus);
|
|
126
|
+
expect(fetchMock).toHaveBeenCalledWith("/api/concurrency", expect.objectContaining({}));
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it("maps network failures to network_error", async () => {
|
|
130
|
+
fetchMock.mockRejectedValue(new Error("offline"));
|
|
131
|
+
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
|
132
|
+
|
|
133
|
+
await expect(fetchConcurrencyStatus()).rejects.toMatchObject({
|
|
134
|
+
code: "network_error",
|
|
135
|
+
message: "offline",
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it("propagates abort signals to fetch", async () => {
|
|
140
|
+
fetchMock.mockImplementation((_url, init) => {
|
|
141
|
+
return new Promise((_resolve, reject) => {
|
|
142
|
+
const signal = (init as RequestInit | undefined)?.signal;
|
|
143
|
+
if (signal?.aborted) {
|
|
144
|
+
reject(new DOMException("Aborted", "AbortError"));
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
signal?.addEventListener("abort", () => {
|
|
148
|
+
reject(new DOMException("Aborted", "AbortError"));
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
});
|
|
152
|
+
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
|
153
|
+
|
|
154
|
+
const controller = new AbortController();
|
|
155
|
+
const promise = fetchConcurrencyStatus(controller.signal);
|
|
156
|
+
controller.abort();
|
|
157
|
+
|
|
158
|
+
await expect(promise).rejects.toMatchObject({ name: "AbortError" });
|
|
159
|
+
expect(fetchMock.mock.calls[0]?.[1]?.signal).toBe(controller.signal);
|
|
160
|
+
});
|
|
161
|
+
});
|
|
62
162
|
});
|
|
@@ -30,6 +30,18 @@ describe("job adapter", () => {
|
|
|
30
30
|
expect(normalizeTasks(null)).toEqual({});
|
|
31
31
|
});
|
|
32
32
|
|
|
33
|
+
it("maps numeric restartCount onto the normalized task", () => {
|
|
34
|
+
expect(normalizeTasks({ t1: { state: "done", restartCount: 2 } })["t1"]?.restartCount).toBe(2);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it("treats null restartCount as undefined", () => {
|
|
38
|
+
expect(normalizeTasks({ t1: { state: "done", restartCount: null } })["t1"]?.restartCount).toBeUndefined();
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("leaves restartCount undefined when absent", () => {
|
|
42
|
+
expect(normalizeTasks({ t1: { state: "done" } })["t1"]?.restartCount).toBeUndefined();
|
|
43
|
+
});
|
|
44
|
+
|
|
33
45
|
it("adapts summary jobs with defaults", () => {
|
|
34
46
|
const job = adaptJobSummary({
|
|
35
47
|
jobId: "job-1",
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import "../../components/__tests__/test-dom";
|
|
2
|
+
|
|
3
|
+
// @vitest-environment jsdom
|
|
4
|
+
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
5
|
+
import { act, renderHook, waitFor } from "@testing-library/react";
|
|
6
|
+
|
|
7
|
+
import { useConcurrencyStatus } from "../hooks/useConcurrencyStatus";
|
|
8
|
+
import type { JobConcurrencyApiStatus } from "../types";
|
|
9
|
+
|
|
10
|
+
const originalFetch = globalThis.fetch;
|
|
11
|
+
const originalEventSource = globalThis.EventSource;
|
|
12
|
+
|
|
13
|
+
type Listener = (event: MessageEvent<string>) => void;
|
|
14
|
+
|
|
15
|
+
class MockEventSource {
|
|
16
|
+
public readonly url: string;
|
|
17
|
+
public closed = false;
|
|
18
|
+
public listeners = new Map<string, Listener>();
|
|
19
|
+
|
|
20
|
+
constructor(url: string) {
|
|
21
|
+
this.url = url;
|
|
22
|
+
MockEventSource.instances.push(this);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
addEventListener(type: string, listener: Listener): void {
|
|
26
|
+
this.listeners.set(type, listener);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
close(): void {
|
|
30
|
+
this.closed = true;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
emit(type: string, data: unknown): void {
|
|
34
|
+
const listener = this.listeners.get(type);
|
|
35
|
+
if (!listener) return;
|
|
36
|
+
listener(new MessageEvent(type, { data: JSON.stringify(data) }));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
static instances: MockEventSource[] = [];
|
|
40
|
+
static reset(): void {
|
|
41
|
+
MockEventSource.instances = [];
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const sampleStatus: JobConcurrencyApiStatus = {
|
|
46
|
+
limit: 3,
|
|
47
|
+
runningCount: 0,
|
|
48
|
+
availableSlots: 3,
|
|
49
|
+
queuedCount: 0,
|
|
50
|
+
activeJobs: [],
|
|
51
|
+
queuedJobs: [],
|
|
52
|
+
staleSlots: [],
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
function makeStatus(overrides: Partial<JobConcurrencyApiStatus> = {}): JobConcurrencyApiStatus {
|
|
56
|
+
return { ...sampleStatus, ...overrides };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function jsonResponse(body: unknown, status = 200): Response {
|
|
60
|
+
return new Response(JSON.stringify(body), { status });
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
describe("useConcurrencyStatus", () => {
|
|
64
|
+
afterEach(() => {
|
|
65
|
+
globalThis.fetch = originalFetch;
|
|
66
|
+
globalThis.EventSource = originalEventSource;
|
|
67
|
+
MockEventSource.reset();
|
|
68
|
+
vi.restoreAllMocks();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("populates data after the initial fetch", async () => {
|
|
72
|
+
const status = makeStatus({ runningCount: 1, availableSlots: 2 });
|
|
73
|
+
globalThis.fetch = vi.fn<typeof fetch>().mockResolvedValue(
|
|
74
|
+
jsonResponse({ ok: true, data: status }),
|
|
75
|
+
) as unknown as typeof fetch;
|
|
76
|
+
globalThis.EventSource = MockEventSource as unknown as typeof EventSource;
|
|
77
|
+
|
|
78
|
+
const { result } = renderHook(() => useConcurrencyStatus());
|
|
79
|
+
|
|
80
|
+
await waitFor(() => expect(result.current.loading).toBe(false));
|
|
81
|
+
expect(result.current.data).toEqual(status);
|
|
82
|
+
expect(result.current.error).toBeNull();
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it("refetches on state:summary SSE events", async () => {
|
|
86
|
+
const first = makeStatus({ runningCount: 1 });
|
|
87
|
+
const second = makeStatus({ runningCount: 2 });
|
|
88
|
+
const fetchMock = vi.fn<typeof fetch>()
|
|
89
|
+
.mockResolvedValueOnce(jsonResponse({ ok: true, data: first }))
|
|
90
|
+
.mockResolvedValueOnce(jsonResponse({ ok: true, data: second }));
|
|
91
|
+
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
|
92
|
+
globalThis.EventSource = MockEventSource as unknown as typeof EventSource;
|
|
93
|
+
|
|
94
|
+
const { result } = renderHook(() => useConcurrencyStatus());
|
|
95
|
+
|
|
96
|
+
await waitFor(() => expect(result.current.data).toEqual(first));
|
|
97
|
+
|
|
98
|
+
act(() => {
|
|
99
|
+
MockEventSource.instances[0]?.emit("state:summary", { ok: true });
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
await waitFor(() => expect(result.current.data).toEqual(second));
|
|
103
|
+
expect(fetchMock).toHaveBeenCalledTimes(2);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it("aborts the in-flight request on unmount", async () => {
|
|
107
|
+
let capturedSignal: AbortSignal | undefined;
|
|
108
|
+
const fetchMock = vi.fn<typeof fetch>().mockImplementation((_url, init) => {
|
|
109
|
+
capturedSignal = (init as RequestInit | undefined)?.signal ?? undefined;
|
|
110
|
+
return new Promise((_resolve, reject) => {
|
|
111
|
+
capturedSignal?.addEventListener("abort", () => {
|
|
112
|
+
reject(new DOMException("Aborted", "AbortError"));
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
|
117
|
+
globalThis.EventSource = MockEventSource as unknown as typeof EventSource;
|
|
118
|
+
|
|
119
|
+
const { unmount } = renderHook(() => useConcurrencyStatus());
|
|
120
|
+
|
|
121
|
+
expect(capturedSignal?.aborted).toBe(false);
|
|
122
|
+
unmount();
|
|
123
|
+
expect(capturedSignal?.aborted).toBe(true);
|
|
124
|
+
expect(MockEventSource.instances[0]?.closed).toBe(true);
|
|
125
|
+
});
|
|
126
|
+
});
|
|
@@ -49,6 +49,7 @@ function normalizeTask(name: string, rawTask: unknown): NormalizedTask {
|
|
|
49
49
|
startedAt: toStringOrNull(task["startedAt"]),
|
|
50
50
|
endedAt: toStringOrNull(task["endedAt"]),
|
|
51
51
|
attempts: typeof task["attempts"] === "number" ? task["attempts"] : undefined,
|
|
52
|
+
restartCount: typeof task["restartCount"] === "number" ? task["restartCount"] : undefined,
|
|
52
53
|
executionTimeMs: typeof task["executionTimeMs"] === "number" ? task["executionTimeMs"] : undefined,
|
|
53
54
|
currentStage: typeof task["currentStage"] === "string" ? task["currentStage"] : undefined,
|
|
54
55
|
failedStage: typeof task["failedStage"] === "string" ? task["failedStage"] : undefined,
|
package/src/ui/client/api.ts
CHANGED
|
@@ -2,9 +2,12 @@ import type {
|
|
|
2
2
|
ApiError,
|
|
3
3
|
ApiErrorCode,
|
|
4
4
|
ApiOkResponse,
|
|
5
|
+
JobConcurrencyApiStatus,
|
|
5
6
|
RestartJobOptions,
|
|
6
7
|
} from "./types";
|
|
7
8
|
|
|
9
|
+
const CONCURRENCY_LIMIT_MESSAGE = "Capacity reached. Wait for a job to finish, then try again.";
|
|
10
|
+
|
|
8
11
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
9
12
|
return typeof value === "object" && value !== null;
|
|
10
13
|
}
|
|
@@ -22,10 +25,12 @@ function normalizeBackendErrorCode(errorData: unknown, status: number): ApiError
|
|
|
22
25
|
code === "spawn_failed" ||
|
|
23
26
|
code === "unknown_error" ||
|
|
24
27
|
code === "network_error" ||
|
|
28
|
+
code === "malformed_response" ||
|
|
25
29
|
code === "dependencies_not_satisfied" ||
|
|
26
30
|
code === "unsupported_lifecycle" ||
|
|
27
31
|
code === "task_not_found" ||
|
|
28
|
-
code === "task_not_pending"
|
|
32
|
+
code === "task_not_pending" ||
|
|
33
|
+
code === "concurrency_limit_reached"
|
|
29
34
|
) {
|
|
30
35
|
return code;
|
|
31
36
|
}
|
|
@@ -45,6 +50,10 @@ function getMessage(value: unknown): string | null {
|
|
|
45
50
|
return typeof value["message"] === "string" ? value["message"] : null;
|
|
46
51
|
}
|
|
47
52
|
|
|
53
|
+
function hasBackendCode(errorData: unknown, code: string): boolean {
|
|
54
|
+
return isRecord(errorData) && errorData["code"] === code;
|
|
55
|
+
}
|
|
56
|
+
|
|
48
57
|
export function getErrorCodeFromStatus(status: number): ApiErrorCode {
|
|
49
58
|
if (status === 404) return "job_not_found";
|
|
50
59
|
if (status === 409) return "conflict";
|
|
@@ -66,25 +75,31 @@ export function getErrorMessageFromStatus(status: number): string {
|
|
|
66
75
|
}
|
|
67
76
|
|
|
68
77
|
export function getRestartErrorMessage(errorData: unknown, status: number): string {
|
|
69
|
-
if (
|
|
78
|
+
if (hasBackendCode(errorData, "job_running")) {
|
|
70
79
|
return "Cannot restart a job while it is still running";
|
|
71
80
|
}
|
|
72
|
-
if (
|
|
81
|
+
if (hasBackendCode(errorData, "spawn_failed")) {
|
|
73
82
|
return "Failed to spawn the restarted job";
|
|
74
83
|
}
|
|
84
|
+
if (hasBackendCode(errorData, "concurrency_limit_reached")) {
|
|
85
|
+
return CONCURRENCY_LIMIT_MESSAGE;
|
|
86
|
+
}
|
|
75
87
|
return getMessage(errorData) ?? getErrorMessageFromStatus(status);
|
|
76
88
|
}
|
|
77
89
|
|
|
78
90
|
export function getStartTaskErrorMessage(errorData: unknown, status: number): string {
|
|
79
|
-
if (
|
|
91
|
+
if (hasBackendCode(errorData, "dependencies_not_satisfied")) {
|
|
80
92
|
return "Cannot start task before its dependencies are complete";
|
|
81
93
|
}
|
|
82
|
-
if (
|
|
94
|
+
if (hasBackendCode(errorData, "task_not_found")) {
|
|
83
95
|
return "Task not found";
|
|
84
96
|
}
|
|
85
|
-
if (
|
|
97
|
+
if (hasBackendCode(errorData, "task_not_pending")) {
|
|
86
98
|
return "Only pending tasks can be started";
|
|
87
99
|
}
|
|
100
|
+
if (hasBackendCode(errorData, "concurrency_limit_reached")) {
|
|
101
|
+
return CONCURRENCY_LIMIT_MESSAGE;
|
|
102
|
+
}
|
|
88
103
|
return getMessage(errorData) ?? getErrorMessageFromStatus(status);
|
|
89
104
|
}
|
|
90
105
|
|
|
@@ -101,7 +116,8 @@ export function getStopErrorMessage(errorData: unknown, status: number): string
|
|
|
101
116
|
async function parseJson(response: Response): Promise<unknown> {
|
|
102
117
|
try {
|
|
103
118
|
return await response.json();
|
|
104
|
-
} catch {
|
|
119
|
+
} catch (error) {
|
|
120
|
+
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
105
121
|
return null;
|
|
106
122
|
}
|
|
107
123
|
}
|
|
@@ -170,3 +186,57 @@ export async function startTask(jobId: string, taskId: string): Promise<ApiOkRes
|
|
|
170
186
|
export async function stopJob(jobId: string): Promise<ApiOkResponse> {
|
|
171
187
|
return postJson(`/api/jobs/${jobId}/stop`, {}, getStopErrorMessage);
|
|
172
188
|
}
|
|
189
|
+
|
|
190
|
+
export async function fetchConcurrencyStatus(
|
|
191
|
+
signal?: AbortSignal,
|
|
192
|
+
): Promise<JobConcurrencyApiStatus> {
|
|
193
|
+
let response: Response;
|
|
194
|
+
let payload: unknown;
|
|
195
|
+
|
|
196
|
+
try {
|
|
197
|
+
response = await fetch("/api/concurrency", { signal });
|
|
198
|
+
if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
|
|
199
|
+
payload = await parseJson(response);
|
|
200
|
+
} catch (error) {
|
|
201
|
+
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
202
|
+
throw {
|
|
203
|
+
code: "network_error",
|
|
204
|
+
message: error instanceof Error ? error.message : "Network request failed",
|
|
205
|
+
} satisfies ApiError;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (!response.ok) {
|
|
209
|
+
throw toApiError(
|
|
210
|
+
response.status,
|
|
211
|
+
getMessage(payload) ?? getErrorMessageFromStatus(response.status),
|
|
212
|
+
payload,
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
if (
|
|
217
|
+
isRecord(payload) &&
|
|
218
|
+
payload["ok"] === true &&
|
|
219
|
+
isJobConcurrencyApiStatus(payload["data"])
|
|
220
|
+
) {
|
|
221
|
+
return payload["data"];
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
throw {
|
|
225
|
+
code: "malformed_response",
|
|
226
|
+
message: "Malformed concurrency status response",
|
|
227
|
+
status: response.status,
|
|
228
|
+
} satisfies ApiError;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function isJobConcurrencyApiStatus(value: unknown): value is JobConcurrencyApiStatus {
|
|
232
|
+
if (!isRecord(value)) return false;
|
|
233
|
+
return (
|
|
234
|
+
typeof value["limit"] === "number" &&
|
|
235
|
+
typeof value["runningCount"] === "number" &&
|
|
236
|
+
typeof value["availableSlots"] === "number" &&
|
|
237
|
+
typeof value["queuedCount"] === "number" &&
|
|
238
|
+
Array.isArray(value["activeJobs"]) &&
|
|
239
|
+
Array.isArray(value["queuedJobs"]) &&
|
|
240
|
+
Array.isArray(value["staleSlots"])
|
|
241
|
+
);
|
|
242
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { useCallback, useEffect, useRef, useState } from "react";
|
|
2
|
+
|
|
3
|
+
import { fetchConcurrencyStatus } from "../api";
|
|
4
|
+
import type {
|
|
5
|
+
ApiError,
|
|
6
|
+
JobConcurrencyApiStatus,
|
|
7
|
+
UseConcurrencyStatusResult,
|
|
8
|
+
} from "../types";
|
|
9
|
+
|
|
10
|
+
const REFETCH_EVENTS = ["state:summary", "state:change"] as const;
|
|
11
|
+
|
|
12
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
13
|
+
return typeof value === "object" && value !== null;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function toApiError(error: unknown): ApiError {
|
|
17
|
+
if (isRecord(error) && typeof error["code"] === "string" && typeof error["message"] === "string") {
|
|
18
|
+
return {
|
|
19
|
+
code: error["code"] as ApiError["code"],
|
|
20
|
+
message: error["message"],
|
|
21
|
+
status: typeof error["status"] === "number" ? error["status"] : undefined,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
return {
|
|
25
|
+
code: "unknown_error",
|
|
26
|
+
message: error instanceof Error ? error.message : "Failed to load concurrency status",
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function useConcurrencyStatus(): UseConcurrencyStatusResult {
|
|
31
|
+
const [loading, setLoading] = useState(true);
|
|
32
|
+
const [data, setData] = useState<JobConcurrencyApiStatus | null>(null);
|
|
33
|
+
const [error, setError] = useState<ApiError | null>(null);
|
|
34
|
+
const abortRef = useRef<AbortController | null>(null);
|
|
35
|
+
const dataRef = useRef<JobConcurrencyApiStatus | null>(null);
|
|
36
|
+
const mountedRef = useRef(true);
|
|
37
|
+
const refetchTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
|
38
|
+
|
|
39
|
+
const load = useCallback((showLoading = false) => {
|
|
40
|
+
abortRef.current?.abort();
|
|
41
|
+
const controller = new AbortController();
|
|
42
|
+
abortRef.current = controller;
|
|
43
|
+
if (showLoading || dataRef.current === null) setLoading(true);
|
|
44
|
+
|
|
45
|
+
void fetchConcurrencyStatus(controller.signal)
|
|
46
|
+
.then((status) => {
|
|
47
|
+
if (!mountedRef.current || abortRef.current !== controller) return;
|
|
48
|
+
dataRef.current = status;
|
|
49
|
+
setData(status);
|
|
50
|
+
setError(null);
|
|
51
|
+
})
|
|
52
|
+
.catch((fetchError) => {
|
|
53
|
+
if (fetchError instanceof DOMException && fetchError.name === "AbortError") return;
|
|
54
|
+
if (!mountedRef.current || abortRef.current !== controller) return;
|
|
55
|
+
setError(toApiError(fetchError));
|
|
56
|
+
})
|
|
57
|
+
.finally(() => {
|
|
58
|
+
if (!mountedRef.current || abortRef.current !== controller) return;
|
|
59
|
+
setLoading(false);
|
|
60
|
+
});
|
|
61
|
+
}, []);
|
|
62
|
+
|
|
63
|
+
useEffect(() => {
|
|
64
|
+
mountedRef.current = true;
|
|
65
|
+
load(true);
|
|
66
|
+
|
|
67
|
+
const source = new EventSource("/api/events");
|
|
68
|
+
const onMessage = () => {
|
|
69
|
+
if (refetchTimerRef.current) return;
|
|
70
|
+
refetchTimerRef.current = setTimeout(() => {
|
|
71
|
+
refetchTimerRef.current = null;
|
|
72
|
+
load(false);
|
|
73
|
+
}, 0);
|
|
74
|
+
};
|
|
75
|
+
for (const eventName of REFETCH_EVENTS) {
|
|
76
|
+
source.addEventListener(eventName, onMessage as EventListener);
|
|
77
|
+
}
|
|
78
|
+
source.onerror = () => {
|
|
79
|
+
if (!mountedRef.current) return;
|
|
80
|
+
setError({
|
|
81
|
+
code: "network_error",
|
|
82
|
+
message: "Live concurrency updates disconnected",
|
|
83
|
+
});
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
return () => {
|
|
87
|
+
mountedRef.current = false;
|
|
88
|
+
if (refetchTimerRef.current) {
|
|
89
|
+
clearTimeout(refetchTimerRef.current);
|
|
90
|
+
refetchTimerRef.current = null;
|
|
91
|
+
}
|
|
92
|
+
abortRef.current?.abort();
|
|
93
|
+
source.close();
|
|
94
|
+
};
|
|
95
|
+
}, [load]);
|
|
96
|
+
|
|
97
|
+
const refetch = useCallback(() => {
|
|
98
|
+
load(true);
|
|
99
|
+
}, [load]);
|
|
100
|
+
|
|
101
|
+
return { loading, data, error, refetch };
|
|
102
|
+
}
|
package/src/ui/client/types.ts
CHANGED
|
@@ -5,10 +5,12 @@ export type ApiErrorCode =
|
|
|
5
5
|
| "spawn_failed"
|
|
6
6
|
| "unknown_error"
|
|
7
7
|
| "network_error"
|
|
8
|
+
| "malformed_response"
|
|
8
9
|
| "dependencies_not_satisfied"
|
|
9
10
|
| "unsupported_lifecycle"
|
|
10
11
|
| "task_not_found"
|
|
11
|
-
| "task_not_pending"
|
|
12
|
+
| "task_not_pending"
|
|
13
|
+
| "concurrency_limit_reached";
|
|
12
14
|
|
|
13
15
|
export interface ApiError {
|
|
14
16
|
code: ApiErrorCode;
|
|
@@ -89,6 +91,7 @@ export interface NormalizedTask {
|
|
|
89
91
|
startedAt: string | null;
|
|
90
92
|
endedAt: string | null;
|
|
91
93
|
attempts?: number;
|
|
94
|
+
restartCount?: number;
|
|
92
95
|
executionTimeMs?: number;
|
|
93
96
|
currentStage?: string;
|
|
94
97
|
failedStage?: string;
|
|
@@ -184,6 +187,36 @@ export interface AllowedActions {
|
|
|
184
187
|
restart: boolean;
|
|
185
188
|
}
|
|
186
189
|
|
|
190
|
+
export interface JobConcurrencyApiStatus {
|
|
191
|
+
limit: number;
|
|
192
|
+
runningCount: number;
|
|
193
|
+
availableSlots: number;
|
|
194
|
+
queuedCount: number;
|
|
195
|
+
activeJobs: Array<{
|
|
196
|
+
jobId: string;
|
|
197
|
+
pid: number | null;
|
|
198
|
+
acquiredAt: string;
|
|
199
|
+
source: "orchestrator" | "restart" | "task-start";
|
|
200
|
+
}>;
|
|
201
|
+
queuedJobs: Array<{
|
|
202
|
+
jobId: string;
|
|
203
|
+
queuedAt: string | null;
|
|
204
|
+
name: string | null;
|
|
205
|
+
pipeline: string | null;
|
|
206
|
+
}>;
|
|
207
|
+
staleSlots: Array<{
|
|
208
|
+
jobId: string;
|
|
209
|
+
reason: "missing_current_job" | "missing_pid" | "dead_pid" | "invalid_json";
|
|
210
|
+
}>;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export interface UseConcurrencyStatusResult {
|
|
214
|
+
loading: boolean;
|
|
215
|
+
data: JobConcurrencyApiStatus | null;
|
|
216
|
+
error: ApiError | null;
|
|
217
|
+
refetch: () => void;
|
|
218
|
+
}
|
|
219
|
+
|
|
187
220
|
export interface SseJobEvent {
|
|
188
221
|
type: SseEventType;
|
|
189
222
|
data: Record<string, unknown>;
|
|
@@ -276,7 +276,17 @@ export default function DAGGrid({
|
|
|
276
276
|
data-role="card-header"
|
|
277
277
|
className={`flex items-center justify-between gap-3 rounded-t-lg border-b px-4 py-2 ${reducedMotion ? "" : "transition-opacity duration-300 ease-in-out"} ${getHeaderClasses(getItemStatus(items[index], index, activeIndex))}`}
|
|
278
278
|
>
|
|
279
|
-
|
|
279
|
+
{(items[index]?.restartCount ?? 0) > 0 ? (
|
|
280
|
+
<span
|
|
281
|
+
data-role="restart-badge"
|
|
282
|
+
className="inline-flex items-center justify-center min-w-[20px] h-5 px-1.5 rounded-sm border border-red-300 bg-red-50 text-red-700 text-[11px] font-medium tabular-nums"
|
|
283
|
+
title={`Restarted ${items[index]!.restartCount} time${items[index]!.restartCount === 1 ? "" : "s"}`}
|
|
284
|
+
aria-label={`Restarted ${items[index]!.restartCount} time${items[index]!.restartCount === 1 ? "" : "s"}`}
|
|
285
|
+
>
|
|
286
|
+
↻ {items[index]!.restartCount}
|
|
287
|
+
</span>
|
|
288
|
+
) : null}
|
|
289
|
+
<div className="min-w-0 flex-1 truncate text-left font-medium">{items[index]?.title ?? formatStepName(items[index]?.id ?? "")}</div>
|
|
280
290
|
<div className="flex items-center gap-2">
|
|
281
291
|
{getItemStatus(items[index], index, activeIndex) === "running" ? (
|
|
282
292
|
<>
|
|
@@ -43,7 +43,8 @@ export default function JobDetail({
|
|
|
43
43
|
prior.stage === item.stage &&
|
|
44
44
|
prior.title === item.title &&
|
|
45
45
|
prior.subtitle === item.subtitle &&
|
|
46
|
-
prior.body === item.body
|
|
46
|
+
prior.body === item.body &&
|
|
47
|
+
prior.restartCount === item.restartCount
|
|
47
48
|
) {
|
|
48
49
|
return prior;
|
|
49
50
|
}
|