@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/http-api.md +80 -4
- package/package.json +1 -4
- package/src/api/__tests__/index.test.ts +443 -32
- package/src/api/index.ts +108 -127
- package/src/cli/__tests__/analyze-task.test.ts +40 -7
- package/src/cli/__tests__/index.test.ts +337 -17
- package/src/cli/__tests__/types.test.ts +0 -18
- package/src/cli/__tests__/update-pipeline-json.test.ts +19 -9
- package/src/cli/analyze-task.ts +3 -14
- package/src/cli/index.ts +73 -61
- package/src/cli/types.ts +0 -13
- package/src/cli/update-pipeline-json.ts +3 -14
- package/src/config/__tests__/paths.test.ts +46 -1
- package/src/config/__tests__/pipeline-registry.test.ts +767 -0
- package/src/config/__tests__/sse-events.test.ts +470 -0
- package/src/config/__tests__/statuses.test.ts +41 -0
- package/src/config/paths.ts +11 -2
- package/src/config/pipeline-registry.ts +258 -0
- package/src/config/sse-events.ts +122 -0
- package/src/config/statuses.ts +20 -1
- package/src/core/__tests__/agent-step.test.ts +115 -0
- package/src/core/__tests__/config.test.ts +545 -105
- package/src/core/__tests__/core-boot-resolution.test.ts +181 -0
- package/src/core/__tests__/job-concurrency.test.ts +260 -0
- package/src/core/__tests__/job-submission.test.ts +396 -0
- package/src/core/__tests__/job-view.test.ts +1341 -0
- package/src/core/__tests__/json-file.test.ts +111 -0
- package/src/core/__tests__/lifecycle-policy.test.ts +81 -7
- package/src/core/__tests__/logger.test.ts +22 -0
- package/src/core/__tests__/orchestrator-no-deprecated-exports.test.ts +41 -0
- package/src/core/__tests__/orchestrator.test.ts +615 -2
- package/src/core/__tests__/pipeline-runner.test.ts +946 -9
- package/src/core/__tests__/redact.test.ts +153 -0
- package/src/core/__tests__/runner-liveness.test.ts +910 -0
- package/src/core/__tests__/seed-naming.test.ts +57 -0
- package/src/core/__tests__/single-derivation.test.ts +159 -0
- package/src/core/__tests__/task-runner.test.ts +594 -3
- package/src/core/__tests__/task-telemetry.test.ts +241 -0
- package/src/core/__tests__/workspace-root-resolution-guard.test.ts +62 -0
- package/src/core/agent-step.ts +4 -1
- package/src/core/agent-types.ts +5 -4
- package/src/core/config.ts +134 -222
- package/src/core/control.ts +3 -0
- package/src/core/job-concurrency.ts +56 -20
- package/src/core/job-submission.ts +133 -0
- package/src/core/job-view.ts +473 -0
- package/src/core/json-file.ts +45 -0
- package/src/core/lifecycle-policy.ts +23 -8
- package/src/core/logger.ts +0 -29
- package/src/core/orchestrator.ts +200 -74
- package/src/core/pipeline-runner.ts +85 -53
- package/src/core/redact.ts +40 -0
- package/src/core/runner-liveness.ts +280 -0
- package/src/core/seed-naming.ts +9 -0
- package/src/core/status-writer.ts +27 -33
- package/src/core/task-runner.ts +356 -319
- package/src/core/task-telemetry.ts +107 -0
- package/src/harness/__tests__/subprocess.test.ts +112 -1
- package/src/harness/subprocess.ts +55 -16
- package/src/llm/__tests__/index.test.ts +684 -33
- package/src/llm/index.ts +310 -67
- package/src/providers/__tests__/alibaba.test.ts +67 -6
- package/src/providers/__tests__/anthropic.test.ts +35 -14
- package/src/providers/__tests__/base.test.ts +62 -0
- package/src/providers/__tests__/claude-code.test.ts +99 -14
- package/src/providers/__tests__/deepseek.test.ts +16 -6
- package/src/providers/__tests__/gemini.test.ts +47 -25
- package/src/providers/__tests__/moonshot.test.ts +27 -0
- package/src/providers/__tests__/openai.test.ts +262 -74
- package/src/providers/__tests__/opencode.test.ts +77 -0
- package/src/providers/__tests__/request-timeout-contract.test.ts +226 -0
- package/src/providers/__tests__/stream-accumulator.test.ts +52 -0
- package/src/providers/__tests__/types.test.ts +85 -0
- package/src/providers/__tests__/zero-fill-guard.test.ts +62 -0
- package/src/providers/__tests__/zhipu.test.ts +27 -14
- package/src/providers/alibaba.ts +20 -39
- package/src/providers/anthropic.ts +23 -11
- package/src/providers/base.ts +19 -0
- package/src/providers/claude-code.ts +27 -18
- package/src/providers/deepseek.ts +9 -28
- package/src/providers/gemini.ts +20 -58
- package/src/providers/moonshot.ts +15 -11
- package/src/providers/openai.ts +79 -61
- package/src/providers/opencode.ts +16 -33
- package/src/providers/stream-accumulator.ts +27 -21
- package/src/providers/types.ts +29 -4
- package/src/providers/zhipu.ts +15 -44
- package/src/task-analysis/__tests__/analyzer.test.ts +0 -0
- package/src/task-analysis/__tests__/enrichers-schema-deducer.test.ts +34 -2
- package/src/task-analysis/__tests__/no-ast-imports.test.ts +52 -0
- package/src/task-analysis/__tests__/repository.test.ts +65 -0
- package/src/task-analysis/__tests__/types.test.ts +63 -130
- package/src/task-analysis/analyzer.ts +91 -0
- package/src/task-analysis/enrichers/schema-deducer.ts +13 -2
- package/src/task-analysis/index.ts +2 -36
- package/src/task-analysis/repository.ts +45 -0
- package/src/task-analysis/types.ts +42 -58
- package/src/ui/client/__tests__/api.test.ts +143 -1
- package/src/ui/client/__tests__/bootstrap.test.ts +178 -62
- package/src/ui/client/__tests__/job-adapter.test.ts +203 -2
- package/src/ui/client/__tests__/load-state.test.ts +70 -0
- package/src/ui/client/__tests__/types.test.ts +66 -3
- package/src/ui/client/__tests__/useJobDetailWithUpdates.test.ts +390 -77
- package/src/ui/client/__tests__/useJobList.test.ts +198 -23
- package/src/ui/client/__tests__/useJobListWithUpdates.test.ts +218 -7
- package/src/ui/client/adapters/__tests__/job-adapter.test.ts +186 -0
- package/src/ui/client/adapters/job-adapter.ts +41 -16
- package/src/ui/client/api.ts +38 -15
- package/src/ui/client/bootstrap.ts +19 -14
- package/src/ui/client/hooks/useJobDetailWithUpdates.ts +73 -97
- package/src/ui/client/hooks/useJobList.ts +26 -31
- package/src/ui/client/hooks/useJobListWithUpdates.ts +42 -76
- package/src/ui/client/load-state.ts +20 -0
- package/src/ui/client/reducers/__tests__/job-events.test.ts +568 -0
- package/src/ui/client/reducers/job-events.ts +137 -0
- package/src/ui/client/types.ts +16 -20
- package/src/ui/components/DAGGrid.tsx +6 -1
- package/src/ui/components/JobDetail.tsx +12 -4
- package/src/ui/components/JobTable.tsx +41 -13
- package/src/ui/components/StageTimeline.tsx +8 -20
- package/src/ui/components/TaskAnalysisDisplay.tsx +48 -6
- package/src/ui/components/__tests__/DAGGrid.test.tsx +36 -0
- package/src/ui/components/__tests__/JobDetail.test.tsx +112 -0
- package/src/ui/components/__tests__/JobTable.test.tsx +137 -1
- package/src/ui/components/__tests__/StageTimeline.test.tsx +5 -6
- package/src/ui/components/__tests__/TaskAnalysisDisplay.test.tsx +64 -0
- package/src/ui/components/types.ts +35 -15
- package/src/ui/dist/assets/{index--RH3sAt3.js → index-DN3-zvtP.js} +4324 -263
- package/src/ui/dist/assets/index-DN3-zvtP.js.map +1 -0
- package/src/ui/dist/assets/style-CtZBnjlR.css +2 -0
- package/src/ui/dist/index.html +2 -2
- package/src/ui/pages/PipelineDetail.tsx +60 -4
- package/src/ui/pages/PromptPipelineDashboard.tsx +59 -7
- package/src/ui/pages/__tests__/PromptPipelineDashboard.test.tsx +114 -32
- package/src/ui/pages/__tests__/pages.test.tsx +236 -42
- package/src/ui/server/__tests__/concurrency-endpoint.test.ts +54 -0
- package/src/ui/server/__tests__/config-bridge-node.test.ts +34 -1
- package/src/ui/server/__tests__/config-bridge.test.ts +9 -1
- package/src/ui/server/__tests__/embedded-assets.test.ts +66 -0
- package/src/ui/server/__tests__/file-endpoints.test.ts +183 -5
- package/src/ui/server/__tests__/gate-endpoints.test.ts +55 -0
- package/src/ui/server/__tests__/index.test.ts +63 -0
- package/src/ui/server/__tests__/job-control-endpoints.test.ts +512 -2
- package/src/ui/server/__tests__/job-endpoints.test.ts +158 -2
- package/src/ui/server/__tests__/read-static-path-guard.test.ts +94 -0
- package/src/ui/server/__tests__/router-root-isolation.test.ts +204 -0
- package/src/ui/server/__tests__/router-threading.test.ts +148 -0
- package/src/ui/server/__tests__/router.test.ts +104 -0
- package/src/ui/server/__tests__/sse-broadcast.test.ts +44 -0
- package/src/ui/server/__tests__/sse-enhancer.test.ts +70 -0
- package/src/ui/server/config-bridge-node.ts +8 -26
- package/src/ui/server/config-bridge.ts +3 -4
- package/src/ui/server/embedded-assets-imports.d.ts +44 -0
- package/src/ui/server/embedded-assets.ts +13 -2
- package/src/ui/server/endpoints/__tests__/meta-endpoint.test.ts +1 -0
- package/src/ui/server/endpoints/__tests__/pipeline-analysis-endpoint.test.ts +167 -0
- package/src/ui/server/endpoints/__tests__/schema-file-endpoint.test.ts +104 -0
- package/src/ui/server/endpoints/__tests__/task-analysis-endpoint.test.ts +103 -0
- package/src/ui/server/endpoints/__tests__/upload-endpoints.test.ts +162 -96
- package/src/ui/server/endpoints/concurrency-endpoint.ts +2 -1
- package/src/ui/server/endpoints/create-pipeline-endpoint.ts +14 -29
- package/src/ui/server/endpoints/file-endpoints.ts +15 -10
- package/src/ui/server/endpoints/gate-endpoints.ts +2 -6
- package/src/ui/server/endpoints/job-control-endpoints.ts +129 -141
- package/src/ui/server/endpoints/job-endpoints.ts +7 -0
- package/src/ui/server/endpoints/meta-endpoint.ts +1 -1
- package/src/ui/server/endpoints/pipeline-analysis-endpoint.ts +99 -11
- package/src/ui/server/endpoints/schema-file-endpoint.ts +11 -3
- package/src/ui/server/endpoints/task-analysis-endpoint.ts +21 -5
- package/src/ui/server/endpoints/task-save-endpoint.ts +1 -2
- package/src/ui/server/endpoints/upload-endpoints.ts +5 -40
- package/src/ui/server/index.ts +19 -14
- package/src/ui/server/job-reader.ts +14 -2
- package/src/ui/server/router.ts +33 -10
- package/src/ui/server/sse-broadcast.ts +14 -3
- package/src/ui/server/sse-enhancer.ts +12 -2
- package/src/ui/state/__tests__/schema-loader.test.ts +11 -1
- package/src/ui/state/__tests__/snapshot.test.ts +120 -14
- package/src/ui/state/__tests__/types.test.ts +104 -5
- package/src/ui/state/snapshot.ts +2 -2
- package/src/ui/state/transformers/__tests__/list-transformer.test.ts +85 -2
- package/src/ui/state/transformers/__tests__/status-transformer.test.ts +250 -22
- package/src/ui/state/transformers/list-transformer.ts +13 -3
- package/src/ui/state/transformers/status-transformer.ts +36 -170
- package/src/ui/state/types.ts +15 -48
- package/src/utils/__tests__/path-containment.test.ts +160 -0
- package/src/{ui/server/utils → utils}/path-containment.ts +21 -0
- package/src/task-analysis/__tests__/enrichers-analysis-writer.test.ts +0 -86
- package/src/task-analysis/__tests__/enrichers-artifact-resolver.test.ts +0 -124
- package/src/task-analysis/__tests__/extractors-artifacts.test.ts +0 -133
- package/src/task-analysis/__tests__/extractors-llm-calls.test.ts +0 -46
- package/src/task-analysis/__tests__/extractors-stages.test.ts +0 -52
- package/src/task-analysis/__tests__/index.test.ts +0 -143
- package/src/task-analysis/__tests__/parser.test.ts +0 -41
- package/src/task-analysis/__tests__/utils-ast.test.ts +0 -82
- package/src/task-analysis/enrichers/analysis-writer.ts +0 -75
- package/src/task-analysis/enrichers/artifact-resolver.ts +0 -89
- package/src/task-analysis/extractors/artifacts.ts +0 -143
- package/src/task-analysis/extractors/llm-calls.ts +0 -117
- package/src/task-analysis/extractors/stages.ts +0 -45
- package/src/task-analysis/parser.ts +0 -20
- package/src/task-analysis/utils/ast.ts +0 -45
- package/src/ui/dist/assets/index--RH3sAt3.js.map +0 -1
- package/src/ui/dist/assets/style-CSSKuMOe.css +0 -2
- package/src/ui/embedded-assets.js +0 -12
- package/src/ui/server/__tests__/path-containment.test.ts +0 -54
|
@@ -3,6 +3,7 @@ import path from "node:path";
|
|
|
3
3
|
import { readJob } from "../job-reader";
|
|
4
4
|
import { sendJson } from "../utils/http-utils";
|
|
5
5
|
import { getMimeType, isTextMime } from "../utils/mime-types";
|
|
6
|
+
import { resolveWithin } from "../../../utils/path-containment";
|
|
6
7
|
|
|
7
8
|
export function validateFileName(filename: string): boolean {
|
|
8
9
|
return Boolean(
|
|
@@ -20,8 +21,7 @@ type TaskFiles = {
|
|
|
20
21
|
tmp: string[];
|
|
21
22
|
};
|
|
22
23
|
|
|
23
|
-
function getJobFilesBase(location: string, jobId: string): string {
|
|
24
|
-
const root = process.env["PO_ROOT"] ?? process.cwd();
|
|
24
|
+
function getJobFilesBase(location: string, jobId: string, root: string): string {
|
|
25
25
|
return path.join(root, "pipeline-data", location, jobId, "files");
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -49,25 +49,27 @@ function getTaskFiles(data: Record<string, unknown>, taskId: string): TaskFiles
|
|
|
49
49
|
};
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
export async function handleTaskFileList(req: Request, jobId: string, taskId: string): Promise<Response> {
|
|
52
|
+
export async function handleTaskFileList(req: Request, jobId: string, taskId: string, root: string): Promise<Response> {
|
|
53
53
|
const type = new URL(req.url).searchParams.get("type") ?? "artifacts";
|
|
54
54
|
if (!validateFileName(type)) return sendJson(400, { ok: false, code: "BAD_REQUEST", message: "invalid file type" });
|
|
55
|
-
const job = await readJob(jobId);
|
|
55
|
+
const job = await readJob(jobId, root);
|
|
56
56
|
if (!job.ok) return sendJson(404, job);
|
|
57
57
|
|
|
58
58
|
const files = getTaskFiles(job.data, taskId);
|
|
59
59
|
return sendJson(200, { ok: true, data: files[type as keyof TaskFiles] ?? [] });
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
-
export async function handleTaskFile(req: Request, jobId: string, taskId: string): Promise<Response> {
|
|
62
|
+
export async function handleTaskFile(req: Request, jobId: string, taskId: string, root: string): Promise<Response> {
|
|
63
63
|
const url = new URL(req.url);
|
|
64
64
|
const type = url.searchParams.get("type") ?? "artifacts";
|
|
65
65
|
const filename = url.searchParams.get("filename") ?? "";
|
|
66
|
-
|
|
66
|
+
const job = await readJob(jobId, root);
|
|
67
|
+
if (!job.ok) return sendJson(404, job);
|
|
68
|
+
|
|
69
|
+
const filePath = resolveWithin(getJobFilesBase(job.location, jobId, root), path.join(type, filename));
|
|
70
|
+
if (filePath === null) {
|
|
67
71
|
return sendJson(400, { ok: false, code: "BAD_REQUEST", message: "invalid filename" });
|
|
68
72
|
}
|
|
69
|
-
const job = await readJob(jobId);
|
|
70
|
-
if (!job.ok) return sendJson(404, job);
|
|
71
73
|
|
|
72
74
|
const taskFiles = getTaskFiles(job.data, taskId);
|
|
73
75
|
const allowedFiles = taskFiles[type as keyof TaskFiles] ?? [];
|
|
@@ -75,9 +77,12 @@ export async function handleTaskFile(req: Request, jobId: string, taskId: string
|
|
|
75
77
|
return sendJson(404, { ok: false, code: "NOT_FOUND", message: `file not found for task: ${type}/${filename}` });
|
|
76
78
|
}
|
|
77
79
|
|
|
78
|
-
const
|
|
80
|
+
const file = Bun.file(filePath);
|
|
81
|
+
if (!(await file.exists())) {
|
|
82
|
+
return sendJson(404, { ok: false, code: "NOT_FOUND", message: `file not found for task: ${type}/${filename}` });
|
|
83
|
+
}
|
|
79
84
|
const mime = getMimeType(filename);
|
|
80
|
-
const buffer = new Uint8Array(await
|
|
85
|
+
const buffer = new Uint8Array(await file.arrayBuffer());
|
|
81
86
|
return sendJson(200, {
|
|
82
87
|
ok: true,
|
|
83
88
|
data: isTextMime(mime) ? new TextDecoder().decode(buffer) : Buffer.from(buffer).toString("base64"),
|
|
@@ -9,12 +9,8 @@ import { releaseJobSlot } from "../../../core/job-concurrency";
|
|
|
9
9
|
import { appendRunEvent } from "../../../core/run-events";
|
|
10
10
|
import { readJobStatus, writeJobStatus, StatusCorruptError } from "../../../core/status-writer";
|
|
11
11
|
import { assertStatusParseable, statusCorruptResponse } from "../utils/status-corruption";
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
isProcessAlive,
|
|
15
|
-
readRunnerPid,
|
|
16
|
-
spawnWithSlot,
|
|
17
|
-
} from "./job-control-endpoints";
|
|
12
|
+
import { acquireConcurrencySlot, spawnWithSlot } from "./job-control-endpoints";
|
|
13
|
+
import { isProcessAlive, readRunnerPid } from "../../../core/runner-liveness";
|
|
18
14
|
|
|
19
15
|
type GateAction = "approve" | "reject";
|
|
20
16
|
|
|
@@ -5,7 +5,6 @@ import { createErrorResponse } from "../config-bridge";
|
|
|
5
5
|
import { Constants } from "../config-bridge-node";
|
|
6
6
|
import { sendJson } from "../utils/http-utils";
|
|
7
7
|
import { getJobDirectoryPath, getPipelineDataDir } from "../../../config/paths";
|
|
8
|
-
import { deriveJobStatusFromTasks } from "../../../config/statuses";
|
|
9
8
|
import { getOrchestratorConfig, getPipelineConfig } from "../../../core/config";
|
|
10
9
|
import {
|
|
11
10
|
releaseJobSlot,
|
|
@@ -13,6 +12,11 @@ import {
|
|
|
13
12
|
updateJobSlotPid,
|
|
14
13
|
type JobSlotLease,
|
|
15
14
|
} from "../../../core/job-concurrency";
|
|
15
|
+
import {
|
|
16
|
+
classifyStaleRunningStatus,
|
|
17
|
+
isProcessAlive,
|
|
18
|
+
readRunnerPid,
|
|
19
|
+
} from "../../../core/runner-liveness";
|
|
16
20
|
import {
|
|
17
21
|
readJobStatus,
|
|
18
22
|
resetJobFromTask,
|
|
@@ -22,6 +26,10 @@ import {
|
|
|
22
26
|
StatusCorruptError,
|
|
23
27
|
type StatusSnapshot,
|
|
24
28
|
} from "../../../core/status-writer";
|
|
29
|
+
import {
|
|
30
|
+
areDependenciesSatisfied,
|
|
31
|
+
firstUnsatisfiedDependency,
|
|
32
|
+
} from "../../../core/lifecycle-policy";
|
|
25
33
|
import { assertKnownJobStatusesParseable, statusCorruptResponse } from "../utils/status-corruption";
|
|
26
34
|
import {
|
|
27
35
|
materializeNormalizedPipelineDefinition,
|
|
@@ -29,29 +37,21 @@ import {
|
|
|
29
37
|
import { readFileWithRetry } from "../file-reader";
|
|
30
38
|
|
|
31
39
|
const RUNNER_PATH = path.resolve(import.meta.dir, "../../../core/pipeline-runner.ts");
|
|
32
|
-
const STALE_RUNNING_GRACE_MS = 30_000;
|
|
33
|
-
|
|
34
|
-
interface StaleRunningInput {
|
|
35
|
-
status: StatusSnapshot;
|
|
36
|
-
pid: number | null;
|
|
37
|
-
pidAlive: boolean;
|
|
38
|
-
nowMs: number;
|
|
39
|
-
staleAfterMs: number;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
type StaleRunningDecision =
|
|
43
|
-
| { stale: false; reason: "not_running" | "live_pid" | "fresh_status" | "missing_last_updated" }
|
|
44
|
-
| { stale: true; runningTaskId: string | null; ageMs: number };
|
|
45
40
|
|
|
46
41
|
const restartingJobs = new Set<string>();
|
|
47
42
|
const stoppingJobs = new Set<string>();
|
|
48
43
|
const startingJobs = new Set<string>();
|
|
49
44
|
|
|
50
|
-
function getOrchestratorRuntimeConfig(): {
|
|
45
|
+
function getOrchestratorRuntimeConfig(): {
|
|
46
|
+
maxConcurrentJobs: number;
|
|
47
|
+
lockFileTimeout: number;
|
|
48
|
+
staleLeaseTimeoutMs: number;
|
|
49
|
+
} {
|
|
51
50
|
const orchestrator = getOrchestratorConfig();
|
|
52
51
|
return {
|
|
53
52
|
maxConcurrentJobs: orchestrator.maxConcurrentJobs,
|
|
54
53
|
lockFileTimeout: orchestrator.lockFileTimeout,
|
|
54
|
+
staleLeaseTimeoutMs: orchestrator.staleLeaseTimeoutMs,
|
|
55
55
|
};
|
|
56
56
|
}
|
|
57
57
|
|
|
@@ -111,13 +111,14 @@ export async function acquireConcurrencySlot(
|
|
|
111
111
|
jobId: string,
|
|
112
112
|
source: JobSlotLease["source"],
|
|
113
113
|
): Promise<{ ok: true } | { ok: false; response: Response }> {
|
|
114
|
-
const { maxConcurrentJobs, lockFileTimeout } = getOrchestratorRuntimeConfig();
|
|
114
|
+
const { maxConcurrentJobs, lockFileTimeout, staleLeaseTimeoutMs } = getOrchestratorRuntimeConfig();
|
|
115
115
|
const result = await tryAcquireJobSlot({
|
|
116
116
|
dataDir: getPipelineDataDir(dataDir),
|
|
117
117
|
jobId,
|
|
118
118
|
maxConcurrentJobs,
|
|
119
119
|
source,
|
|
120
120
|
lockTimeoutMs: lockFileTimeout,
|
|
121
|
+
staleLeaseTimeoutMs,
|
|
121
122
|
});
|
|
122
123
|
if (result.ok) return { ok: true };
|
|
123
124
|
if (result.reason === "already_held") {
|
|
@@ -174,25 +175,6 @@ export async function spawnWithSlot(
|
|
|
174
175
|
}
|
|
175
176
|
}
|
|
176
177
|
|
|
177
|
-
export async function readRunnerPid(jobDir: string): Promise<number | null> {
|
|
178
|
-
try {
|
|
179
|
-
const content = await Bun.file(path.join(jobDir, "runner.pid")).text();
|
|
180
|
-
const pid = parseInt(content.trim(), 10);
|
|
181
|
-
return Number.isNaN(pid) ? null : pid;
|
|
182
|
-
} catch {
|
|
183
|
-
return null;
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
export function isProcessAlive(pid: number): boolean {
|
|
188
|
-
try {
|
|
189
|
-
process.kill(pid, 0);
|
|
190
|
-
return true;
|
|
191
|
-
} catch {
|
|
192
|
-
return false;
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
|
|
196
178
|
const KILL_GRACE_MS = 1500;
|
|
197
179
|
const KILL_POLL_MS = 100;
|
|
198
180
|
|
|
@@ -204,43 +186,6 @@ function isNonTerminalTaskState(state: unknown): boolean {
|
|
|
204
186
|
return state === "pending" || state === "running";
|
|
205
187
|
}
|
|
206
188
|
|
|
207
|
-
function isDependencySatisfied(state: unknown): boolean {
|
|
208
|
-
return state === "done";
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
function getTaskDerivedStatus(snapshot: StatusSnapshot): string {
|
|
212
|
-
return deriveJobStatusFromTasks(
|
|
213
|
-
Object.values(snapshot.tasks).map((task) => ({ state: task.state })),
|
|
214
|
-
);
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
function findRunningTaskId(snapshot: StatusSnapshot): string | null {
|
|
218
|
-
if (snapshot.current && snapshot.tasks[snapshot.current]?.state === "running") {
|
|
219
|
-
return snapshot.current;
|
|
220
|
-
}
|
|
221
|
-
return Object.keys(snapshot.tasks).find((taskId) => snapshot.tasks[taskId]?.state === "running") ?? null;
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
export function classifyStaleRunningStatus(input: StaleRunningInput): StaleRunningDecision {
|
|
225
|
-
if (getTaskDerivedStatus(input.status) !== "running") {
|
|
226
|
-
return { stale: false, reason: "not_running" };
|
|
227
|
-
}
|
|
228
|
-
if (input.pid !== null && input.pidAlive) {
|
|
229
|
-
return { stale: false, reason: "live_pid" };
|
|
230
|
-
}
|
|
231
|
-
const updatedMs = Date.parse(input.status.lastUpdated);
|
|
232
|
-
if (!Number.isFinite(updatedMs)) {
|
|
233
|
-
// No live PID and no parseable timestamp: the runner crashed before writing a
|
|
234
|
-
// valid lastUpdated. There is no evidence of liveness, so treat as stale.
|
|
235
|
-
return { stale: true, runningTaskId: findRunningTaskId(input.status), ageMs: 0 };
|
|
236
|
-
}
|
|
237
|
-
const ageMs = input.nowMs - updatedMs;
|
|
238
|
-
if (ageMs < input.staleAfterMs) {
|
|
239
|
-
return { stale: false, reason: "fresh_status" };
|
|
240
|
-
}
|
|
241
|
-
return { stale: true, runningTaskId: findRunningTaskId(input.status), ageMs };
|
|
242
|
-
}
|
|
243
|
-
|
|
244
189
|
function isTaskLikelyInProgress(task: Record<string, unknown>): boolean {
|
|
245
190
|
const state = task["state"];
|
|
246
191
|
const nonTerminal = isNonTerminalTaskState(state);
|
|
@@ -552,7 +497,7 @@ export async function handleJobRestart(
|
|
|
552
497
|
pid,
|
|
553
498
|
pidAlive,
|
|
554
499
|
nowMs: Date.now(),
|
|
555
|
-
staleAfterMs:
|
|
500
|
+
staleAfterMs: getOrchestratorConfig().staleRunningGraceMs,
|
|
556
501
|
});
|
|
557
502
|
// Note: "live_pid" can never be returned here because pidAlive is guaranteed false
|
|
558
503
|
// by the early-return guard above; only "not_running", "fresh_status", or stale=true
|
|
@@ -562,6 +507,31 @@ export async function handleJobRestart(
|
|
|
562
507
|
}
|
|
563
508
|
}
|
|
564
509
|
|
|
510
|
+
if (fromTask) {
|
|
511
|
+
if (!status) {
|
|
512
|
+
return sendJson(500, createErrorResponse("status_unavailable", `job "${jobId}" status could not be read`));
|
|
513
|
+
}
|
|
514
|
+
const pipelineTaskOrder = await loadPipelineTaskOrder(dataDir, jobDir, status);
|
|
515
|
+
if (!pipelineTaskOrder) {
|
|
516
|
+
return sendJson(500, createErrorResponse("status_unavailable", `pipeline task order for job "${jobId}" could not be read`));
|
|
517
|
+
}
|
|
518
|
+
const taskIndex = pipelineTaskOrder.indexOf(fromTask);
|
|
519
|
+
if (taskIndex === -1) {
|
|
520
|
+
return sendJson(404, createErrorResponse("task_not_found", `task "${fromTask}" was not found in the pipeline definition`));
|
|
521
|
+
}
|
|
522
|
+
const predecessors = pipelineTaskOrder.slice(0, taskIndex).map((name) => ({
|
|
523
|
+
name,
|
|
524
|
+
state: status.tasks[name]?.state ?? "pending",
|
|
525
|
+
}));
|
|
526
|
+
if (!areDependenciesSatisfied(predecessors.map((p) => p.state))) {
|
|
527
|
+
const blocker = firstUnsatisfiedDependency(predecessors)!;
|
|
528
|
+
return sendJson(412, createErrorResponse(
|
|
529
|
+
"dependencies_not_satisfied",
|
|
530
|
+
`task "${blocker.name}" must be done or skipped before "${fromTask}" (state: "${blocker.state}")`,
|
|
531
|
+
));
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
565
535
|
const slot = await acquireConcurrencySlot(dataDir, jobId, "restart");
|
|
566
536
|
if (!slot.ok) return slot.response;
|
|
567
537
|
|
|
@@ -631,81 +601,93 @@ export async function handleJobStop(
|
|
|
631
601
|
|
|
632
602
|
const jobDir = getJobDirectoryPath(dataDir, jobId, lifecycle as "current" | "complete");
|
|
633
603
|
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
604
|
+
try {
|
|
605
|
+
// Kill the runner process via PID file
|
|
606
|
+
let pidFound = false;
|
|
607
|
+
let usedSignal: string | null = null;
|
|
608
|
+
|
|
609
|
+
const pid = await readRunnerPid(jobDir);
|
|
610
|
+
if (pid !== null) {
|
|
611
|
+
pidFound = true;
|
|
612
|
+
try {
|
|
613
|
+
const result = await killProcess(pid);
|
|
614
|
+
usedSignal = result.signal;
|
|
615
|
+
} catch (err) {
|
|
616
|
+
console.error(`[handleJobStop] Error killing pid ${pid} for job ${jobId}:`, err);
|
|
617
|
+
}
|
|
618
|
+
await cleanupRunnerPid(jobDir);
|
|
646
619
|
}
|
|
647
|
-
await cleanupRunnerPid(jobDir);
|
|
648
|
-
}
|
|
649
620
|
|
|
650
|
-
|
|
651
|
-
|
|
621
|
+
try { await assertKnownJobStatusesParseable(dataDir, jobId); }
|
|
622
|
+
catch (err) { if (err instanceof StatusCorruptError) return statusCorruptResponse(err); throw err; }
|
|
652
623
|
|
|
653
|
-
|
|
624
|
+
let resetTask: string | null = null;
|
|
654
625
|
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
626
|
+
// Reset running task and clear root-level fields in a single atomic write.
|
|
627
|
+
try {
|
|
628
|
+
await writeJobStatus(jobDir, (snapshot) => {
|
|
629
|
+
if (snapshot.current && snapshot.tasks[snapshot.current]?.state === "running") {
|
|
630
|
+
resetTask = snapshot.current;
|
|
631
|
+
} else {
|
|
632
|
+
for (const taskId of Object.keys(snapshot.tasks)) {
|
|
633
|
+
if (snapshot.tasks[taskId]!.state === "running") {
|
|
634
|
+
resetTask = taskId;
|
|
635
|
+
break;
|
|
636
|
+
}
|
|
665
637
|
}
|
|
666
638
|
}
|
|
667
|
-
}
|
|
668
639
|
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
640
|
+
if (!resetTask) {
|
|
641
|
+
resetTask = findRecoveryTask(snapshot);
|
|
642
|
+
}
|
|
672
643
|
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
644
|
+
if (resetTask) {
|
|
645
|
+
const task = snapshot.tasks[resetTask];
|
|
646
|
+
if (task) {
|
|
647
|
+
task.state = "pending";
|
|
648
|
+
task.currentStage = null;
|
|
649
|
+
delete task.endedAt;
|
|
650
|
+
delete task.startedAt;
|
|
651
|
+
delete task.failedStage;
|
|
652
|
+
delete task.error;
|
|
653
|
+
task.attempts = 0;
|
|
654
|
+
task.restartCount = 0;
|
|
655
|
+
task.refinementAttempts = 0;
|
|
656
|
+
task.tokenUsage = [];
|
|
657
|
+
}
|
|
658
|
+
} else if (snapshot.current === null && snapshot.state === "pending" && Object.keys(snapshot.tasks).length > 0) {
|
|
659
|
+
const taskIds = Object.keys(snapshot.tasks);
|
|
660
|
+
snapshot.current = taskIds[0] ?? null;
|
|
686
661
|
}
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
snapshot.current =
|
|
662
|
+
|
|
663
|
+
snapshot.state = "pending";
|
|
664
|
+
snapshot.current = resetTask ?? snapshot.current;
|
|
665
|
+
snapshot.currentStage = null;
|
|
666
|
+
snapshot.progress = getProgressPercent(snapshot);
|
|
667
|
+
});
|
|
668
|
+
} catch (err) {
|
|
669
|
+
if (err instanceof StatusCorruptError) return statusCorruptResponse(err);
|
|
670
|
+
throw err;
|
|
690
671
|
}
|
|
691
672
|
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
673
|
+
return sendJson(202, {
|
|
674
|
+
ok: true,
|
|
675
|
+
jobId,
|
|
676
|
+
stopped: pidFound,
|
|
677
|
+
resetTask,
|
|
678
|
+
signal: usedSignal,
|
|
679
|
+
});
|
|
680
|
+
} finally {
|
|
681
|
+
try {
|
|
682
|
+
await releaseJobSlot(
|
|
683
|
+
getPipelineDataDir(dataDir),
|
|
684
|
+
jobId,
|
|
685
|
+
getOrchestratorRuntimeConfig().lockFileTimeout,
|
|
686
|
+
);
|
|
687
|
+
} catch (releaseErr) {
|
|
688
|
+
console.error(`[handleJobStop] Failed to release slot for job ${jobId}:`, releaseErr);
|
|
689
|
+
}
|
|
700
690
|
}
|
|
701
|
-
|
|
702
|
-
return sendJson(202, {
|
|
703
|
-
ok: true,
|
|
704
|
-
jobId,
|
|
705
|
-
stopped: pidFound,
|
|
706
|
-
resetTask,
|
|
707
|
-
signal: usedSignal,
|
|
708
|
-
});
|
|
709
691
|
} finally {
|
|
710
692
|
endStop(jobId);
|
|
711
693
|
}
|
|
@@ -764,7 +746,7 @@ export async function handleTaskStart(
|
|
|
764
746
|
pid,
|
|
765
747
|
pidAlive,
|
|
766
748
|
nowMs: Date.now(),
|
|
767
|
-
staleAfterMs:
|
|
749
|
+
staleAfterMs: getOrchestratorConfig().staleRunningGraceMs,
|
|
768
750
|
});
|
|
769
751
|
if (!staleDecision.stale && staleDecision.reason !== "not_running") {
|
|
770
752
|
return sendJson(409, createErrorResponse("job_running", "Job is currently running (task-level running)"));
|
|
@@ -789,10 +771,16 @@ export async function handleTaskStart(
|
|
|
789
771
|
return sendJson(404, createErrorResponse("task_not_found", `task "${taskId}" was not found in the pipeline definition`));
|
|
790
772
|
}
|
|
791
773
|
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
774
|
+
const predecessors = pipelineTaskOrder.slice(0, taskIndex).map((name) => ({
|
|
775
|
+
name,
|
|
776
|
+
state: snapshot.tasks[name]?.state ?? "pending",
|
|
777
|
+
}));
|
|
778
|
+
if (!areDependenciesSatisfied(predecessors.map((p) => p.state))) {
|
|
779
|
+
const blocker = firstUnsatisfiedDependency(predecessors)!;
|
|
780
|
+
return sendJson(412, createErrorResponse(
|
|
781
|
+
"dependencies_not_satisfied",
|
|
782
|
+
`task "${blocker.name}" must be done or skipped before "${taskId}" (state: "${blocker.state}")`,
|
|
783
|
+
));
|
|
796
784
|
}
|
|
797
785
|
|
|
798
786
|
const slot = await acquireConcurrencySlot(dataDir, jobId, "task-start");
|
|
@@ -60,6 +60,12 @@ export async function handleJobDetail(jobId: string): Promise<Response> {
|
|
|
60
60
|
|
|
61
61
|
const result = await readJob(jobId);
|
|
62
62
|
if (!result.ok) {
|
|
63
|
+
if (result.code === Constants.ERROR_CODES.INVALID_JSON) {
|
|
64
|
+
return sendJson(
|
|
65
|
+
409,
|
|
66
|
+
createErrorResponse(Constants.ERROR_CODES.STATUS_CORRUPT, result.message, result.path),
|
|
67
|
+
);
|
|
68
|
+
}
|
|
63
69
|
const status = result.code === Constants.ERROR_CODES.JOB_NOT_FOUND ? 404 : 400;
|
|
64
70
|
return sendJson(status, result);
|
|
65
71
|
}
|
|
@@ -74,5 +80,6 @@ export async function handleJobDetail(jobId: string): Promise<Response> {
|
|
|
74
80
|
}
|
|
75
81
|
|
|
76
82
|
const [apiJob] = transformJobListForAPI([job], { includePipelineMetadata: true });
|
|
83
|
+
if (apiJob) apiJob.costs = job.costs;
|
|
77
84
|
return sendJson(200, { ok: true, data: apiJob });
|
|
78
85
|
}
|
|
@@ -1,15 +1,51 @@
|
|
|
1
|
+
import { dirname, resolve } from "node:path";
|
|
2
|
+
|
|
3
|
+
import { resolvePipelineSync, resolveTaskRegistryPath } from "../../../config/pipeline-registry";
|
|
4
|
+
import { loadFreshModule } from "../../../core/module-loader";
|
|
5
|
+
import { analyzeTask } from "../../../task-analysis/analyzer";
|
|
6
|
+
import { TaskAnalysisRepository } from "../../../task-analysis/repository";
|
|
7
|
+
import type { PersistedTaskAnalysis, TaskAnalysis } from "../../../task-analysis/types";
|
|
1
8
|
import { acquireLock, releaseLock } from "../../state/analysis-lock";
|
|
2
9
|
|
|
3
|
-
|
|
10
|
+
/** Doc-shaped placeholder persisted when per-task analysis throws unexpectedly. */
|
|
11
|
+
function emptyAnalysis(): TaskAnalysis {
|
|
12
|
+
return {
|
|
13
|
+
summary: "",
|
|
14
|
+
stages: [],
|
|
15
|
+
artifacts: { reads: [], writes: [] },
|
|
16
|
+
models: [],
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Loads the task-name -> module-path map (default export of the task registry module). */
|
|
21
|
+
async function loadTaskRegistryMap(registryPath: string): Promise<Record<string, string>> {
|
|
22
|
+
const mod = await loadFreshModule(registryPath);
|
|
23
|
+
return (mod["default"] ?? {}) as Record<string, string>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function handlePipelineAnalysis(_req: Request, slug: string, root: string): Promise<Response> {
|
|
4
27
|
const lock = acquireLock(slug);
|
|
5
28
|
if (!lock.acquired) {
|
|
6
|
-
return new Response(
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
29
|
+
return new Response(
|
|
30
|
+
JSON.stringify({ ok: false, code: "BAD_REQUEST", message: `analysis lock held by ${lock.heldBy}` }),
|
|
31
|
+
{
|
|
32
|
+
status: 409,
|
|
33
|
+
headers: { "Content-Type": "application/json" },
|
|
34
|
+
},
|
|
35
|
+
);
|
|
10
36
|
}
|
|
11
37
|
|
|
12
38
|
const encoder = new TextEncoder();
|
|
39
|
+
// Shared across start()'s finally and cancel() so the lock is released exactly
|
|
40
|
+
// once regardless of which path runs first (client disconnect vs. completion).
|
|
41
|
+
let released = false;
|
|
42
|
+
const releaseOnce = (): void => {
|
|
43
|
+
if (!released) {
|
|
44
|
+
released = true;
|
|
45
|
+
releaseLock(slug);
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
|
|
13
49
|
const response = new Response(
|
|
14
50
|
new ReadableStream<Uint8Array>({
|
|
15
51
|
start(controller) {
|
|
@@ -17,19 +53,71 @@ export async function handlePipelineAnalysis(_req: Request, slug: string): Promi
|
|
|
17
53
|
controller.enqueue(encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`));
|
|
18
54
|
};
|
|
19
55
|
|
|
20
|
-
|
|
56
|
+
void (async () => {
|
|
21
57
|
try {
|
|
22
|
-
|
|
58
|
+
// Resolution failures (unknown slug, unloadable registry) must not abort the
|
|
59
|
+
// run: emit an error and still reach `complete`.
|
|
60
|
+
let registryDir = "";
|
|
61
|
+
let taskMap: Record<string, string> = {};
|
|
62
|
+
try {
|
|
63
|
+
const { tasksDir } = resolvePipelineSync(root, slug);
|
|
64
|
+
const registryPath = resolveTaskRegistryPath(tasksDir);
|
|
65
|
+
registryDir = dirname(registryPath);
|
|
66
|
+
taskMap = await loadTaskRegistryMap(registryPath);
|
|
67
|
+
} catch (err) {
|
|
68
|
+
send("error", { message: err instanceof Error ? err.message : String(err) });
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const taskNames = Object.keys(taskMap);
|
|
72
|
+
send("started", { slug, totalTasks: taskNames.length });
|
|
73
|
+
|
|
74
|
+
for (const taskName of taskNames) {
|
|
75
|
+
send("task:start", { task: taskName });
|
|
76
|
+
try {
|
|
77
|
+
const sourcePath = resolve(registryDir, taskMap[taskName]!);
|
|
78
|
+
const source = await Bun.file(sourcePath).text();
|
|
79
|
+
const result = await analyzeTask(source);
|
|
80
|
+
const persisted: PersistedTaskAnalysis = {
|
|
81
|
+
summary: result.summary,
|
|
82
|
+
stages: result.stages,
|
|
83
|
+
artifacts: result.artifacts,
|
|
84
|
+
models: result.models,
|
|
85
|
+
analyzedAt: new Date().toISOString(),
|
|
86
|
+
analysisStatus: result.analysisStatus,
|
|
87
|
+
...(result.analysisError !== undefined ? { analysisError: result.analysisError } : {}),
|
|
88
|
+
};
|
|
89
|
+
await TaskAnalysisRepository.write(root, slug, taskName, persisted);
|
|
90
|
+
} catch (err) {
|
|
91
|
+
// Any unexpected throw (source read, persist) persists an `error`
|
|
92
|
+
// placeholder and reports it, without aborting the run.
|
|
93
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
94
|
+
const placeholder: PersistedTaskAnalysis = {
|
|
95
|
+
...emptyAnalysis(),
|
|
96
|
+
analyzedAt: new Date().toISOString(),
|
|
97
|
+
analysisStatus: "error",
|
|
98
|
+
analysisError: message,
|
|
99
|
+
};
|
|
100
|
+
try {
|
|
101
|
+
await TaskAnalysisRepository.write(root, slug, taskName, placeholder);
|
|
102
|
+
} catch {}
|
|
103
|
+
send("error", { task: taskName, message });
|
|
104
|
+
}
|
|
105
|
+
send("task:complete", { task: taskName });
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
send("complete", { slug });
|
|
109
|
+
} catch (err) {
|
|
110
|
+
send("error", { message: err instanceof Error ? err.message : String(err) });
|
|
23
111
|
send("complete", { slug });
|
|
24
|
-
controller.close();
|
|
25
112
|
} finally {
|
|
26
|
-
|
|
113
|
+
controller.close();
|
|
114
|
+
releaseOnce();
|
|
27
115
|
}
|
|
28
|
-
});
|
|
116
|
+
})();
|
|
29
117
|
},
|
|
30
118
|
cancel() {
|
|
31
119
|
try {
|
|
32
|
-
|
|
120
|
+
releaseOnce();
|
|
33
121
|
} catch {}
|
|
34
122
|
},
|
|
35
123
|
}),
|
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
|
|
3
|
+
import { resolveUnderRoot } from "../../../utils/path-containment";
|
|
3
4
|
import { sendJson } from "../utils/http-utils";
|
|
4
5
|
|
|
5
|
-
export async function handleSchemaFile(
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
export async function handleSchemaFile(
|
|
7
|
+
_req: Request,
|
|
8
|
+
slug: string,
|
|
9
|
+
filename: string,
|
|
10
|
+
root: string,
|
|
11
|
+
): Promise<Response> {
|
|
12
|
+
const filePath = resolveUnderRoot(path.join(root, "pipeline-config"), slug, "schemas", filename);
|
|
13
|
+
if (filePath === null) {
|
|
14
|
+
return sendJson(400, { ok: false, code: "BAD_REQUEST", message: "invalid path" });
|
|
15
|
+
}
|
|
8
16
|
if (!(await Bun.file(filePath).exists())) {
|
|
9
17
|
return sendJson(404, { ok: false, code: "NOT_FOUND", message: "schema file not found" });
|
|
10
18
|
}
|