@ryanfw/prompt-orchestration-pipeline 1.3.3 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/http-api.md +14 -4
- package/package.json +1 -4
- package/src/api/__tests__/index.test.ts +144 -41
- package/src/api/index.ts +15 -83
- package/src/cli/__tests__/analyze-task.test.ts +40 -7
- package/src/cli/__tests__/index.test.ts +337 -17
- package/src/cli/__tests__/types.test.ts +0 -18
- package/src/cli/__tests__/update-pipeline-json.test.ts +19 -9
- package/src/cli/analyze-task.ts +3 -14
- package/src/cli/index.ts +73 -61
- package/src/cli/types.ts +0 -13
- package/src/cli/update-pipeline-json.ts +3 -14
- package/src/config/__tests__/paths.test.ts +46 -1
- package/src/config/__tests__/pipeline-registry.test.ts +767 -0
- package/src/config/__tests__/sse-events.test.ts +470 -0
- package/src/config/paths.ts +11 -2
- package/src/config/pipeline-registry.ts +258 -0
- package/src/config/sse-events.ts +122 -0
- package/src/core/__tests__/agent-step.test.ts +115 -0
- package/src/core/__tests__/config.test.ts +517 -107
- package/src/core/__tests__/core-boot-resolution.test.ts +181 -0
- package/src/core/__tests__/job-concurrency.test.ts +200 -0
- package/src/core/__tests__/job-submission.test.ts +396 -0
- package/src/core/__tests__/job-view.test.ts +485 -3
- package/src/core/__tests__/json-file.test.ts +111 -0
- package/src/core/__tests__/lifecycle-policy.test.ts +81 -7
- package/src/core/__tests__/logger.test.ts +22 -0
- package/src/core/__tests__/orchestrator-no-deprecated-exports.test.ts +41 -0
- package/src/core/__tests__/orchestrator.test.ts +373 -2
- package/src/core/__tests__/pipeline-runner.test.ts +946 -9
- package/src/core/__tests__/redact.test.ts +153 -0
- package/src/core/__tests__/runner-liveness.test.ts +45 -0
- package/src/core/__tests__/seed-naming.test.ts +57 -0
- package/src/core/__tests__/single-derivation.test.ts +1 -1
- package/src/core/__tests__/task-runner.test.ts +594 -3
- package/src/core/__tests__/task-telemetry.test.ts +241 -0
- package/src/core/__tests__/workspace-root-resolution-guard.test.ts +62 -0
- package/src/core/agent-step.ts +4 -1
- package/src/core/agent-types.ts +5 -4
- package/src/core/config.ts +127 -234
- package/src/core/control.ts +3 -0
- package/src/core/job-concurrency.ts +50 -19
- package/src/core/job-submission.ts +133 -0
- package/src/core/job-view.ts +162 -18
- package/src/core/json-file.ts +45 -0
- package/src/core/lifecycle-policy.ts +23 -8
- package/src/core/logger.ts +0 -29
- package/src/core/orchestrator.ts +124 -70
- package/src/core/pipeline-runner.ts +85 -53
- package/src/core/redact.ts +40 -0
- package/src/core/runner-liveness.ts +8 -4
- package/src/core/seed-naming.ts +9 -0
- package/src/core/status-writer.ts +3 -28
- package/src/core/task-runner.ts +356 -319
- package/src/core/task-telemetry.ts +107 -0
- package/src/harness/__tests__/subprocess.test.ts +112 -1
- package/src/harness/subprocess.ts +55 -16
- package/src/llm/__tests__/index.test.ts +684 -33
- package/src/llm/index.ts +310 -67
- package/src/providers/__tests__/alibaba.test.ts +67 -6
- package/src/providers/__tests__/anthropic.test.ts +35 -14
- package/src/providers/__tests__/base.test.ts +62 -0
- package/src/providers/__tests__/claude-code.test.ts +99 -14
- package/src/providers/__tests__/deepseek.test.ts +16 -6
- package/src/providers/__tests__/gemini.test.ts +47 -25
- package/src/providers/__tests__/moonshot.test.ts +27 -0
- package/src/providers/__tests__/openai.test.ts +262 -74
- package/src/providers/__tests__/opencode.test.ts +77 -0
- package/src/providers/__tests__/request-timeout-contract.test.ts +226 -0
- package/src/providers/__tests__/stream-accumulator.test.ts +52 -0
- package/src/providers/__tests__/types.test.ts +85 -0
- package/src/providers/__tests__/zero-fill-guard.test.ts +62 -0
- package/src/providers/__tests__/zhipu.test.ts +27 -14
- package/src/providers/alibaba.ts +20 -39
- package/src/providers/anthropic.ts +23 -11
- package/src/providers/base.ts +19 -0
- package/src/providers/claude-code.ts +27 -18
- package/src/providers/deepseek.ts +9 -28
- package/src/providers/gemini.ts +20 -58
- package/src/providers/moonshot.ts +15 -11
- package/src/providers/openai.ts +79 -61
- package/src/providers/opencode.ts +16 -33
- package/src/providers/stream-accumulator.ts +27 -21
- package/src/providers/types.ts +29 -4
- package/src/providers/zhipu.ts +15 -44
- package/src/task-analysis/__tests__/analyzer.test.ts +0 -0
- package/src/task-analysis/__tests__/enrichers-schema-deducer.test.ts +34 -2
- package/src/task-analysis/__tests__/no-ast-imports.test.ts +52 -0
- package/src/task-analysis/__tests__/repository.test.ts +65 -0
- package/src/task-analysis/__tests__/types.test.ts +63 -130
- package/src/task-analysis/analyzer.ts +91 -0
- package/src/task-analysis/enrichers/schema-deducer.ts +13 -2
- package/src/task-analysis/index.ts +2 -36
- package/src/task-analysis/repository.ts +45 -0
- package/src/task-analysis/types.ts +42 -58
- package/src/ui/client/__tests__/api.test.ts +143 -1
- package/src/ui/client/__tests__/bootstrap.test.ts +178 -62
- package/src/ui/client/__tests__/job-adapter.test.ts +125 -2
- package/src/ui/client/__tests__/load-state.test.ts +70 -0
- package/src/ui/client/__tests__/types.test.ts +66 -3
- package/src/ui/client/__tests__/useJobDetailWithUpdates.test.ts +390 -77
- package/src/ui/client/__tests__/useJobList.test.ts +198 -23
- package/src/ui/client/__tests__/useJobListWithUpdates.test.ts +218 -7
- package/src/ui/client/adapters/__tests__/job-adapter.test.ts +186 -0
- package/src/ui/client/adapters/job-adapter.ts +31 -16
- package/src/ui/client/api.ts +38 -15
- package/src/ui/client/bootstrap.ts +19 -14
- package/src/ui/client/hooks/useJobDetailWithUpdates.ts +73 -97
- package/src/ui/client/hooks/useJobList.ts +26 -31
- package/src/ui/client/hooks/useJobListWithUpdates.ts +42 -76
- package/src/ui/client/load-state.ts +20 -0
- package/src/ui/client/reducers/__tests__/job-events.test.ts +568 -0
- package/src/ui/client/reducers/job-events.ts +137 -0
- package/src/ui/client/types.ts +14 -20
- package/src/ui/components/DAGGrid.tsx +6 -1
- package/src/ui/components/JobDetail.tsx +12 -4
- package/src/ui/components/JobTable.tsx +13 -2
- package/src/ui/components/StageTimeline.tsx +8 -20
- package/src/ui/components/TaskAnalysisDisplay.tsx +48 -6
- package/src/ui/components/__tests__/DAGGrid.test.tsx +36 -0
- package/src/ui/components/__tests__/JobDetail.test.tsx +112 -0
- package/src/ui/components/__tests__/JobTable.test.tsx +83 -0
- package/src/ui/components/__tests__/StageTimeline.test.tsx +5 -6
- package/src/ui/components/__tests__/TaskAnalysisDisplay.test.tsx +64 -0
- package/src/ui/components/types.ts +33 -15
- package/src/ui/dist/assets/{index-L6cvsCAx.js → index-DN3-zvtP.js} +4299 -251
- package/src/ui/dist/assets/index-DN3-zvtP.js.map +1 -0
- package/src/ui/dist/assets/style-CtZBnjlR.css +2 -0
- package/src/ui/dist/index.html +2 -2
- package/src/ui/pages/PipelineDetail.tsx +60 -4
- package/src/ui/pages/PromptPipelineDashboard.tsx +59 -7
- package/src/ui/pages/__tests__/PromptPipelineDashboard.test.tsx +114 -32
- package/src/ui/pages/__tests__/pages.test.tsx +236 -42
- package/src/ui/server/__tests__/concurrency-endpoint.test.ts +54 -0
- package/src/ui/server/__tests__/config-bridge-node.test.ts +34 -1
- package/src/ui/server/__tests__/embedded-assets.test.ts +66 -0
- package/src/ui/server/__tests__/file-endpoints.test.ts +183 -5
- package/src/ui/server/__tests__/gate-endpoints.test.ts +55 -0
- package/src/ui/server/__tests__/index.test.ts +63 -0
- package/src/ui/server/__tests__/job-control-endpoints.test.ts +455 -1
- package/src/ui/server/__tests__/job-endpoints.test.ts +43 -1
- package/src/ui/server/__tests__/read-static-path-guard.test.ts +94 -0
- package/src/ui/server/__tests__/router-root-isolation.test.ts +204 -0
- package/src/ui/server/__tests__/router-threading.test.ts +148 -0
- package/src/ui/server/__tests__/router.test.ts +104 -0
- package/src/ui/server/__tests__/sse-broadcast.test.ts +44 -0
- package/src/ui/server/__tests__/sse-enhancer.test.ts +39 -0
- package/src/ui/server/config-bridge-node.ts +8 -26
- package/src/ui/server/embedded-assets-imports.d.ts +44 -0
- package/src/ui/server/embedded-assets.ts +13 -2
- package/src/ui/server/endpoints/__tests__/pipeline-analysis-endpoint.test.ts +167 -0
- package/src/ui/server/endpoints/__tests__/schema-file-endpoint.test.ts +104 -0
- package/src/ui/server/endpoints/__tests__/task-analysis-endpoint.test.ts +103 -0
- package/src/ui/server/endpoints/__tests__/upload-endpoints.test.ts +162 -96
- package/src/ui/server/endpoints/concurrency-endpoint.ts +2 -1
- package/src/ui/server/endpoints/create-pipeline-endpoint.ts +14 -29
- package/src/ui/server/endpoints/file-endpoints.ts +15 -10
- package/src/ui/server/endpoints/job-control-endpoints.ts +122 -73
- package/src/ui/server/endpoints/job-endpoints.ts +1 -0
- package/src/ui/server/endpoints/pipeline-analysis-endpoint.ts +99 -11
- package/src/ui/server/endpoints/schema-file-endpoint.ts +11 -3
- package/src/ui/server/endpoints/task-analysis-endpoint.ts +21 -5
- package/src/ui/server/endpoints/task-save-endpoint.ts +1 -2
- package/src/ui/server/endpoints/upload-endpoints.ts +5 -40
- package/src/ui/server/index.ts +19 -14
- package/src/ui/server/job-reader.ts +14 -2
- package/src/ui/server/router.ts +33 -10
- package/src/ui/server/sse-broadcast.ts +14 -3
- package/src/ui/server/sse-enhancer.ts +12 -2
- package/src/ui/state/__tests__/schema-loader.test.ts +11 -1
- package/src/ui/state/__tests__/snapshot.test.ts +70 -15
- package/src/ui/state/__tests__/types.test.ts +6 -0
- package/src/ui/state/snapshot.ts +1 -1
- package/src/ui/state/transformers/__tests__/list-transformer.test.ts +42 -0
- package/src/ui/state/transformers/__tests__/status-transformer.test.ts +45 -3
- package/src/ui/state/transformers/list-transformer.ts +6 -0
- package/src/ui/state/types.ts +6 -1
- package/src/utils/__tests__/path-containment.test.ts +160 -0
- package/src/{ui/server/utils → utils}/path-containment.ts +21 -0
- package/src/task-analysis/__tests__/enrichers-analysis-writer.test.ts +0 -86
- package/src/task-analysis/__tests__/enrichers-artifact-resolver.test.ts +0 -124
- package/src/task-analysis/__tests__/extractors-artifacts.test.ts +0 -133
- package/src/task-analysis/__tests__/extractors-llm-calls.test.ts +0 -46
- package/src/task-analysis/__tests__/extractors-stages.test.ts +0 -52
- package/src/task-analysis/__tests__/index.test.ts +0 -143
- package/src/task-analysis/__tests__/parser.test.ts +0 -41
- package/src/task-analysis/__tests__/utils-ast.test.ts +0 -82
- package/src/task-analysis/enrichers/analysis-writer.ts +0 -75
- package/src/task-analysis/enrichers/artifact-resolver.ts +0 -89
- package/src/task-analysis/extractors/artifacts.ts +0 -143
- package/src/task-analysis/extractors/llm-calls.ts +0 -117
- package/src/task-analysis/extractors/stages.ts +0 -45
- package/src/task-analysis/parser.ts +0 -20
- package/src/task-analysis/utils/ast.ts +0 -45
- package/src/ui/dist/assets/index-L6cvsCAx.js.map +0 -1
- package/src/ui/dist/assets/style-CSSKuMOe.css +0 -2
- package/src/ui/embedded-assets.js +0 -12
- package/src/ui/server/__tests__/path-containment.test.ts +0 -54
|
@@ -26,6 +26,10 @@ import {
|
|
|
26
26
|
StatusCorruptError,
|
|
27
27
|
type StatusSnapshot,
|
|
28
28
|
} from "../../../core/status-writer";
|
|
29
|
+
import {
|
|
30
|
+
areDependenciesSatisfied,
|
|
31
|
+
firstUnsatisfiedDependency,
|
|
32
|
+
} from "../../../core/lifecycle-policy";
|
|
29
33
|
import { assertKnownJobStatusesParseable, statusCorruptResponse } from "../utils/status-corruption";
|
|
30
34
|
import {
|
|
31
35
|
materializeNormalizedPipelineDefinition,
|
|
@@ -38,11 +42,16 @@ const restartingJobs = new Set<string>();
|
|
|
38
42
|
const stoppingJobs = new Set<string>();
|
|
39
43
|
const startingJobs = new Set<string>();
|
|
40
44
|
|
|
41
|
-
function getOrchestratorRuntimeConfig(): {
|
|
45
|
+
function getOrchestratorRuntimeConfig(): {
|
|
46
|
+
maxConcurrentJobs: number;
|
|
47
|
+
lockFileTimeout: number;
|
|
48
|
+
staleLeaseTimeoutMs: number;
|
|
49
|
+
} {
|
|
42
50
|
const orchestrator = getOrchestratorConfig();
|
|
43
51
|
return {
|
|
44
52
|
maxConcurrentJobs: orchestrator.maxConcurrentJobs,
|
|
45
53
|
lockFileTimeout: orchestrator.lockFileTimeout,
|
|
54
|
+
staleLeaseTimeoutMs: orchestrator.staleLeaseTimeoutMs,
|
|
46
55
|
};
|
|
47
56
|
}
|
|
48
57
|
|
|
@@ -102,13 +111,14 @@ export async function acquireConcurrencySlot(
|
|
|
102
111
|
jobId: string,
|
|
103
112
|
source: JobSlotLease["source"],
|
|
104
113
|
): Promise<{ ok: true } | { ok: false; response: Response }> {
|
|
105
|
-
const { maxConcurrentJobs, lockFileTimeout } = getOrchestratorRuntimeConfig();
|
|
114
|
+
const { maxConcurrentJobs, lockFileTimeout, staleLeaseTimeoutMs } = getOrchestratorRuntimeConfig();
|
|
106
115
|
const result = await tryAcquireJobSlot({
|
|
107
116
|
dataDir: getPipelineDataDir(dataDir),
|
|
108
117
|
jobId,
|
|
109
118
|
maxConcurrentJobs,
|
|
110
119
|
source,
|
|
111
120
|
lockTimeoutMs: lockFileTimeout,
|
|
121
|
+
staleLeaseTimeoutMs,
|
|
112
122
|
});
|
|
113
123
|
if (result.ok) return { ok: true };
|
|
114
124
|
if (result.reason === "already_held") {
|
|
@@ -176,10 +186,6 @@ function isNonTerminalTaskState(state: unknown): boolean {
|
|
|
176
186
|
return state === "pending" || state === "running";
|
|
177
187
|
}
|
|
178
188
|
|
|
179
|
-
function isDependencySatisfied(state: unknown): boolean {
|
|
180
|
-
return state === "done";
|
|
181
|
-
}
|
|
182
|
-
|
|
183
189
|
function isTaskLikelyInProgress(task: Record<string, unknown>): boolean {
|
|
184
190
|
const state = task["state"];
|
|
185
191
|
const nonTerminal = isNonTerminalTaskState(state);
|
|
@@ -501,6 +507,31 @@ export async function handleJobRestart(
|
|
|
501
507
|
}
|
|
502
508
|
}
|
|
503
509
|
|
|
510
|
+
if (fromTask) {
|
|
511
|
+
if (!status) {
|
|
512
|
+
return sendJson(500, createErrorResponse("status_unavailable", `job "${jobId}" status could not be read`));
|
|
513
|
+
}
|
|
514
|
+
const pipelineTaskOrder = await loadPipelineTaskOrder(dataDir, jobDir, status);
|
|
515
|
+
if (!pipelineTaskOrder) {
|
|
516
|
+
return sendJson(500, createErrorResponse("status_unavailable", `pipeline task order for job "${jobId}" could not be read`));
|
|
517
|
+
}
|
|
518
|
+
const taskIndex = pipelineTaskOrder.indexOf(fromTask);
|
|
519
|
+
if (taskIndex === -1) {
|
|
520
|
+
return sendJson(404, createErrorResponse("task_not_found", `task "${fromTask}" was not found in the pipeline definition`));
|
|
521
|
+
}
|
|
522
|
+
const predecessors = pipelineTaskOrder.slice(0, taskIndex).map((name) => ({
|
|
523
|
+
name,
|
|
524
|
+
state: status.tasks[name]?.state ?? "pending",
|
|
525
|
+
}));
|
|
526
|
+
if (!areDependenciesSatisfied(predecessors.map((p) => p.state))) {
|
|
527
|
+
const blocker = firstUnsatisfiedDependency(predecessors)!;
|
|
528
|
+
return sendJson(412, createErrorResponse(
|
|
529
|
+
"dependencies_not_satisfied",
|
|
530
|
+
`task "${blocker.name}" must be done or skipped before "${fromTask}" (state: "${blocker.state}")`,
|
|
531
|
+
));
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
504
535
|
const slot = await acquireConcurrencySlot(dataDir, jobId, "restart");
|
|
505
536
|
if (!slot.ok) return slot.response;
|
|
506
537
|
|
|
@@ -570,81 +601,93 @@ export async function handleJobStop(
|
|
|
570
601
|
|
|
571
602
|
const jobDir = getJobDirectoryPath(dataDir, jobId, lifecycle as "current" | "complete");
|
|
572
603
|
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
604
|
+
try {
|
|
605
|
+
// Kill the runner process via PID file
|
|
606
|
+
let pidFound = false;
|
|
607
|
+
let usedSignal: string | null = null;
|
|
608
|
+
|
|
609
|
+
const pid = await readRunnerPid(jobDir);
|
|
610
|
+
if (pid !== null) {
|
|
611
|
+
pidFound = true;
|
|
612
|
+
try {
|
|
613
|
+
const result = await killProcess(pid);
|
|
614
|
+
usedSignal = result.signal;
|
|
615
|
+
} catch (err) {
|
|
616
|
+
console.error(`[handleJobStop] Error killing pid ${pid} for job ${jobId}:`, err);
|
|
617
|
+
}
|
|
618
|
+
await cleanupRunnerPid(jobDir);
|
|
585
619
|
}
|
|
586
|
-
await cleanupRunnerPid(jobDir);
|
|
587
|
-
}
|
|
588
620
|
|
|
589
|
-
|
|
590
|
-
|
|
621
|
+
try { await assertKnownJobStatusesParseable(dataDir, jobId); }
|
|
622
|
+
catch (err) { if (err instanceof StatusCorruptError) return statusCorruptResponse(err); throw err; }
|
|
591
623
|
|
|
592
|
-
|
|
624
|
+
let resetTask: string | null = null;
|
|
593
625
|
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
626
|
+
// Reset running task and clear root-level fields in a single atomic write.
|
|
627
|
+
try {
|
|
628
|
+
await writeJobStatus(jobDir, (snapshot) => {
|
|
629
|
+
if (snapshot.current && snapshot.tasks[snapshot.current]?.state === "running") {
|
|
630
|
+
resetTask = snapshot.current;
|
|
631
|
+
} else {
|
|
632
|
+
for (const taskId of Object.keys(snapshot.tasks)) {
|
|
633
|
+
if (snapshot.tasks[taskId]!.state === "running") {
|
|
634
|
+
resetTask = taskId;
|
|
635
|
+
break;
|
|
636
|
+
}
|
|
604
637
|
}
|
|
605
638
|
}
|
|
606
|
-
}
|
|
607
639
|
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
640
|
+
if (!resetTask) {
|
|
641
|
+
resetTask = findRecoveryTask(snapshot);
|
|
642
|
+
}
|
|
611
643
|
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
644
|
+
if (resetTask) {
|
|
645
|
+
const task = snapshot.tasks[resetTask];
|
|
646
|
+
if (task) {
|
|
647
|
+
task.state = "pending";
|
|
648
|
+
task.currentStage = null;
|
|
649
|
+
delete task.endedAt;
|
|
650
|
+
delete task.startedAt;
|
|
651
|
+
delete task.failedStage;
|
|
652
|
+
delete task.error;
|
|
653
|
+
task.attempts = 0;
|
|
654
|
+
task.restartCount = 0;
|
|
655
|
+
task.refinementAttempts = 0;
|
|
656
|
+
task.tokenUsage = [];
|
|
657
|
+
}
|
|
658
|
+
} else if (snapshot.current === null && snapshot.state === "pending" && Object.keys(snapshot.tasks).length > 0) {
|
|
659
|
+
const taskIds = Object.keys(snapshot.tasks);
|
|
660
|
+
snapshot.current = taskIds[0] ?? null;
|
|
625
661
|
}
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
snapshot.current =
|
|
662
|
+
|
|
663
|
+
snapshot.state = "pending";
|
|
664
|
+
snapshot.current = resetTask ?? snapshot.current;
|
|
665
|
+
snapshot.currentStage = null;
|
|
666
|
+
snapshot.progress = getProgressPercent(snapshot);
|
|
667
|
+
});
|
|
668
|
+
} catch (err) {
|
|
669
|
+
if (err instanceof StatusCorruptError) return statusCorruptResponse(err);
|
|
670
|
+
throw err;
|
|
629
671
|
}
|
|
630
672
|
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
673
|
+
return sendJson(202, {
|
|
674
|
+
ok: true,
|
|
675
|
+
jobId,
|
|
676
|
+
stopped: pidFound,
|
|
677
|
+
resetTask,
|
|
678
|
+
signal: usedSignal,
|
|
679
|
+
});
|
|
680
|
+
} finally {
|
|
681
|
+
try {
|
|
682
|
+
await releaseJobSlot(
|
|
683
|
+
getPipelineDataDir(dataDir),
|
|
684
|
+
jobId,
|
|
685
|
+
getOrchestratorRuntimeConfig().lockFileTimeout,
|
|
686
|
+
);
|
|
687
|
+
} catch (releaseErr) {
|
|
688
|
+
console.error(`[handleJobStop] Failed to release slot for job ${jobId}:`, releaseErr);
|
|
689
|
+
}
|
|
639
690
|
}
|
|
640
|
-
|
|
641
|
-
return sendJson(202, {
|
|
642
|
-
ok: true,
|
|
643
|
-
jobId,
|
|
644
|
-
stopped: pidFound,
|
|
645
|
-
resetTask,
|
|
646
|
-
signal: usedSignal,
|
|
647
|
-
});
|
|
648
691
|
} finally {
|
|
649
692
|
endStop(jobId);
|
|
650
693
|
}
|
|
@@ -728,10 +771,16 @@ export async function handleTaskStart(
|
|
|
728
771
|
return sendJson(404, createErrorResponse("task_not_found", `task "${taskId}" was not found in the pipeline definition`));
|
|
729
772
|
}
|
|
730
773
|
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
774
|
+
const predecessors = pipelineTaskOrder.slice(0, taskIndex).map((name) => ({
|
|
775
|
+
name,
|
|
776
|
+
state: snapshot.tasks[name]?.state ?? "pending",
|
|
777
|
+
}));
|
|
778
|
+
if (!areDependenciesSatisfied(predecessors.map((p) => p.state))) {
|
|
779
|
+
const blocker = firstUnsatisfiedDependency(predecessors)!;
|
|
780
|
+
return sendJson(412, createErrorResponse(
|
|
781
|
+
"dependencies_not_satisfied",
|
|
782
|
+
`task "${blocker.name}" must be done or skipped before "${taskId}" (state: "${blocker.state}")`,
|
|
783
|
+
));
|
|
735
784
|
}
|
|
736
785
|
|
|
737
786
|
const slot = await acquireConcurrencySlot(dataDir, jobId, "task-start");
|
|
@@ -80,5 +80,6 @@ export async function handleJobDetail(jobId: string): Promise<Response> {
|
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
const [apiJob] = transformJobListForAPI([job], { includePipelineMetadata: true });
|
|
83
|
+
if (apiJob) apiJob.costs = job.costs;
|
|
83
84
|
return sendJson(200, { ok: true, data: apiJob });
|
|
84
85
|
}
|
|
@@ -1,15 +1,51 @@
|
|
|
1
|
+
import { dirname, resolve } from "node:path";
|
|
2
|
+
|
|
3
|
+
import { resolvePipelineSync, resolveTaskRegistryPath } from "../../../config/pipeline-registry";
|
|
4
|
+
import { loadFreshModule } from "../../../core/module-loader";
|
|
5
|
+
import { analyzeTask } from "../../../task-analysis/analyzer";
|
|
6
|
+
import { TaskAnalysisRepository } from "../../../task-analysis/repository";
|
|
7
|
+
import type { PersistedTaskAnalysis, TaskAnalysis } from "../../../task-analysis/types";
|
|
1
8
|
import { acquireLock, releaseLock } from "../../state/analysis-lock";
|
|
2
9
|
|
|
3
|
-
|
|
10
|
+
/** Doc-shaped placeholder persisted when per-task analysis throws unexpectedly. */
|
|
11
|
+
function emptyAnalysis(): TaskAnalysis {
|
|
12
|
+
return {
|
|
13
|
+
summary: "",
|
|
14
|
+
stages: [],
|
|
15
|
+
artifacts: { reads: [], writes: [] },
|
|
16
|
+
models: [],
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Loads the task-name -> module-path map (default export of the task registry module). */
|
|
21
|
+
async function loadTaskRegistryMap(registryPath: string): Promise<Record<string, string>> {
|
|
22
|
+
const mod = await loadFreshModule(registryPath);
|
|
23
|
+
return (mod["default"] ?? {}) as Record<string, string>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function handlePipelineAnalysis(_req: Request, slug: string, root: string): Promise<Response> {
|
|
4
27
|
const lock = acquireLock(slug);
|
|
5
28
|
if (!lock.acquired) {
|
|
6
|
-
return new Response(
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
29
|
+
return new Response(
|
|
30
|
+
JSON.stringify({ ok: false, code: "BAD_REQUEST", message: `analysis lock held by ${lock.heldBy}` }),
|
|
31
|
+
{
|
|
32
|
+
status: 409,
|
|
33
|
+
headers: { "Content-Type": "application/json" },
|
|
34
|
+
},
|
|
35
|
+
);
|
|
10
36
|
}
|
|
11
37
|
|
|
12
38
|
const encoder = new TextEncoder();
|
|
39
|
+
// Shared across start()'s finally and cancel() so the lock is released exactly
|
|
40
|
+
// once regardless of which path runs first (client disconnect vs. completion).
|
|
41
|
+
let released = false;
|
|
42
|
+
const releaseOnce = (): void => {
|
|
43
|
+
if (!released) {
|
|
44
|
+
released = true;
|
|
45
|
+
releaseLock(slug);
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
|
|
13
49
|
const response = new Response(
|
|
14
50
|
new ReadableStream<Uint8Array>({
|
|
15
51
|
start(controller) {
|
|
@@ -17,19 +53,71 @@ export async function handlePipelineAnalysis(_req: Request, slug: string): Promi
|
|
|
17
53
|
controller.enqueue(encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`));
|
|
18
54
|
};
|
|
19
55
|
|
|
20
|
-
|
|
56
|
+
void (async () => {
|
|
21
57
|
try {
|
|
22
|
-
|
|
58
|
+
// Resolution failures (unknown slug, unloadable registry) must not abort the
|
|
59
|
+
// run: emit an error and still reach `complete`.
|
|
60
|
+
let registryDir = "";
|
|
61
|
+
let taskMap: Record<string, string> = {};
|
|
62
|
+
try {
|
|
63
|
+
const { tasksDir } = resolvePipelineSync(root, slug);
|
|
64
|
+
const registryPath = resolveTaskRegistryPath(tasksDir);
|
|
65
|
+
registryDir = dirname(registryPath);
|
|
66
|
+
taskMap = await loadTaskRegistryMap(registryPath);
|
|
67
|
+
} catch (err) {
|
|
68
|
+
send("error", { message: err instanceof Error ? err.message : String(err) });
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const taskNames = Object.keys(taskMap);
|
|
72
|
+
send("started", { slug, totalTasks: taskNames.length });
|
|
73
|
+
|
|
74
|
+
for (const taskName of taskNames) {
|
|
75
|
+
send("task:start", { task: taskName });
|
|
76
|
+
try {
|
|
77
|
+
const sourcePath = resolve(registryDir, taskMap[taskName]!);
|
|
78
|
+
const source = await Bun.file(sourcePath).text();
|
|
79
|
+
const result = await analyzeTask(source);
|
|
80
|
+
const persisted: PersistedTaskAnalysis = {
|
|
81
|
+
summary: result.summary,
|
|
82
|
+
stages: result.stages,
|
|
83
|
+
artifacts: result.artifacts,
|
|
84
|
+
models: result.models,
|
|
85
|
+
analyzedAt: new Date().toISOString(),
|
|
86
|
+
analysisStatus: result.analysisStatus,
|
|
87
|
+
...(result.analysisError !== undefined ? { analysisError: result.analysisError } : {}),
|
|
88
|
+
};
|
|
89
|
+
await TaskAnalysisRepository.write(root, slug, taskName, persisted);
|
|
90
|
+
} catch (err) {
|
|
91
|
+
// Any unexpected throw (source read, persist) persists an `error`
|
|
92
|
+
// placeholder and reports it, without aborting the run.
|
|
93
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
94
|
+
const placeholder: PersistedTaskAnalysis = {
|
|
95
|
+
...emptyAnalysis(),
|
|
96
|
+
analyzedAt: new Date().toISOString(),
|
|
97
|
+
analysisStatus: "error",
|
|
98
|
+
analysisError: message,
|
|
99
|
+
};
|
|
100
|
+
try {
|
|
101
|
+
await TaskAnalysisRepository.write(root, slug, taskName, placeholder);
|
|
102
|
+
} catch {}
|
|
103
|
+
send("error", { task: taskName, message });
|
|
104
|
+
}
|
|
105
|
+
send("task:complete", { task: taskName });
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
send("complete", { slug });
|
|
109
|
+
} catch (err) {
|
|
110
|
+
send("error", { message: err instanceof Error ? err.message : String(err) });
|
|
23
111
|
send("complete", { slug });
|
|
24
|
-
controller.close();
|
|
25
112
|
} finally {
|
|
26
|
-
|
|
113
|
+
controller.close();
|
|
114
|
+
releaseOnce();
|
|
27
115
|
}
|
|
28
|
-
});
|
|
116
|
+
})();
|
|
29
117
|
},
|
|
30
118
|
cancel() {
|
|
31
119
|
try {
|
|
32
|
-
|
|
120
|
+
releaseOnce();
|
|
33
121
|
} catch {}
|
|
34
122
|
},
|
|
35
123
|
}),
|
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
|
|
3
|
+
import { resolveUnderRoot } from "../../../utils/path-containment";
|
|
3
4
|
import { sendJson } from "../utils/http-utils";
|
|
4
5
|
|
|
5
|
-
export async function handleSchemaFile(
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
export async function handleSchemaFile(
|
|
7
|
+
_req: Request,
|
|
8
|
+
slug: string,
|
|
9
|
+
filename: string,
|
|
10
|
+
root: string,
|
|
11
|
+
): Promise<Response> {
|
|
12
|
+
const filePath = resolveUnderRoot(path.join(root, "pipeline-config"), slug, "schemas", filename);
|
|
13
|
+
if (filePath === null) {
|
|
14
|
+
return sendJson(400, { ok: false, code: "BAD_REQUEST", message: "invalid path" });
|
|
15
|
+
}
|
|
8
16
|
if (!(await Bun.file(filePath).exists())) {
|
|
9
17
|
return sendJson(404, { ok: false, code: "NOT_FOUND", message: "schema file not found" });
|
|
10
18
|
}
|
|
@@ -1,12 +1,28 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
|
|
3
|
+
import { TaskAnalysisRepository } from "../../../task-analysis/repository";
|
|
4
|
+
import { resolveUnderRoot } from "../../../utils/path-containment";
|
|
3
5
|
import { sendJson } from "../utils/http-utils";
|
|
4
6
|
|
|
5
|
-
export async function handleTaskAnalysis(
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
export async function handleTaskAnalysis(
|
|
8
|
+
_req: Request,
|
|
9
|
+
slug: string,
|
|
10
|
+
taskId: string,
|
|
11
|
+
root: string,
|
|
12
|
+
): Promise<Response> {
|
|
13
|
+
const validatedPath = resolveUnderRoot(
|
|
14
|
+
path.join(root, "pipeline-config"),
|
|
15
|
+
slug,
|
|
16
|
+
"tasks",
|
|
17
|
+
`${taskId}.analysis.json`,
|
|
18
|
+
);
|
|
19
|
+
if (validatedPath === null) {
|
|
20
|
+
return sendJson(400, { ok: false, code: "BAD_PATH", message: "invalid task-analysis path" });
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const analysis = await TaskAnalysisRepository.readFromPath(validatedPath);
|
|
24
|
+
if (analysis === null) {
|
|
9
25
|
return sendJson(404, { ok: false, code: "NOT_FOUND", message: "analysis file not found" });
|
|
10
26
|
}
|
|
11
|
-
return sendJson(200, { ok: true, data:
|
|
27
|
+
return sendJson(200, { ok: true, data: analysis });
|
|
12
28
|
}
|
|
@@ -3,12 +3,11 @@ import path from "node:path";
|
|
|
3
3
|
|
|
4
4
|
import { sendJson } from "../utils/http-utils";
|
|
5
5
|
|
|
6
|
-
export async function handleTaskSave(req: Request): Promise<Response> {
|
|
6
|
+
export async function handleTaskSave(req: Request, root: string): Promise<Response> {
|
|
7
7
|
const body = (await req.json()) as Record<string, unknown>;
|
|
8
8
|
const slug = typeof body["slug"] === "string" ? body["slug"] : "";
|
|
9
9
|
const taskId = typeof body["taskId"] === "string" ? body["taskId"] : "";
|
|
10
10
|
const content = typeof body["content"] === "string" ? body["content"] : "";
|
|
11
|
-
const root = process.env["PO_ROOT"] ?? process.cwd();
|
|
12
11
|
const taskPath = path.join(root, "pipeline-config", slug, "tasks", `${taskId}.md`);
|
|
13
12
|
await mkdir(path.dirname(taskPath), { recursive: true });
|
|
14
13
|
await Bun.write(taskPath, content);
|
|
@@ -1,21 +1,14 @@
|
|
|
1
|
-
import { mkdir } from "node:fs/promises";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
|
|
4
1
|
import { parseMultipartFormData, sendJson } from "../utils/http-utils";
|
|
5
2
|
import { extractSeedZip } from "../zip-utils";
|
|
6
3
|
import { createErrorResponse } from "../config-bridge";
|
|
7
4
|
import { Constants } from "../config-bridge-node";
|
|
8
|
-
import {
|
|
5
|
+
import { submitUploadedSeed } from "../../../core/job-submission.ts";
|
|
9
6
|
|
|
10
7
|
export interface SeedUploadResult {
|
|
11
8
|
seedObject: Record<string, unknown>;
|
|
12
9
|
artifacts?: Array<{ filename: string; content: Uint8Array }>;
|
|
13
10
|
}
|
|
14
11
|
|
|
15
|
-
function nextJobId(): string {
|
|
16
|
-
return `job-${Date.now()}`;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
12
|
export async function normalizeSeedUpload(req: Request): Promise<SeedUploadResult> {
|
|
20
13
|
const contentType = req.headers.get("content-type") ?? "";
|
|
21
14
|
if (contentType.includes("application/json")) {
|
|
@@ -34,39 +27,11 @@ export async function handleSeedUploadDirect(
|
|
|
34
27
|
dataDir: string,
|
|
35
28
|
artifacts: Array<{ filename: string; content: Uint8Array }> = [],
|
|
36
29
|
): Promise<Response> {
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
// Stage artifacts under staging/{jobId}/ (before the trigger file).
|
|
42
|
-
// The orchestrator owns current/{jobId}/ and moves staged artifacts into
|
|
43
|
-
// current/{jobId}/files/artifacts/ during promotion.
|
|
44
|
-
if (artifacts.length > 0) {
|
|
45
|
-
const stagingJobDir = path.join(pipelineData, "staging", jobId);
|
|
46
|
-
|
|
47
|
-
for (const artifact of artifacts) {
|
|
48
|
-
if (resolveWithin(stagingJobDir, artifact.filename) === null) {
|
|
49
|
-
return sendJson(400, createErrorResponse(Constants.ERROR_CODES.BAD_REQUEST, `unsafe artifact path: ${artifact.filename}`));
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
for (const artifact of artifacts) {
|
|
54
|
-
const resolved = resolveWithin(stagingJobDir, artifact.filename)!;
|
|
55
|
-
await mkdir(path.dirname(resolved), { recursive: true });
|
|
56
|
-
await Bun.write(resolved, artifact.content);
|
|
57
|
-
}
|
|
30
|
+
const result = await submitUploadedSeed({ root: dataDir, seedObject, artifacts });
|
|
31
|
+
if (!result.success) {
|
|
32
|
+
return sendJson(400, createErrorResponse(Constants.ERROR_CODES.BAD_REQUEST, result.message));
|
|
58
33
|
}
|
|
59
|
-
|
|
60
|
-
// Write the seed to pending/ as the trigger file for the orchestrator.
|
|
61
|
-
// The orchestrator watches pending/*.json and handles the full lifecycle:
|
|
62
|
-
// move to current/, write tasks-status.json, and spawn the pipeline runner.
|
|
63
|
-
await mkdir(pendingDir, { recursive: true });
|
|
64
|
-
await Bun.write(
|
|
65
|
-
path.join(pendingDir, `${jobId}-seed.json`),
|
|
66
|
-
JSON.stringify(seedObject, null, 2),
|
|
67
|
-
);
|
|
68
|
-
|
|
69
|
-
return sendJson(201, { ok: true, data: { jobId } });
|
|
34
|
+
return sendJson(201, { ok: true, data: { jobId: result.jobId } });
|
|
70
35
|
}
|
|
71
36
|
|
|
72
37
|
export async function handleSeedUpload(req: Request, dataDir: string): Promise<Response> {
|
package/src/ui/server/index.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { emitSseEvent, type SseEvent } from "../../config/sse-events.ts";
|
|
4
|
+
import { getPipelineDataDir, resolveWorkspaceRoot } from "../../config/paths";
|
|
5
|
+
import { getConfig, getOrchestratorConfig } from "../../core/config";
|
|
5
6
|
import { loadEnvironment } from "../../core/environment";
|
|
6
7
|
import { getConcurrencyRuntimePaths } from "../../core/job-concurrency";
|
|
7
8
|
import { createLogger } from "../../core/logger";
|
|
@@ -19,7 +20,8 @@ import { sseRegistry } from "./sse-registry";
|
|
|
19
20
|
const logger = createLogger("ui-server");
|
|
20
21
|
|
|
21
22
|
export interface ServerOptions {
|
|
22
|
-
|
|
23
|
+
/** Workspace root (not the `pipeline-data` subdir); omitted values fall back to `PO_ROOT` at boot. */
|
|
24
|
+
dataDir?: string;
|
|
23
25
|
port?: number;
|
|
24
26
|
}
|
|
25
27
|
|
|
@@ -62,8 +64,8 @@ export async function initializeWatcher(dataDir: string): Promise<void> {
|
|
|
62
64
|
);
|
|
63
65
|
}
|
|
64
66
|
|
|
65
|
-
export function createServer(dataDir
|
|
66
|
-
const resolvedDataDir = dataDir
|
|
67
|
+
export function createServer(dataDir: string): { fetch: (req: Request) => Promise<Response> } {
|
|
68
|
+
const resolvedDataDir = dataDir;
|
|
67
69
|
const rawOrigins = process.env["PO_CORS_ORIGINS"];
|
|
68
70
|
const allowNull = process.env["PO_CORS_ALLOW_NULL_ORIGIN"];
|
|
69
71
|
const cors = parseCorsConfig(rawOrigins, allowNull === "1" || allowNull === "true");
|
|
@@ -72,18 +74,21 @@ export function createServer(dataDir?: string): { fetch: (req: Request) => Promi
|
|
|
72
74
|
}
|
|
73
75
|
|
|
74
76
|
export async function startServer(options: ServerOptions): Promise<ServerHandle> {
|
|
75
|
-
|
|
76
|
-
throw new Error("PO_ROOT is required in non-test environments");
|
|
77
|
-
}
|
|
77
|
+
const root = resolveWorkspaceRoot(options.dataDir);
|
|
78
78
|
|
|
79
|
-
await loadEnvironment({ rootDir:
|
|
80
|
-
|
|
79
|
+
await loadEnvironment({ rootDir: root });
|
|
80
|
+
getConfig();
|
|
81
|
+
initPATHS(root);
|
|
81
82
|
|
|
82
83
|
const port = options.port ?? 4000;
|
|
83
|
-
const app = createServer(
|
|
84
|
+
const app = createServer(root);
|
|
84
85
|
|
|
85
86
|
let heartbeat: ReturnType<typeof setInterval> | null = setInterval(() => {
|
|
86
|
-
|
|
87
|
+
const event: SseEvent = {
|
|
88
|
+
type: "heartbeat",
|
|
89
|
+
data: { ok: true, timestamp: new Date().toISOString() },
|
|
90
|
+
};
|
|
91
|
+
emitSseEvent(sseRegistry, event);
|
|
87
92
|
}, 30_000);
|
|
88
93
|
|
|
89
94
|
let server: Bun.Server<undefined> | null = null;
|
|
@@ -112,7 +117,7 @@ export async function startServer(options: ServerOptions): Promise<ServerHandle>
|
|
|
112
117
|
]);
|
|
113
118
|
await startup;
|
|
114
119
|
try {
|
|
115
|
-
await initializeWatcher(
|
|
120
|
+
await initializeWatcher(root);
|
|
116
121
|
} catch (error) {
|
|
117
122
|
if (heartbeat) clearInterval(heartbeat);
|
|
118
123
|
heartbeat = null;
|
|
@@ -164,7 +169,7 @@ if (
|
|
|
164
169
|
process.argv[1] !== process.execPath &&
|
|
165
170
|
import.meta.url === Bun.pathToFileURL(process.argv[1]).href
|
|
166
171
|
) {
|
|
167
|
-
const dataDir = process.env["PO_ROOT"]
|
|
172
|
+
const dataDir = process.env["PO_ROOT"];
|
|
168
173
|
startServer({ dataDir }).catch((err) => {
|
|
169
174
|
console.error("Failed to start server:", err);
|
|
170
175
|
process.exit(1);
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
|
|
1
3
|
import type { ErrorEnvelope } from "./config-bridge";
|
|
2
4
|
import {
|
|
3
5
|
Constants,
|
|
4
6
|
createErrorResponse,
|
|
5
7
|
getTasksStatusPath,
|
|
6
8
|
isLocked,
|
|
9
|
+
resolvePipelinePaths,
|
|
7
10
|
validateJobId,
|
|
8
11
|
} from "./config-bridge-node";
|
|
9
12
|
import { getFileReadingStats, readFileWithRetry } from "./file-reader";
|
|
@@ -30,13 +33,22 @@ export interface JobReadingStats {
|
|
|
30
33
|
|
|
31
34
|
const READ_LOCATIONS = ["current", "complete"] as const;
|
|
32
35
|
|
|
33
|
-
|
|
36
|
+
function getTasksStatusPathForRoot(
|
|
37
|
+
jobId: string,
|
|
38
|
+
location: (typeof READ_LOCATIONS)[number],
|
|
39
|
+
root?: string,
|
|
40
|
+
): string {
|
|
41
|
+
if (root === undefined) return getTasksStatusPath(jobId, location);
|
|
42
|
+
return join(resolvePipelinePaths(root)[location], jobId, "tasks-status.json");
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export async function readJob(jobId: string, root?: string): Promise<JobReadResult> {
|
|
34
46
|
if (!validateJobId(jobId)) {
|
|
35
47
|
return { ...createErrorResponse(Constants.ERROR_CODES.BAD_REQUEST, "invalid job id"), jobId, location: "" };
|
|
36
48
|
}
|
|
37
49
|
|
|
38
50
|
for (const location of READ_LOCATIONS) {
|
|
39
|
-
const path =
|
|
51
|
+
const path = getTasksStatusPathForRoot(jobId, location, root);
|
|
40
52
|
const result = await readFileWithRetry(path);
|
|
41
53
|
if (!result.ok) {
|
|
42
54
|
if (result.code === Constants.ERROR_CODES.NOT_FOUND) continue;
|