@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/docs/http-api.md +80 -4
  2. package/package.json +1 -4
  3. package/src/api/__tests__/index.test.ts +443 -32
  4. package/src/api/index.ts +108 -127
  5. package/src/cli/__tests__/analyze-task.test.ts +40 -7
  6. package/src/cli/__tests__/index.test.ts +337 -17
  7. package/src/cli/__tests__/types.test.ts +0 -18
  8. package/src/cli/__tests__/update-pipeline-json.test.ts +19 -9
  9. package/src/cli/analyze-task.ts +3 -14
  10. package/src/cli/index.ts +73 -61
  11. package/src/cli/types.ts +0 -13
  12. package/src/cli/update-pipeline-json.ts +3 -14
  13. package/src/config/__tests__/paths.test.ts +46 -1
  14. package/src/config/__tests__/pipeline-registry.test.ts +767 -0
  15. package/src/config/__tests__/sse-events.test.ts +470 -0
  16. package/src/config/__tests__/statuses.test.ts +41 -0
  17. package/src/config/paths.ts +11 -2
  18. package/src/config/pipeline-registry.ts +258 -0
  19. package/src/config/sse-events.ts +122 -0
  20. package/src/config/statuses.ts +20 -1
  21. package/src/core/__tests__/agent-step.test.ts +115 -0
  22. package/src/core/__tests__/config.test.ts +545 -105
  23. package/src/core/__tests__/core-boot-resolution.test.ts +181 -0
  24. package/src/core/__tests__/job-concurrency.test.ts +260 -0
  25. package/src/core/__tests__/job-submission.test.ts +396 -0
  26. package/src/core/__tests__/job-view.test.ts +1341 -0
  27. package/src/core/__tests__/json-file.test.ts +111 -0
  28. package/src/core/__tests__/lifecycle-policy.test.ts +81 -7
  29. package/src/core/__tests__/logger.test.ts +22 -0
  30. package/src/core/__tests__/orchestrator-no-deprecated-exports.test.ts +41 -0
  31. package/src/core/__tests__/orchestrator.test.ts +615 -2
  32. package/src/core/__tests__/pipeline-runner.test.ts +946 -9
  33. package/src/core/__tests__/redact.test.ts +153 -0
  34. package/src/core/__tests__/runner-liveness.test.ts +910 -0
  35. package/src/core/__tests__/seed-naming.test.ts +57 -0
  36. package/src/core/__tests__/single-derivation.test.ts +159 -0
  37. package/src/core/__tests__/task-runner.test.ts +594 -3
  38. package/src/core/__tests__/task-telemetry.test.ts +241 -0
  39. package/src/core/__tests__/workspace-root-resolution-guard.test.ts +62 -0
  40. package/src/core/agent-step.ts +4 -1
  41. package/src/core/agent-types.ts +5 -4
  42. package/src/core/config.ts +134 -222
  43. package/src/core/control.ts +3 -0
  44. package/src/core/job-concurrency.ts +56 -20
  45. package/src/core/job-submission.ts +133 -0
  46. package/src/core/job-view.ts +473 -0
  47. package/src/core/json-file.ts +45 -0
  48. package/src/core/lifecycle-policy.ts +23 -8
  49. package/src/core/logger.ts +0 -29
  50. package/src/core/orchestrator.ts +200 -74
  51. package/src/core/pipeline-runner.ts +85 -53
  52. package/src/core/redact.ts +40 -0
  53. package/src/core/runner-liveness.ts +280 -0
  54. package/src/core/seed-naming.ts +9 -0
  55. package/src/core/status-writer.ts +27 -33
  56. package/src/core/task-runner.ts +356 -319
  57. package/src/core/task-telemetry.ts +107 -0
  58. package/src/harness/__tests__/subprocess.test.ts +112 -1
  59. package/src/harness/subprocess.ts +55 -16
  60. package/src/llm/__tests__/index.test.ts +684 -33
  61. package/src/llm/index.ts +310 -67
  62. package/src/providers/__tests__/alibaba.test.ts +67 -6
  63. package/src/providers/__tests__/anthropic.test.ts +35 -14
  64. package/src/providers/__tests__/base.test.ts +62 -0
  65. package/src/providers/__tests__/claude-code.test.ts +99 -14
  66. package/src/providers/__tests__/deepseek.test.ts +16 -6
  67. package/src/providers/__tests__/gemini.test.ts +47 -25
  68. package/src/providers/__tests__/moonshot.test.ts +27 -0
  69. package/src/providers/__tests__/openai.test.ts +262 -74
  70. package/src/providers/__tests__/opencode.test.ts +77 -0
  71. package/src/providers/__tests__/request-timeout-contract.test.ts +226 -0
  72. package/src/providers/__tests__/stream-accumulator.test.ts +52 -0
  73. package/src/providers/__tests__/types.test.ts +85 -0
  74. package/src/providers/__tests__/zero-fill-guard.test.ts +62 -0
  75. package/src/providers/__tests__/zhipu.test.ts +27 -14
  76. package/src/providers/alibaba.ts +20 -39
  77. package/src/providers/anthropic.ts +23 -11
  78. package/src/providers/base.ts +19 -0
  79. package/src/providers/claude-code.ts +27 -18
  80. package/src/providers/deepseek.ts +9 -28
  81. package/src/providers/gemini.ts +20 -58
  82. package/src/providers/moonshot.ts +15 -11
  83. package/src/providers/openai.ts +79 -61
  84. package/src/providers/opencode.ts +16 -33
  85. package/src/providers/stream-accumulator.ts +27 -21
  86. package/src/providers/types.ts +29 -4
  87. package/src/providers/zhipu.ts +15 -44
  88. package/src/task-analysis/__tests__/analyzer.test.ts +0 -0
  89. package/src/task-analysis/__tests__/enrichers-schema-deducer.test.ts +34 -2
  90. package/src/task-analysis/__tests__/no-ast-imports.test.ts +52 -0
  91. package/src/task-analysis/__tests__/repository.test.ts +65 -0
  92. package/src/task-analysis/__tests__/types.test.ts +63 -130
  93. package/src/task-analysis/analyzer.ts +91 -0
  94. package/src/task-analysis/enrichers/schema-deducer.ts +13 -2
  95. package/src/task-analysis/index.ts +2 -36
  96. package/src/task-analysis/repository.ts +45 -0
  97. package/src/task-analysis/types.ts +42 -58
  98. package/src/ui/client/__tests__/api.test.ts +143 -1
  99. package/src/ui/client/__tests__/bootstrap.test.ts +178 -62
  100. package/src/ui/client/__tests__/job-adapter.test.ts +203 -2
  101. package/src/ui/client/__tests__/load-state.test.ts +70 -0
  102. package/src/ui/client/__tests__/types.test.ts +66 -3
  103. package/src/ui/client/__tests__/useJobDetailWithUpdates.test.ts +390 -77
  104. package/src/ui/client/__tests__/useJobList.test.ts +198 -23
  105. package/src/ui/client/__tests__/useJobListWithUpdates.test.ts +218 -7
  106. package/src/ui/client/adapters/__tests__/job-adapter.test.ts +186 -0
  107. package/src/ui/client/adapters/job-adapter.ts +41 -16
  108. package/src/ui/client/api.ts +38 -15
  109. package/src/ui/client/bootstrap.ts +19 -14
  110. package/src/ui/client/hooks/useJobDetailWithUpdates.ts +73 -97
  111. package/src/ui/client/hooks/useJobList.ts +26 -31
  112. package/src/ui/client/hooks/useJobListWithUpdates.ts +42 -76
  113. package/src/ui/client/load-state.ts +20 -0
  114. package/src/ui/client/reducers/__tests__/job-events.test.ts +568 -0
  115. package/src/ui/client/reducers/job-events.ts +137 -0
  116. package/src/ui/client/types.ts +16 -20
  117. package/src/ui/components/DAGGrid.tsx +6 -1
  118. package/src/ui/components/JobDetail.tsx +12 -4
  119. package/src/ui/components/JobTable.tsx +41 -13
  120. package/src/ui/components/StageTimeline.tsx +8 -20
  121. package/src/ui/components/TaskAnalysisDisplay.tsx +48 -6
  122. package/src/ui/components/__tests__/DAGGrid.test.tsx +36 -0
  123. package/src/ui/components/__tests__/JobDetail.test.tsx +112 -0
  124. package/src/ui/components/__tests__/JobTable.test.tsx +137 -1
  125. package/src/ui/components/__tests__/StageTimeline.test.tsx +5 -6
  126. package/src/ui/components/__tests__/TaskAnalysisDisplay.test.tsx +64 -0
  127. package/src/ui/components/types.ts +35 -15
  128. package/src/ui/dist/assets/{index--RH3sAt3.js → index-DN3-zvtP.js} +4324 -263
  129. package/src/ui/dist/assets/index-DN3-zvtP.js.map +1 -0
  130. package/src/ui/dist/assets/style-CtZBnjlR.css +2 -0
  131. package/src/ui/dist/index.html +2 -2
  132. package/src/ui/pages/PipelineDetail.tsx +60 -4
  133. package/src/ui/pages/PromptPipelineDashboard.tsx +59 -7
  134. package/src/ui/pages/__tests__/PromptPipelineDashboard.test.tsx +114 -32
  135. package/src/ui/pages/__tests__/pages.test.tsx +236 -42
  136. package/src/ui/server/__tests__/concurrency-endpoint.test.ts +54 -0
  137. package/src/ui/server/__tests__/config-bridge-node.test.ts +34 -1
  138. package/src/ui/server/__tests__/config-bridge.test.ts +9 -1
  139. package/src/ui/server/__tests__/embedded-assets.test.ts +66 -0
  140. package/src/ui/server/__tests__/file-endpoints.test.ts +183 -5
  141. package/src/ui/server/__tests__/gate-endpoints.test.ts +55 -0
  142. package/src/ui/server/__tests__/index.test.ts +63 -0
  143. package/src/ui/server/__tests__/job-control-endpoints.test.ts +512 -2
  144. package/src/ui/server/__tests__/job-endpoints.test.ts +158 -2
  145. package/src/ui/server/__tests__/read-static-path-guard.test.ts +94 -0
  146. package/src/ui/server/__tests__/router-root-isolation.test.ts +204 -0
  147. package/src/ui/server/__tests__/router-threading.test.ts +148 -0
  148. package/src/ui/server/__tests__/router.test.ts +104 -0
  149. package/src/ui/server/__tests__/sse-broadcast.test.ts +44 -0
  150. package/src/ui/server/__tests__/sse-enhancer.test.ts +70 -0
  151. package/src/ui/server/config-bridge-node.ts +8 -26
  152. package/src/ui/server/config-bridge.ts +3 -4
  153. package/src/ui/server/embedded-assets-imports.d.ts +44 -0
  154. package/src/ui/server/embedded-assets.ts +13 -2
  155. package/src/ui/server/endpoints/__tests__/meta-endpoint.test.ts +1 -0
  156. package/src/ui/server/endpoints/__tests__/pipeline-analysis-endpoint.test.ts +167 -0
  157. package/src/ui/server/endpoints/__tests__/schema-file-endpoint.test.ts +104 -0
  158. package/src/ui/server/endpoints/__tests__/task-analysis-endpoint.test.ts +103 -0
  159. package/src/ui/server/endpoints/__tests__/upload-endpoints.test.ts +162 -96
  160. package/src/ui/server/endpoints/concurrency-endpoint.ts +2 -1
  161. package/src/ui/server/endpoints/create-pipeline-endpoint.ts +14 -29
  162. package/src/ui/server/endpoints/file-endpoints.ts +15 -10
  163. package/src/ui/server/endpoints/gate-endpoints.ts +2 -6
  164. package/src/ui/server/endpoints/job-control-endpoints.ts +129 -141
  165. package/src/ui/server/endpoints/job-endpoints.ts +7 -0
  166. package/src/ui/server/endpoints/meta-endpoint.ts +1 -1
  167. package/src/ui/server/endpoints/pipeline-analysis-endpoint.ts +99 -11
  168. package/src/ui/server/endpoints/schema-file-endpoint.ts +11 -3
  169. package/src/ui/server/endpoints/task-analysis-endpoint.ts +21 -5
  170. package/src/ui/server/endpoints/task-save-endpoint.ts +1 -2
  171. package/src/ui/server/endpoints/upload-endpoints.ts +5 -40
  172. package/src/ui/server/index.ts +19 -14
  173. package/src/ui/server/job-reader.ts +14 -2
  174. package/src/ui/server/router.ts +33 -10
  175. package/src/ui/server/sse-broadcast.ts +14 -3
  176. package/src/ui/server/sse-enhancer.ts +12 -2
  177. package/src/ui/state/__tests__/schema-loader.test.ts +11 -1
  178. package/src/ui/state/__tests__/snapshot.test.ts +120 -14
  179. package/src/ui/state/__tests__/types.test.ts +104 -5
  180. package/src/ui/state/snapshot.ts +2 -2
  181. package/src/ui/state/transformers/__tests__/list-transformer.test.ts +85 -2
  182. package/src/ui/state/transformers/__tests__/status-transformer.test.ts +250 -22
  183. package/src/ui/state/transformers/list-transformer.ts +13 -3
  184. package/src/ui/state/transformers/status-transformer.ts +36 -170
  185. package/src/ui/state/types.ts +15 -48
  186. package/src/utils/__tests__/path-containment.test.ts +160 -0
  187. package/src/{ui/server/utils → utils}/path-containment.ts +21 -0
  188. package/src/task-analysis/__tests__/enrichers-analysis-writer.test.ts +0 -86
  189. package/src/task-analysis/__tests__/enrichers-artifact-resolver.test.ts +0 -124
  190. package/src/task-analysis/__tests__/extractors-artifacts.test.ts +0 -133
  191. package/src/task-analysis/__tests__/extractors-llm-calls.test.ts +0 -46
  192. package/src/task-analysis/__tests__/extractors-stages.test.ts +0 -52
  193. package/src/task-analysis/__tests__/index.test.ts +0 -143
  194. package/src/task-analysis/__tests__/parser.test.ts +0 -41
  195. package/src/task-analysis/__tests__/utils-ast.test.ts +0 -82
  196. package/src/task-analysis/enrichers/analysis-writer.ts +0 -75
  197. package/src/task-analysis/enrichers/artifact-resolver.ts +0 -89
  198. package/src/task-analysis/extractors/artifacts.ts +0 -143
  199. package/src/task-analysis/extractors/llm-calls.ts +0 -117
  200. package/src/task-analysis/extractors/stages.ts +0 -45
  201. package/src/task-analysis/parser.ts +0 -20
  202. package/src/task-analysis/utils/ast.ts +0 -45
  203. package/src/ui/dist/assets/index--RH3sAt3.js.map +0 -1
  204. package/src/ui/dist/assets/style-CSSKuMOe.css +0 -2
  205. package/src/ui/embedded-assets.js +0 -12
  206. package/src/ui/server/__tests__/path-containment.test.ts +0 -54
@@ -6,8 +6,8 @@ import { getConfig, getPipelineConfig } from "./config";
6
6
  import { validatePipelineOrThrow } from "./validation";
7
7
  import { loadFreshModule } from "./module-loader";
8
8
  import { atomicWrite, writeJobStatus, flushJobStatus, StatusCorruptError, type GateInfo } from "./status-writer";
9
- import { decideTransition } from "./lifecycle-policy";
10
- import { runPipeline } from "./task-runner";
9
+ import { areDependenciesSatisfied, decideTransition } from "./lifecycle-policy";
10
+ import { runPipeline, recordAgentUsage } from "./task-runner";
11
11
  import type { AuditLogEntry, PipelineResult } from "./task-runner";
12
12
  import type { TaskStateValue } from "../config/statuses";
13
13
  import { ensureTaskSymlinkBridge } from "./symlink-bridge";
@@ -19,6 +19,8 @@ import { TaskState, normalizeTaskState } from "../config/statuses";
19
19
  import { releaseJobSlot } from "./job-concurrency";
20
20
  import { ControlValidationError, parseControlFile, validateControlDirectives, type ControlDirectives } from "./control";
21
21
  import { appendRunEvent } from "./run-events";
22
+ import { resolveWorkspaceRoot } from "../config/paths";
23
+ import { resolveTaskRegistryPath } from "../config/pipeline-registry";
22
24
  import {
23
25
  getTaskName,
24
26
  normalizePipelineTasks,
@@ -136,7 +138,7 @@ export interface ResolvedJobConfig {
136
138
  pipelineSlug: string;
137
139
  pipelineJsonPath: string;
138
140
  tasksDir: string;
139
- taskRegistryPath: string; // Fully resolved module path: PO_TASK_REGISTRY or join(tasksDir, "index.js")
141
+ taskRegistryPath: string; // PO_TASK_REGISTRY or resolveTaskRegistryPath(tasksDir): prefers index.js, falls back to index.ts
140
142
  workDir: string;
141
143
  statusPath: string;
142
144
  startFromTask: string | null;
@@ -301,6 +303,7 @@ async function applyControlStatus(args: {
301
303
  if (args.directives !== null) {
302
304
  taskEntry.controlApplied = true;
303
305
  }
306
+ delete taskEntry.controlError;
304
307
  delete taskEntry.retrying;
305
308
  delete taskEntry.nextRetryAt;
306
309
  delete taskEntry.lastRetryError;
@@ -393,40 +396,54 @@ async function applyControlFileIfPresent(args: {
393
396
  return { gate };
394
397
  }
395
398
 
396
- async function failControlValidation(args: {
399
+ async function haltForInvalidControl(args: {
397
400
  workDir: string;
398
401
  dataDir: string;
399
402
  jobId: string;
400
403
  taskName: string;
404
+ executionTimeMs: number;
405
+ refinementAttempts: number;
406
+ endedAt?: string;
401
407
  error: ControlValidationError;
402
- }): Promise<never> {
403
- const normalized = normalizeError(args.error);
408
+ }): Promise<void> {
404
409
  await appendRunEvent(args.workDir, {
405
410
  type: "control_invalid",
406
411
  task: args.taskName,
407
- message: normalized.message,
412
+ message: args.error.message,
408
413
  at: new Date().toISOString(),
409
414
  });
410
415
 
411
416
  await writeJobStatus(args.workDir, (snapshot) => {
412
- snapshot.state = "failed";
417
+ snapshot.state = "waiting";
413
418
  snapshot.current = args.taskName;
414
419
  snapshot.currentStage = null;
415
- const raw = (snapshot.tasks[args.taskName] ?? {}) as Record<string, unknown>;
416
- raw["state"] = TaskState.FAILED;
417
- raw["endedAt"] = new Date().toISOString();
418
- raw["error"] = { name: normalized.name, message: normalized.message, stack: normalized.stack };
419
- raw["failedStage"] = "control";
420
- delete raw["stageLogPath"];
421
- delete raw["errorContext"];
422
- delete raw["retrying"];
423
- delete raw["nextRetryAt"];
424
- delete raw["lastRetryError"];
425
- snapshot.tasks[args.taskName] = raw as typeof snapshot.tasks[string];
420
+ const taskEntry = (snapshot.tasks[args.taskName] ?? {}) as Record<string, unknown>;
421
+ taskEntry["state"] = TaskState.DONE;
422
+ taskEntry["endedAt"] = args.endedAt ?? new Date().toISOString();
423
+ taskEntry["executionTimeMs"] = args.executionTimeMs;
424
+ taskEntry["refinementAttempts"] = args.refinementAttempts;
425
+ taskEntry["controlApplied"] = false;
426
+ taskEntry["controlError"] = {
427
+ message: args.error.message,
428
+ violations: args.error.violations.length > 0 ? args.error.violations : undefined,
429
+ };
430
+ delete taskEntry["error"];
431
+ delete taskEntry["failedStage"];
432
+ delete taskEntry["stageLogPath"];
433
+ delete taskEntry["errorContext"];
434
+ delete taskEntry["retrying"];
435
+ delete taskEntry["nextRetryAt"];
436
+ delete taskEntry["lastRetryError"];
437
+ snapshot.tasks[args.taskName] = taskEntry as typeof snapshot.tasks[string];
438
+ snapshot.gate = {
439
+ afterTask: args.taskName,
440
+ kind: "control_invalid",
441
+ message: `Control file invalid for ${args.taskName}: ${args.error.message}. Repair tasks/${args.taskName}/control.json and approve to retry.`,
442
+ requestedAt: new Date().toISOString(),
443
+ };
426
444
  });
427
445
 
428
446
  await releaseJobSlotBestEffort(args.dataDir, args.jobId);
429
- process.exit(1);
430
447
  }
431
448
 
432
449
  async function markControlRecoveryChecked(workDir: string, taskName: string): Promise<void> {
@@ -434,6 +451,7 @@ async function markControlRecoveryChecked(workDir: string, taskName: string): Pr
434
451
  const taskEntry = snapshot.tasks[taskName];
435
452
  if (!taskEntry || taskEntry.state !== TaskState.DONE || taskEntry.controlApplied === true) return;
436
453
  taskEntry.controlApplied = true;
454
+ delete taskEntry.controlError;
437
455
  });
438
456
  }
439
457
 
@@ -447,6 +465,7 @@ function buildGateInfo(
447
465
  if (directivePause) {
448
466
  return {
449
467
  afterTask: taskName,
468
+ kind: "approval",
450
469
  message: directivePause.message,
451
470
  artifacts: directivePause.artifacts,
452
471
  requestedAt,
@@ -459,6 +478,7 @@ function buildGateInfo(
459
478
  if (selectedEntry.gate === true) {
460
479
  return {
461
480
  afterTask: taskName,
481
+ kind: "approval",
462
482
  message: defaultMessage,
463
483
  requestedAt,
464
484
  };
@@ -466,6 +486,7 @@ function buildGateInfo(
466
486
 
467
487
  return {
468
488
  afterTask: taskName,
489
+ kind: "approval",
469
490
  message: selectedEntry.gate.message ?? defaultMessage,
470
491
  artifacts: selectedEntry.gate.artifacts,
471
492
  requestedAt,
@@ -497,9 +518,8 @@ export function normalizeError(e: unknown): NormalizedError {
497
518
  return { message: String(e) };
498
519
  }
499
520
 
500
- /** Resolves all runtime configuration for a pipeline job from environment variables and seed.json. */
501
- export async function resolveJobConfig(jobId: string): Promise<ResolvedJobConfig> {
502
- const poRoot = resolve(process.env["PO_ROOT"] ?? process.cwd());
521
+ /** Builds runtime configuration for a pipeline job from a resolved root, environment variables, and seed.json. */
522
+ export async function resolveJobConfig(jobId: string, poRoot: string): Promise<ResolvedJobConfig> {
503
523
  const dataDir = process.env["PO_DATA_DIR"] ?? "pipeline-data";
504
524
  const currentDir = process.env["PO_CURRENT_DIR"] ?? join(poRoot, dataDir, "current");
505
525
  const completeDir = process.env["PO_COMPLETE_DIR"] ?? join(poRoot, dataDir, "complete");
@@ -531,7 +551,7 @@ export async function resolveJobConfig(jobId: string): Promise<ResolvedJobConfig
531
551
  pipelineJsonPath = runScopedPipelineJsonPath;
532
552
  }
533
553
 
534
- const taskRegistryPath = process.env["PO_TASK_REGISTRY"] ?? join(tasksDir, "index.js");
554
+ const taskRegistryPath = process.env["PO_TASK_REGISTRY"] ?? resolveTaskRegistryPath(tasksDir);
535
555
  const startFromTask = process.env["PO_START_FROM_TASK"] ?? null;
536
556
  const runSingleTask = process.env["PO_RUN_SINGLE_TASK"] === "true";
537
557
 
@@ -635,12 +655,13 @@ const RETRY_BACKOFF_MULTIPLIER = 2;
635
655
 
636
656
  /** Runs a pipeline job end-to-end for the given job ID. */
637
657
  export async function runPipelineJob(jobId: string): Promise<void> {
658
+ getConfig();
638
659
  let workDir: string | undefined;
639
660
  let activeTaskName: string | null = null;
640
- const poRoot = resolve(process.env["PO_ROOT"] ?? process.cwd());
661
+ const poRoot = resolveWorkspaceRoot();
641
662
  let dataDir: string | undefined = resolve(poRoot, process.env["PO_DATA_DIR"] ?? "pipeline-data");
642
663
  try {
643
- const config = await resolveJobConfig(jobId);
664
+ const config = await resolveJobConfig(jobId, poRoot);
644
665
  workDir = config.workDir;
645
666
  dataDir = resolve(config.poRoot, config.dataDir);
646
667
  await writePidFile(config.workDir);
@@ -727,13 +748,17 @@ export async function runPipelineJob(jobId: string): Promise<void> {
727
748
  } catch (error) {
728
749
  if (!(error instanceof ControlValidationError)) throw error;
729
750
  activeTaskName = null;
730
- await failControlValidation({
751
+ await haltForInvalidControl({
731
752
  workDir: config.workDir,
732
753
  dataDir,
733
754
  jobId,
734
755
  taskName,
756
+ executionTimeMs: typeof taskStatus?.executionTimeMs === "number" ? taskStatus.executionTimeMs : 0,
757
+ refinementAttempts: typeof taskStatus?.refinementAttempts === "number" ? taskStatus.refinementAttempts : 0,
758
+ endedAt: typeof taskStatus?.endedAt === "string" ? taskStatus.endedAt : undefined,
735
759
  error,
736
760
  });
761
+ return;
737
762
  }
738
763
  }
739
764
  await loadDoneArtifact(config.workDir, taskName, pipelineArtifacts);
@@ -764,25 +789,21 @@ export async function runPipelineJob(jobId: string): Promise<void> {
764
789
  const taskName = selectedEntry.name;
765
790
  const taskState = selectedTaskState;
766
791
 
767
- let predecessorsReady = true;
768
- for (const entry of pipelineTasks) {
769
- if (entry.name === taskName) break;
770
- const state = normalizeTaskState(status.tasks[entry.name]?.state ?? TaskState.PENDING);
771
- if (state !== TaskState.DONE && state !== TaskState.SKIPPED) {
772
- predecessorsReady = false;
773
- break;
774
- }
775
- }
792
+ const predecessorStates = pipelineTasks
793
+ .slice(0, pipelineTasks.findIndex((entry) => entry.name === taskName))
794
+ .map((entry) => normalizeTaskState(status.tasks[entry.name]?.state ?? TaskState.PENDING));
795
+ const dependenciesReady = areDependenciesSatisfied(predecessorStates);
776
796
 
777
- // Check lifecycle policy (bypassed when startFromTask is set)
778
- if (!startFromTask) {
779
- const decision = decideTransition({ op: "start", taskState, dependenciesReady: predecessorsReady });
780
- if (!decision.ok) {
781
- throw Object.assign(
782
- new Error(`Lifecycle policy blocked task start: ${taskName} (reason: ${decision.reason})`),
783
- { httpStatus: 409, error: "unsupported_lifecycle" }
784
- );
785
- }
797
+ const decision = decideTransition({
798
+ op: startFromTask ? "restart" : "start",
799
+ taskState,
800
+ dependenciesReady,
801
+ });
802
+ if (!decision.ok) {
803
+ throw Object.assign(
804
+ new Error(`Lifecycle policy blocked task start: ${taskName} (reason: ${decision.reason})`),
805
+ { httpStatus: 409, error: "unsupported_lifecycle" }
806
+ );
786
807
  }
787
808
 
788
809
  // Update status to RUNNING
@@ -823,12 +844,6 @@ export async function runPipelineJob(jobId: string): Promise<void> {
823
844
  const agentExecutionTimeMs = Date.now() - agentStartTime;
824
845
 
825
846
  if (agentResult.ok) {
826
- const usageKey = `${agentEntry.harness}:${agentEntry.model ?? "default"}`;
827
- const inputTokens = agentResult.usage?.inputTokens ?? 0;
828
- const outputTokens = agentResult.usage?.outputTokens ?? 0;
829
- const costUsd = agentResult.costUsd ?? 0;
830
- const usageTuple = [usageKey, inputTokens, outputTokens, costUsd];
831
-
832
847
  await writeJobStatus(config.workDir, (snapshot) => {
833
848
  snapshot.state = "done";
834
849
  snapshot.current = null;
@@ -838,7 +853,11 @@ export async function runPipelineJob(jobId: string): Promise<void> {
838
853
  taskEntry.endedAt = new Date().toISOString();
839
854
  taskEntry.executionTimeMs = agentExecutionTimeMs;
840
855
  const usage = (taskEntry.tokenUsage ?? []) as unknown[];
841
- usage.push(usageTuple);
856
+ recordAgentUsage(
857
+ { harness: agentEntry.harness, model: agentEntry.model },
858
+ agentResult,
859
+ (t) => usage.push(t),
860
+ );
842
861
  taskEntry.tokenUsage = usage;
843
862
  snapshot.tasks[taskName] = taskEntry;
844
863
  });
@@ -850,6 +869,16 @@ export async function runPipelineJob(jobId: string): Promise<void> {
850
869
  taskEntry.state = "failed";
851
870
  taskEntry.endedAt = new Date().toISOString();
852
871
  taskEntry.error = agentResult.error ?? "Agent step failed";
872
+ recordAgentUsage(
873
+ { harness: agentEntry.harness, model: agentEntry.model },
874
+ agentResult,
875
+ (t) => {
876
+ const usage = (taskEntry.tokenUsage ?? []) as unknown[];
877
+ usage.push(t);
878
+ taskEntry.tokenUsage = usage;
879
+ },
880
+ { failed: true },
881
+ );
853
882
  snapshot.tasks[taskName] = taskEntry;
854
883
  });
855
884
  activeTaskName = null;
@@ -1005,13 +1034,16 @@ export async function runPipelineJob(jobId: string): Promise<void> {
1005
1034
  } catch (error) {
1006
1035
  if (!(error instanceof ControlValidationError)) throw error;
1007
1036
  activeTaskName = null;
1008
- await failControlValidation({
1037
+ await haltForInvalidControl({
1009
1038
  workDir: config.workDir,
1010
1039
  dataDir,
1011
1040
  jobId,
1012
1041
  taskName,
1042
+ executionTimeMs,
1043
+ refinementAttempts,
1013
1044
  error,
1014
1045
  });
1046
+ return;
1015
1047
  }
1016
1048
 
1017
1049
  if (appliedControl === null) {
@@ -0,0 +1,40 @@
1
+ export const REDACTED = "[REDACTED]";
2
+
3
+ const SECRET_KEY_PATTERN = /^(authorization|bearer|token|api[-_]?key|apikey|secret|password|x-api-key)$/i;
4
+
5
+ export interface RedactOptions {
6
+ secrets?: string[];
7
+ }
8
+
9
+ export function redactSecrets(value: unknown, options?: RedactOptions): unknown {
10
+ const secrets = options?.secrets ?? [];
11
+ const seen = new WeakSet<object>();
12
+ return redactValue(value, secrets, seen);
13
+ }
14
+
15
+ function redactString(s: string, secrets: string[]): string {
16
+ let out = s;
17
+ for (const secret of secrets) {
18
+ if (secret === "") continue;
19
+ out = out.split(secret).join(REDACTED);
20
+ }
21
+ return out;
22
+ }
23
+
24
+ function redactValue(value: unknown, secrets: string[], seen: WeakSet<object>): unknown {
25
+ if (typeof value === "function") return value;
26
+ if (value === null) return null;
27
+ if (typeof value === "string") return redactString(value, secrets);
28
+ if (typeof value !== "object") return value;
29
+ if (seen.has(value as object)) return "[Circular]";
30
+ seen.add(value as object);
31
+ if (Array.isArray(value)) {
32
+ return value.map((el) => redactValue(el, secrets, seen));
33
+ }
34
+ return Object.fromEntries(
35
+ Object.entries(value).map(([k, v]) => [
36
+ k,
37
+ SECRET_KEY_PATTERN.test(k) ? REDACTED : redactValue(v, secrets, seen),
38
+ ]),
39
+ );
40
+ }
@@ -0,0 +1,280 @@
1
+ import { readdir } from "node:fs/promises";
2
+ import path from "node:path";
3
+
4
+ import { deriveJobStatusFromTasks } from "../config/statuses";
5
+ import { createLogger } from "./logger";
6
+ import { readJobStatusResult, writeJobStatus } from "./status-writer";
7
+ import type { StatusSnapshot } from "./status-writer";
8
+ import { readJobSlotLease, withRuntimeLock } from "./job-concurrency";
9
+ import type { JobSlotLease } from "./job-concurrency";
10
+
11
+ export const STALE_RUNNING_GRACE_MS = 30_000;
12
+
13
+ export interface StaleRunningInput {
14
+ status: StatusSnapshot;
15
+ pid: number | null;
16
+ pidAlive: boolean;
17
+ nowMs: number;
18
+ staleAfterMs: number;
19
+ }
20
+
21
+ export type StaleRunningDecision =
22
+ | { stale: false; reason: "not_running" | "live_pid" | "fresh_status" }
23
+ | { stale: true; runningTaskId: string | null; ageMs: number };
24
+
25
+ export type ReconcileSource = "child-exit" | "boot-sweep";
26
+
27
+ export type ReconcileReason =
28
+ | "marked-failed"
29
+ | "not-running"
30
+ | "no-running-task"
31
+ | "live-pid"
32
+ | "live-lease"
33
+ | "fresh-status"
34
+ | "unreadable";
35
+
36
+ export interface ReconcileResult {
37
+ reconciled: boolean;
38
+ taskId?: string;
39
+ reason: ReconcileReason;
40
+ }
41
+
42
+ export interface ReconcileOptions {
43
+ dataDir: string;
44
+ jobId: string;
45
+ jobDir: string;
46
+ source: ReconcileSource;
47
+ lockTimeoutMs: number;
48
+ staleLeaseTimeoutMs?: number;
49
+ code?: number | null;
50
+ signal?: string | null;
51
+ requireStaleStatus?: boolean;
52
+ staleAfterMs?: number;
53
+ now?: () => number;
54
+ isAlive?: (pid: number) => boolean;
55
+ }
56
+
57
+ export interface SweepOptions {
58
+ dataDir: string;
59
+ currentDir: string;
60
+ lockTimeoutMs: number;
61
+ staleLeaseTimeoutMs?: number;
62
+ staleAfterMs?: number;
63
+ now?: () => number;
64
+ isAlive?: (pid: number) => boolean;
65
+ }
66
+
67
+ export interface SweepResult {
68
+ scanned: number;
69
+ reconciled: string[];
70
+ }
71
+
72
+ export async function readRunnerPid(jobDir: string): Promise<number | null> {
73
+ try {
74
+ const content = await Bun.file(path.join(jobDir, "runner.pid")).text();
75
+ const pid = parseInt(content.trim(), 10);
76
+ return Number.isNaN(pid) ? null : pid;
77
+ } catch {
78
+ return null;
79
+ }
80
+ }
81
+
82
+ export function isProcessAlive(pid: number): boolean {
83
+ if (!Number.isInteger(pid) || pid <= 0) return false;
84
+ try {
85
+ process.kill(pid, 0);
86
+ return true;
87
+ } catch (err) {
88
+ return (err as NodeJS.ErrnoException).code === "EPERM";
89
+ }
90
+ }
91
+
92
+ function getTaskDerivedStatus(snapshot: StatusSnapshot): string {
93
+ return deriveJobStatusFromTasks(
94
+ Object.values(snapshot.tasks).map((task) => ({ state: task.state })),
95
+ );
96
+ }
97
+
98
+ export function findRunningTaskId(snapshot: StatusSnapshot): string | null {
99
+ if (snapshot.current && snapshot.tasks[snapshot.current]?.state === "running") {
100
+ return snapshot.current;
101
+ }
102
+ return Object.keys(snapshot.tasks).find((taskId) => snapshot.tasks[taskId]?.state === "running") ?? null;
103
+ }
104
+
105
+ export function classifyStaleRunningStatus(input: StaleRunningInput): StaleRunningDecision {
106
+ if (getTaskDerivedStatus(input.status) !== "running") {
107
+ return { stale: false, reason: "not_running" };
108
+ }
109
+ if (input.pid !== null && input.pidAlive) {
110
+ return { stale: false, reason: "live_pid" };
111
+ }
112
+ const updatedMs = Date.parse(input.status.lastUpdated);
113
+ if (!Number.isFinite(updatedMs)) {
114
+ return { stale: true, runningTaskId: findRunningTaskId(input.status), ageMs: 0 };
115
+ }
116
+ const ageMs = input.nowMs - updatedMs;
117
+ if (ageMs < input.staleAfterMs) {
118
+ return { stale: false, reason: "fresh_status" };
119
+ }
120
+ return { stale: true, runningTaskId: findRunningTaskId(input.status), ageMs };
121
+ }
122
+
123
+ const reconcileLogger = createLogger("runner-liveness");
124
+
125
+ function isLeaseLiveOrInFlight(
126
+ lease: JobSlotLease,
127
+ nowMs: number,
128
+ staleLeaseTimeoutMs: number,
129
+ isAlive: (pid: number) => boolean,
130
+ ): boolean {
131
+ if (typeof lease.pid === "number" && lease.pid > 0) {
132
+ return isAlive(lease.pid);
133
+ }
134
+ const acquiredMs = Date.parse(lease.acquiredAt);
135
+ if (!Number.isFinite(acquiredMs)) return false;
136
+ return nowMs - acquiredMs < staleLeaseTimeoutMs;
137
+ }
138
+
139
+ function isFreshStatus(snapshot: StatusSnapshot, nowMs: number, staleAfterMs: number): boolean {
140
+ const updatedMs = Date.parse(snapshot.lastUpdated);
141
+ return Number.isFinite(updatedMs) && nowMs - updatedMs < staleAfterMs;
142
+ }
143
+
144
+ export async function reconcileInterruptedJob(opts: ReconcileOptions): Promise<ReconcileResult> {
145
+ const {
146
+ dataDir,
147
+ jobId,
148
+ jobDir,
149
+ source,
150
+ lockTimeoutMs,
151
+ code,
152
+ signal,
153
+ requireStaleStatus = false,
154
+ staleAfterMs = STALE_RUNNING_GRACE_MS,
155
+ now = Date.now,
156
+ isAlive = isProcessAlive,
157
+ } = opts;
158
+ const staleLeaseTimeoutMs = opts.staleLeaseTimeoutMs ?? lockTimeoutMs;
159
+
160
+ return withRuntimeLock(dataDir, lockTimeoutMs, async () => {
161
+ const statusResult = await readJobStatusResult(jobDir);
162
+ if (!statusResult.ok) {
163
+ return { reconciled: false, reason: "unreadable" } satisfies ReconcileResult;
164
+ }
165
+ const snapshot = statusResult.value;
166
+
167
+ const derived = getTaskDerivedStatus(snapshot);
168
+ if (derived !== "running" && snapshot.state !== "running") {
169
+ return { reconciled: false, reason: "not-running" } satisfies ReconcileResult;
170
+ }
171
+
172
+ const pid = await readRunnerPid(jobDir);
173
+ if (pid !== null && isAlive(pid)) {
174
+ return { reconciled: false, reason: "live-pid" } satisfies ReconcileResult;
175
+ }
176
+
177
+ const lease = await readJobSlotLease(dataDir, jobId);
178
+ if (lease && isLeaseLiveOrInFlight(lease, now(), staleLeaseTimeoutMs, isAlive)) {
179
+ return { reconciled: false, reason: "live-lease" } satisfies ReconcileResult;
180
+ }
181
+
182
+ if (requireStaleStatus && derived === "running") {
183
+ const decision = classifyStaleRunningStatus({
184
+ status: snapshot,
185
+ pid,
186
+ pidAlive: false,
187
+ nowMs: now(),
188
+ staleAfterMs,
189
+ });
190
+ if (!decision.stale) {
191
+ return {
192
+ reconciled: false,
193
+ reason: decision.reason === "not_running" ? "not-running" : "fresh-status",
194
+ } satisfies ReconcileResult;
195
+ }
196
+ } else if (requireStaleStatus && snapshot.state === "running") {
197
+ if (isFreshStatus(snapshot, now(), staleAfterMs)) {
198
+ return { reconciled: false, reason: "fresh-status" } satisfies ReconcileResult;
199
+ }
200
+ }
201
+
202
+ const taskId = findRunningTaskId(snapshot);
203
+ const nowMs = now();
204
+ await writeJobStatus(jobDir, (s) => {
205
+ if (taskId !== null) {
206
+ const task = s.tasks[taskId];
207
+ if (task) {
208
+ task.state = "failed";
209
+ task.endedAt = new Date(nowMs).toISOString();
210
+ task.error = "runner exited without completing this task";
211
+ task.errorContext = { kind: "runner-interrupted", code, signal, source };
212
+ }
213
+ } else {
214
+ s.current = null;
215
+ s.currentStage = null;
216
+ s.error = {
217
+ name: "RunnerInterrupted",
218
+ message: "runner exited while the job was still marked running",
219
+ };
220
+ s.errorContext = { kind: "runner-interrupted", code, signal, source };
221
+ }
222
+ s.state = "failed";
223
+ });
224
+
225
+ reconcileLogger.log("reconciled interrupted job", { jobId, taskId, pid, code, signal, path: source });
226
+
227
+ return taskId === null
228
+ ? { reconciled: true, reason: "marked-failed" } satisfies ReconcileResult
229
+ : { reconciled: true, taskId, reason: "marked-failed" } satisfies ReconcileResult;
230
+ });
231
+ }
232
+
233
+ function isErrnoException(value: unknown): value is NodeJS.ErrnoException {
234
+ return value instanceof Error && typeof (value as NodeJS.ErrnoException).code === "string";
235
+ }
236
+
237
+ export async function reconcileOrphanedCurrentJobs(opts: SweepOptions): Promise<SweepResult> {
238
+ const { dataDir, currentDir, lockTimeoutMs, staleLeaseTimeoutMs, staleAfterMs, now, isAlive } = opts;
239
+
240
+ let entries: string[];
241
+ try {
242
+ const dirents = await readdir(currentDir, { withFileTypes: true });
243
+ entries = dirents.filter((entry) => entry.isDirectory()).map((entry) => entry.name);
244
+ } catch (err) {
245
+ if (isErrnoException(err) && err.code === "ENOENT") {
246
+ return { scanned: 0, reconciled: [] };
247
+ }
248
+ throw err;
249
+ }
250
+
251
+ const reconciled: string[] = [];
252
+ for (const entry of entries) {
253
+ const jobId = entry;
254
+ const jobDir = path.join(currentDir, entry);
255
+ const result = await reconcileInterruptedJob({
256
+ dataDir,
257
+ jobId,
258
+ jobDir,
259
+ source: "boot-sweep",
260
+ requireStaleStatus: true,
261
+ staleAfterMs,
262
+ lockTimeoutMs,
263
+ staleLeaseTimeoutMs,
264
+ now,
265
+ isAlive,
266
+ });
267
+ if (result.reconciled) {
268
+ reconciled.push(jobId);
269
+ }
270
+ }
271
+
272
+ if (entries.length > 0) {
273
+ reconcileLogger.log("reconciled orphaned current jobs", {
274
+ scanned: entries.length,
275
+ reconciled: reconciled.length,
276
+ });
277
+ }
278
+
279
+ return { scanned: entries.length, reconciled };
280
+ }
@@ -0,0 +1,9 @@
1
+ export const SEED_FILENAME_PATTERN = /^([A-Za-z0-9][A-Za-z0-9-_]*)-seed\.json$/;
2
+
3
+ export function seedFilename(jobId: string): string {
4
+ return `${jobId}-seed.json`;
5
+ }
6
+
7
+ export function parseSeedFilename(name: string): string | null {
8
+ return SEED_FILENAME_PATTERN.exec(name)?.[1] ?? null;
9
+ }