@ryanfw/prompt-orchestration-pipeline 1.2.7 → 1.2.8

package/src/core/config.ts

@@ -16,6 +16,7 @@ export interface TaskRunnerConfig {
   maxRefinementAttempts: number;
   stageTimeout: number;
   llmRequestTimeout: number;
+  maxAttempts: number;
 }
 
 export interface LLMConfig {
@@ -95,6 +96,7 @@ export const defaultConfig = {
     maxRefinementAttempts: 3,
     stageTimeout: 300000,
     llmRequestTimeout: 3600000,
+    maxAttempts: 3,
   },
   llm: {
     defaultProvider: "openai",
@@ -184,6 +186,18 @@ function loadFromEnvironment(config: AppConfig): AppConfig {
     overrides["llm"] = { maxConcurrency: parseInt(maxConcurrencyRaw, 10) };
   }
 
+  const maxAttemptsRaw = process.env["PO_TASK_MAX_ATTEMPTS"];
+  if (maxAttemptsRaw) {
+    const maxAttempts = Number(maxAttemptsRaw);
+    if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
+      throw new Error(`PO_TASK_MAX_ATTEMPTS must be an integer >= 1, got ${maxAttemptsRaw}`);
+    }
+    overrides["taskRunner"] = {
+      ...((overrides["taskRunner"] as PlainObject | undefined) ?? {}),
+      maxAttempts,
+    };
+  }
+
   const shutdownTimeoutRaw = process.env["PO_SHUTDOWN_TIMEOUT"];
   if (shutdownTimeoutRaw) {
     overrides["orchestrator"] = { shutdownTimeout: parseInt(shutdownTimeoutRaw, 10) };
@@ -227,6 +241,9 @@ function validateConfig(config: AppConfig): void {
   if (config.taskRunner.maxRefinementAttempts < 1) {
     errors.push(`taskRunner.maxRefinementAttempts must be >= 1, got ${config.taskRunner.maxRefinementAttempts}`);
   }
+  if (!Number.isInteger(config.taskRunner.maxAttempts) || config.taskRunner.maxAttempts < 1) {
+    errors.push(`taskRunner.maxAttempts must be an integer >= 1, got ${config.taskRunner.maxAttempts}`);
+  }
   if (!(VALID_LOG_LEVELS as readonly string[]).includes(config.logging.level)) {
     errors.push(`logging.level must be one of ${VALID_LOG_LEVELS.join(", ")}, got ${config.logging.level}`);
   }

package/src/core/pipeline-runner.ts

@@ -2,13 +2,13 @@ import { join, dirname, resolve, basename } from "node:path";
 import { pathToFileURL } from "node:url";
 import { unlink, mkdir, rename, appendFile } from "node:fs/promises";
 import { unlinkSync } from "node:fs";
-import { getPipelineConfig } from "./config";
+import { getConfig, getPipelineConfig } from "./config";
 import { validatePipelineOrThrow } from "./validation";
 import { loadFreshModule } from "./module-loader";
 import { writeJobStatus } from "./status-writer";
 import { decideTransition } from "./lifecycle-policy";
 import { runPipeline } from "./task-runner";
-import type { AuditLogEntry } from "./task-runner";
+import type { AuditLogEntry, PipelineResult } from "./task-runner";
 import { ensureTaskSymlinkBridge } from "./symlink-bridge";
 import { validateTaskSymlinks, repairTaskSymlinks, cleanupTaskSymlinks } from "./symlink-utils";
 import { createTaskFileIO, generateLogName } from "./file-io";
@@ -98,6 +98,7 @@ export interface TaskStatus {
   attempts?: number;
   executionTimeMs?: number;
   refinementAttempts?: number;
+  restartCount?: number;
   error?: NormalizedError;
   failedStage?: string;
   stageLogPath?: string;
@@ -256,6 +257,10 @@ export async function loadTaskRegistry(registryPath: string): Promise<TaskRegist
 
 // ─── Pipeline job entry point ─────────────────────────────────────────────────
 
+const INITIAL_RETRY_DELAY_MS = 2_000;
+const MAX_RETRY_DELAY_MS = 30_000;
+const RETRY_BACKOFF_MULTIPLIER = 2;
+
 /** Runs a pipeline job end-to-end for the given job ID. */
 export async function runPipelineJob(jobId: string): Promise<void> {
   let workDir: string | undefined;
@@ -414,8 +419,38 @@ export async function runPipelineJob(jobId: string): Promise<void> {
       },
     };
 
-    // Delegate to task runner
-    const result = await runPipeline(relocatedEntryPath, taskExecutionContext);
+    // Delegate to task runner with bounded retry loop.
+    // Guard against partial test mocks or malformed runtime config.
+    const configuredMaxAttempts = getConfig().taskRunner?.maxAttempts;
+    const maxAttempts = Number.isInteger(configuredMaxAttempts) ? configuredMaxAttempts : 3;
+    const cap = Math.max(1, maxAttempts);
+
+    let result: PipelineResult | undefined;
+    for (let attempt = 1; attempt <= cap; attempt++) {
+      result = await runPipeline(relocatedEntryPath, taskExecutionContext);
+      if (result.ok) break;
+      if (attempt >= cap) break;
+
+      const delay = Math.min(
+        INITIAL_RETRY_DELAY_MS * RETRY_BACKOFF_MULTIPLIER ** (attempt - 1),
+        MAX_RETRY_DELAY_MS,
+      );
+
+      await writeJobStatus(config.workDir, (snapshot) => {
+        const entry = snapshot.tasks[taskName] ?? {};
+        const currentAttempts = typeof entry.attempts === "number" ? entry.attempts : attempt;
+        entry.state = "running";
+        entry.attempts = currentAttempts + 1;
+        entry.restartCount = (entry.restartCount ?? 0) + 1;
+        delete entry.failedStage;
+        delete entry.error;
+        snapshot.tasks[taskName] = entry;
+      });
+
+      await Bun.sleep(delay);
+    }
+
+    if (!result) throw new Error("Retry loop produced no result");
 
     if (result.ok) {
       // Compute execution time from logs

package/src/core/status-writer.ts

@@ -14,6 +14,7 @@ export interface TaskEntry {
   failedStage?: string;
   error?: string;
   attempts?: number;
+  restartCount?: number;
   refinementAttempts?: number;
   tokenUsage?: unknown[];
   startedAt?: string;
@@ -227,6 +228,7 @@ export function resetJobFromTask(jobDir: string, fromTask: string, options?: Res
       delete task.failedStage;
       delete task.error;
       task.attempts = 0;
+      task.restartCount = 0;
       task.refinementAttempts = 0;
       if (options?.clearTokenUsage !== false) {
         task.tokenUsage = [];
@@ -253,6 +255,7 @@ export function resetJobToCleanSlate(jobDir: string, options?: ResetOptions): Pr
       delete task.failedStage;
       delete task.error;
       task.attempts = 0;
+      task.restartCount = 0;
       task.refinementAttempts = 0;
       if (options?.clearTokenUsage !== false) {
         task.tokenUsage = [];
@@ -280,6 +283,7 @@ export function resetSingleTask(jobDir: string, taskId: string, options?: ResetO
     delete task.failedStage;
     delete task.error;
     task.attempts = 0;
+    task.restartCount = 0;
     task.refinementAttempts = 0;
     if (options?.clearTokenUsage !== false) {
       task.tokenUsage = [];

package/src/core/task-runner.ts

@@ -795,7 +795,7 @@ export async function runPipeline(
 
   // Write done status (best-effort)
   try {
-    const lastStage = KNOWN_STAGES[KNOWN_STAGES.length - 1]!;
+    const lastStage = KNOWN_STAGES.at(-1)!;
     const doneProgress = computeDeterministicProgress(
       pipelineTasks ?? [taskName],
       taskName,

package/src/providers/__tests__/base.test.ts

@@ -165,7 +165,7 @@ describe("isRetryableError", () => {
 });
 
 describe("DEFAULT_REQUEST_TIMEOUT_MS", () => {
-  it("is 3 600 000 ms", () => {
+  it("is 1 hour (3 600 000 ms)", () => {
     expect(DEFAULT_REQUEST_TIMEOUT_MS).toBe(3_600_000);
   });
 });

package/src/ui/client/__tests__/job-adapter.test.ts

@@ -30,6 +30,18 @@ describe("job adapter", () => {
     expect(normalizeTasks(null)).toEqual({});
   });
 
+  it("maps numeric restartCount onto the normalized task", () => {
+    expect(normalizeTasks({ t1: { state: "done", restartCount: 2 } })["t1"]?.restartCount).toBe(2);
+  });
+
+  it("treats null restartCount as undefined", () => {
+    expect(normalizeTasks({ t1: { state: "done", restartCount: null } })["t1"]?.restartCount).toBeUndefined();
+  });
+
+  it("leaves restartCount undefined when absent", () => {
+    expect(normalizeTasks({ t1: { state: "done" } })["t1"]?.restartCount).toBeUndefined();
+  });
+
   it("adapts summary jobs with defaults", () => {
     const job = adaptJobSummary({
       jobId: "job-1",

package/src/ui/client/adapters/job-adapter.ts

@@ -49,6 +49,7 @@ function normalizeTask(name: string, rawTask: unknown): NormalizedTask {
     startedAt: toStringOrNull(task["startedAt"]),
     endedAt: toStringOrNull(task["endedAt"]),
     attempts: typeof task["attempts"] === "number" ? task["attempts"] : undefined,
+    restartCount: typeof task["restartCount"] === "number" ? task["restartCount"] : undefined,
     executionTimeMs: typeof task["executionTimeMs"] === "number" ? task["executionTimeMs"] : undefined,
     currentStage: typeof task["currentStage"] === "string" ? task["currentStage"] : undefined,
     failedStage: typeof task["failedStage"] === "string" ? task["failedStage"] : undefined,

package/src/ui/client/types.ts

@@ -89,6 +89,7 @@ export interface NormalizedTask {
   startedAt: string | null;
   endedAt: string | null;
   attempts?: number;
+  restartCount?: number;
   executionTimeMs?: number;
   currentStage?: string;
   failedStage?: string;

package/src/ui/components/DAGGrid.tsx

@@ -276,7 +276,17 @@ export default function DAGGrid({
                 data-role="card-header"
                 className={`flex items-center justify-between gap-3 rounded-t-lg border-b px-4 py-2 ${reducedMotion ? "" : "transition-opacity duration-300 ease-in-out"} ${getHeaderClasses(getItemStatus(items[index], index, activeIndex))}`}
               >
-                <div className="truncate font-medium">{items[index]?.title ?? formatStepName(items[index]?.id ?? "")}</div>
+                {(items[index]?.restartCount ?? 0) > 0 ? (
+                  <span
+                    data-role="restart-badge"
+                    className="inline-flex items-center justify-center min-w-[20px] h-5 px-1.5 rounded-sm border border-red-300 bg-red-50 text-red-700 text-[11px] font-medium tabular-nums"
+                    title={`Restarted ${items[index]!.restartCount} time${items[index]!.restartCount === 1 ? "" : "s"}`}
+                    aria-label={`Restarted ${items[index]!.restartCount} time${items[index]!.restartCount === 1 ? "" : "s"}`}
+                  >
+                    ↻ {items[index]!.restartCount}
+                  </span>
+                ) : null}
+                <div className="min-w-0 flex-1 truncate text-left font-medium">{items[index]?.title ?? formatStepName(items[index]?.id ?? "")}</div>
                 <div className="flex items-center gap-2">
                   {getItemStatus(items[index], index, activeIndex) === "running" ? (
                     <>

package/src/ui/components/JobDetail.tsx

@@ -43,7 +43,8 @@ export default function JobDetail({
         prior.stage === item.stage &&
         prior.title === item.title &&
         prior.subtitle === item.subtitle &&
-        prior.body === item.body
+        prior.body === item.body &&
+        prior.restartCount === item.restartCount
       ) {
         return prior;
       }

package/src/ui/components/__tests__/DAGGrid.test.tsx

@@ -117,3 +117,95 @@ test("DAGGrid opens task details from keyboard interaction and renders connector
   expect(view.getByText("Build · done")).toBeTruthy();
   expect(view.container.querySelectorAll('path[marker-end="url(#dag-grid-arrow)"]').length).toBe(1);
 });
+
+test("DAGGrid does not render the restart badge when restartCount is absent or zero", () => {
+  installMatchMedia();
+
+  const view = render(
+    <DAGGrid
+      items={[makeItem({ id: "build", title: "Build", status: "pending" })]}
+      jobId="job-1"
+      taskById={{ build: makeTaskState({ name: "build", state: "pending" }) }}
+      pipelineTasks={["build"]}
+      filesByTypeForItem={() => emptyFiles}
+      geometryAdapter={geometryAdapter}
+    />,
+  );
+
+  expect(view.container.querySelector('[data-role="restart-badge"]')).toBeNull();
+
+  const viewZero = render(
+    <DAGGrid
+      items={[makeItem({ id: "build", title: "Build", status: "pending", restartCount: 0 })]}
+      jobId="job-1"
+      taskById={{ build: makeTaskState({ name: "build", state: "pending" }) }}
+      pipelineTasks={["build"]}
+      filesByTypeForItem={() => emptyFiles}
+      geometryAdapter={geometryAdapter}
+    />,
+  );
+
+  expect(viewZero.container.querySelector('[data-role="restart-badge"]')).toBeNull();
+});
+
+test("DAGGrid renders the restart badge with singular label when restartCount is 1", () => {
+  installMatchMedia();
+
+  const view = render(
+    <DAGGrid
+      items={[makeItem({ id: "build", title: "Build", status: "pending", restartCount: 1 })]}
+      jobId="job-1"
+      taskById={{ build: makeTaskState({ name: "build", state: "pending" }) }}
+      pipelineTasks={["build"]}
+      filesByTypeForItem={() => emptyFiles}
+      geometryAdapter={geometryAdapter}
+    />,
+  );
+
+  const badge = view.container.querySelector('[data-role="restart-badge"]');
+  expect(badge).not.toBeNull();
+  expect(badge!.getAttribute("aria-label")).toBe("Restarted 1 time");
+  expect(badge!.getAttribute("title")).toBe("Restarted 1 time");
+  expect(badge!.textContent).toBe("↻ 1");
+});
+
+test("DAGGrid renders the restart badge with plural label when restartCount is greater than 1", () => {
+  installMatchMedia();
+
+  const view = render(
+    <DAGGrid
+      items={[makeItem({ id: "build", title: "Build", status: "pending", restartCount: 5 })]}
+      jobId="job-1"
+      taskById={{ build: makeTaskState({ name: "build", state: "pending" }) }}
+      pipelineTasks={["build"]}
+      filesByTypeForItem={() => emptyFiles}
+      geometryAdapter={geometryAdapter}
+    />,
+  );
+
+  const badge = view.container.querySelector('[data-role="restart-badge"]');
+  expect(badge).not.toBeNull();
+  expect(badge!.getAttribute("aria-label")).toBe("Restarted 5 times");
+  expect(badge!.textContent).toBe("↻ 5");
+});
+
+test("DAGGrid renders the restart badge as the first child of the card header", () => {
+  installMatchMedia();
+
+  const view = render(
+    <DAGGrid
+      items={[makeItem({ id: "build", title: "Build", status: "pending", restartCount: 2 })]}
+      jobId="job-1"
+      taskById={{ build: makeTaskState({ name: "build", state: "pending" }) }}
+      pipelineTasks={["build"]}
+      filesByTypeForItem={() => emptyFiles}
+      geometryAdapter={geometryAdapter}
+    />,
+  );
+
+  const header = view.container.querySelector('[data-role="card-header"]');
+  expect(header).not.toBeNull();
+  const firstChild = header!.firstElementChild;
+  expect(firstChild).not.toBeNull();
+  expect(firstChild!.getAttribute("data-role")).toBe("restart-badge");
+});

package/src/ui/components/__tests__/JobDetail.test.tsx

@@ -0,0 +1,62 @@
+import "./test-dom";
+
+import { render } from "@testing-library/react";
+import { afterEach, expect, mock, test } from "bun:test";
+
+import type { DagItem, JobDetail as JobDetailType, PipelineType } from "../types";
+
+const capturedItems: DagItem[][] = [];
+
+mock.module("../DAGGrid", () => ({
+  default: ({ items }: { items: DagItem[] }) => {
+    capturedItems.push(items);
+    return null;
+  },
+}));
+
+const JobDetail = (await import("../JobDetail")).default;
+
+afterEach(() => {
+  document.body.innerHTML = "";
+  capturedItems.length = 0;
+});
+
+function makeJob(restartCount: number): JobDetailType {
+  return {
+    id: "job-1",
+    name: "Job One",
+    status: "running",
+    tasks: {
+      build: {
+        name: "build",
+        state: "running",
+        startedAt: 1_000,
+        files: { artifacts: [], logs: [], tmp: [] },
+        restartCount,
+      },
+    },
+    pipeline: { tasks: ["build"] },
+    current: "build",
+  };
+}
+
+const pipeline: PipelineType = {
+  name: "Test",
+  slug: "test",
+  description: "",
+  tasks: [{ name: "build" }],
+};
+
+test("JobDetail re-emits dag item when only restartCount changes", () => {
+  const view = render(<JobDetail job={makeJob(1)} pipeline={pipeline} />);
+  expect(capturedItems.length).toBeGreaterThan(0);
+  const firstItems = capturedItems[capturedItems.length - 1] as DagItem[];
+  const firstBuild = firstItems.find((item) => item.id === "build");
+  expect(firstBuild?.restartCount).toBe(1);
+
+  view.rerender(<JobDetail job={makeJob(2)} pipeline={pipeline} />);
+  const lastItems = capturedItems[capturedItems.length - 1] as DagItem[];
+  const secondBuild = lastItems.find((item) => item.id === "build");
+  expect(secondBuild?.restartCount).toBe(2);
+  expect(secondBuild).not.toBe(firstBuild);
+});

package/src/ui/components/types.ts

@@ -43,6 +43,7 @@ export interface TaskStateObject {
   artifacts?: string[];
   tokenUsage?: Record<string, unknown>;
   attempts?: number;
+  restartCount?: number;
   executionTimeMs?: number;
 }
 
@@ -105,6 +106,7 @@ export interface DagItem {
   body: string | null;
   startedAt: string | number;
   endedAt: string | number | null;
+  restartCount?: number;
 }
 
 export interface PipelineTask {

package/src/ui/dist/assets/{index-SKy2shWc.js → index-CNlnQmK4.js}

@@ -20453,11 +20453,11 @@ const MODEL_CONFIG_RAW = {
     tokenCostOutPerMillion: 4.5
   },
   // Alibaba (Qwen via DashScope, international/Singapore deployment, base tier)
-  "alibaba:qwen3-max": {
+  "alibaba:qwen3.6-max-preview": {
     provider: "alibaba",
-    model: "qwen3-max",
-    tokenCostInPerMillion: 1.2,
-    tokenCostOutPerMillion: 6
+    model: "qwen3.6-max-preview",
+    tokenCostInPerMillion: 1.3,
+    tokenCostOutPerMillion: 7.8
   },
   "alibaba:qwen3.6-plus": {
     provider: "alibaba",
@@ -20465,6 +20465,18 @@ const MODEL_CONFIG_RAW = {
     tokenCostInPerMillion: 0.276,
     tokenCostOutPerMillion: 1.651
   },
+  "alibaba:qwen3.6-flash": {
+    provider: "alibaba",
+    model: "qwen3.6-flash",
+    tokenCostInPerMillion: 0.025,
+    tokenCostOutPerMillion: 1.5
+  },
+  "alibaba:qwen3-max": {
+    provider: "alibaba",
+    model: "qwen3-max",
+    tokenCostInPerMillion: 1.2,
+    tokenCostOutPerMillion: 6
+  },
   "alibaba:qwen3.5-plus": {
     provider: "alibaba",
     model: "qwen3.5-plus",
@@ -20514,13 +20526,18 @@ const VALID_MODEL_ALIASES = new Set(
 );
 function aliasToFunctionName(alias) {
   if (typeof alias !== "string") {
-    throw new Error(`Invalid model alias: expected string, got ${typeof alias}`);
+    throw new Error(
+      `Invalid model alias: expected string, got ${typeof alias}`
+    );
   }
   if (!alias.includes(":")) {
     throw new Error(`Invalid model alias: "${alias}" does not contain a colon`);
   }
   const model = alias.split(":").slice(1).join(":");
-  return model.replace(/[-.]([a-z0-9])/gi, (_, char) => char.toUpperCase());
+  return model.replace(
+    /[-.]([a-z0-9])/gi,
+    (_, char) => char.toUpperCase()
+  );
 }
 Object.freeze(
   Object.fromEntries(
@@ -20542,7 +20559,13 @@ function buildProviderFunctionsIndex() {
       index2[provider] = [];
     }
     index2[provider].push(
-      Object.freeze({ alias, provider, model, functionName, fullPath })
+      Object.freeze({
+        alias,
+        provider,
+        model,
+        functionName,
+        fullPath
+      })
     );
   }
   for (const provider of Object.keys(index2)) {
@@ -21874,6 +21897,7 @@ function normalizeTask(name2, rawTask) {
     startedAt: toStringOrNull(task["startedAt"]),
     endedAt: toStringOrNull(task["endedAt"]),
     attempts: typeof task["attempts"] === "number" ? task["attempts"] : void 0,
+    restartCount: typeof task["restartCount"] === "number" ? task["restartCount"] : void 0,
     executionTimeMs: typeof task["executionTimeMs"] === "number" ? task["executionTimeMs"] : void 0,
     currentStage: typeof task["currentStage"] === "string" ? task["currentStage"] : void 0,
     failedStage: typeof task["failedStage"] === "string" ? task["failedStage"] : void 0,
@@ -22283,7 +22307,8 @@ function computeDagItems(job, pipeline) {
       subtitle: null,
       body: task?.error?.message ?? null,
       startedAt: task?.startedAt ?? 0,
-      endedAt: task?.endedAt ?? null
+      endedAt: task?.endedAt ?? null,
+      restartCount: task?.restartCount ?? 0
     };
   });
 }
@@ -51031,7 +51056,20 @@ function DAGGrid({
                     "data-role": "card-header",
                     className: `flex items-center justify-between gap-3 rounded-t-lg border-b px-4 py-2 ${reducedMotion ? "" : "transition-opacity duration-300 ease-in-out"} ${getHeaderClasses(getItemStatus(items[index2], index2, activeIndex))}`,
                     children: [
-                      /* @__PURE__ */ jsxRuntimeExports.jsx("div", { className: "truncate font-medium", children: items[index2]?.title ?? formatStepName(items[index2]?.id ?? "") }),
+                      (items[index2]?.restartCount ?? 0) > 0 ? /* @__PURE__ */ jsxRuntimeExports.jsxs(
+                        "span",
+                        {
+                          "data-role": "restart-badge",
+                          className: "inline-flex items-center justify-center min-w-[20px] h-5 px-1.5 rounded-sm border border-red-300 bg-red-50 text-red-700 text-[11px] font-medium tabular-nums",
+                          title: `Restarted ${items[index2].restartCount} time${items[index2].restartCount === 1 ? "" : "s"}`,
+                          "aria-label": `Restarted ${items[index2].restartCount} time${items[index2].restartCount === 1 ? "" : "s"}`,
+                          children: [
+                            "↻ ",
+                            items[index2].restartCount
+                          ]
+                        }
+                      ) : null,
+                      /* @__PURE__ */ jsxRuntimeExports.jsx("div", { className: "min-w-0 flex-1 truncate text-left font-medium", children: items[index2]?.title ?? formatStepName(items[index2]?.id ?? "") }),
                       /* @__PURE__ */ jsxRuntimeExports.jsx("div", { className: "flex items-center gap-2", children: getItemStatus(items[index2], index2, activeIndex) === "running" ? /* @__PURE__ */ jsxRuntimeExports.jsxs(jsxRuntimeExports.Fragment, { children: [
                         /* @__PURE__ */ jsxRuntimeExports.jsxs("div", { className: "relative h-4 w-4", "aria-label": "Active", children: [
                           /* @__PURE__ */ jsxRuntimeExports.jsx("span", { className: "sr-only", children: "Active" }),
@@ -51185,7 +51223,7 @@ function JobDetail({
     const previous2 = prevDagItemsRef.current;
     const reused = enriched.map((item, index2) => {
       const prior = previous2[index2];
-      if (prior && prior.id === item.id && prior.status === item.status && prior.stage === item.stage && prior.title === item.title && prior.subtitle === item.subtitle && prior.body === item.body) {
+      if (prior && prior.id === item.id && prior.status === item.status && prior.stage === item.stage && prior.title === item.title && prior.subtitle === item.subtitle && prior.body === item.body && prior.restartCount === item.restartCount) {
         return prior;
       }
       return item;