@ryanfw/prompt-orchestration-pipeline 1.3.3 → 1.3.4

package/src/config/pipeline-registry.ts

@@ -0,0 +1,258 @@
+import { join, isAbsolute, dirname } from "node:path";
+import {
+  existsSync,
+  readFileSync,
+  openSync,
+  closeSync,
+  fsyncSync,
+  unlinkSync,
+} from "node:fs";
+import { mkdir, writeFile, rename } from "node:fs/promises";
+
+export const REGISTRY_VERSION = 1;
+
+export interface PipelineRegistryEntry {
+  name: string;
+  description: string;
+  /** Relative to root or absolute. Default: pipeline-config/<slug> */
+  configDir?: string;
+  /** Relative to root or absolute. Default: <configDir>/tasks */
+  tasksDir?: string;
+}
+
+export interface PipelineRegistry {
+  version: number;
+  pipelines: Record<string, PipelineRegistryEntry>;
+}
+
+export interface ResolvedPipeline {
+  pipelineJsonPath: string;
+  tasksDir: string;
+}
+
+export class RegistryError extends Error {}
+
+export function registryFilePath(root: string): string {
+  return join(root, "pipeline-config", "registry.json");
+}
+
+const REGISTRY_REPAIR_HINT =
+  "regenerate it with 'pipeline-orchestrator init' or fix the file";
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return (
+    typeof value === "object" && value !== null && !Array.isArray(value)
+  );
+}
+
+function fail(filePath: string, reason: string): never {
+  throw new RegistryError(`${filePath}: ${reason}; ${REGISTRY_REPAIR_HINT}`);
+}
+
+export function readRegistrySync(root: string): PipelineRegistry {
+  const filePath = registryFilePath(root);
+
+  if (!existsSync(filePath)) {
+    return { version: REGISTRY_VERSION, pipelines: {} };
+  }
+
+  const text = readFileSync(filePath, "utf8");
+  if (text.trim() === "") {
+    fail(filePath, "registry file is empty");
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch (err) {
+    const cause = err instanceof Error ? err.message : String(err);
+    fail(filePath, `not valid JSON (${cause})`);
+  }
+
+  if (!isPlainObject(parsed)) {
+    fail(filePath, "registry must be a JSON object");
+  }
+
+  const pipelines = (parsed as Record<string, unknown>)["pipelines"];
+  if (!isPlainObject(pipelines)) {
+    if ("slugs" in parsed) {
+      return { version: REGISTRY_VERSION, pipelines: {} };
+    }
+    fail(filePath, "missing 'pipelines' object");
+  }
+
+  for (const [slug, raw] of Object.entries(pipelines)) {
+    const entry = raw as Record<string, unknown>;
+    if (entry === null || typeof entry !== "object" || Array.isArray(entry)) {
+      fail(filePath, `entry '${slug}': must be an object`);
+    }
+    if (typeof entry["name"] !== "string") {
+      fail(filePath, `entry '${slug}': 'name' must be a string`);
+    }
+    if (typeof entry["description"] !== "string") {
+      fail(filePath, `entry '${slug}': 'description' must be a string`);
+    }
+    if (
+      entry["configDir"] !== undefined &&
+      typeof entry["configDir"] !== "string"
+    ) {
+      fail(
+        filePath,
+        `entry '${slug}': 'configDir' must be a string when present`,
+      );
+    }
+    if (
+      entry["tasksDir"] !== undefined &&
+      typeof entry["tasksDir"] !== "string"
+    ) {
+      fail(
+        filePath,
+        `entry '${slug}': 'tasksDir' must be a string when present`,
+      );
+    }
+  }
+
+  const version = (parsed as Record<string, unknown>)["version"];
+  if (
+    version !== undefined &&
+    (typeof version !== "number" || version !== REGISTRY_VERSION)
+  ) {
+    fail(
+      filePath,
+      `unsupported registry version ${JSON.stringify(version)} (expected ${REGISTRY_VERSION})`,
+    );
+  }
+
+  return {
+    version: REGISTRY_VERSION,
+    pipelines: pipelines as Record<string, PipelineRegistryEntry>,
+  };
+}
+
+export function resolvePipelineEntry(
+  root: string,
+  slug: string,
+  entry: PipelineRegistryEntry,
+): { configDir: string; tasksDir: string } {
+  const configDir = !entry.configDir
+    ? join(root, "pipeline-config", slug)
+    : isAbsolute(entry.configDir)
+      ? entry.configDir
+      : join(root, entry.configDir);
+
+  const tasksDir = !entry.tasksDir
+    ? join(configDir, "tasks")
+    : isAbsolute(entry.tasksDir)
+      ? entry.tasksDir
+      : join(root, entry.tasksDir);
+
+  return { configDir, tasksDir };
+}
+
+export function resolvePipelineSync(
+  root: string,
+  slug: string,
+): ResolvedPipeline {
+  const reg = readRegistrySync(root);
+  const entry = reg.pipelines[slug];
+  if (entry === undefined) {
+    throw new Error(`Pipeline '${slug}' not found in registry`);
+  }
+  const resolved = resolvePipelineEntry(root, slug, entry);
+  return {
+    pipelineJsonPath: join(resolved.configDir, "pipeline.json"),
+    tasksDir: resolved.tasksDir,
+  };
+}
+
+export function resolveAllPipelinesSync(
+  root: string,
+): Record<string, { configDir: string; tasksDir: string }> {
+  const reg = readRegistrySync(root);
+  const result: Record<string, { configDir: string; tasksDir: string }> = {};
+  for (const [slug, entry] of Object.entries(reg.pipelines)) {
+    result[slug] = resolvePipelineEntry(root, slug, entry);
+  }
+  return result;
+}
+
+/**
+ * Resolve the task-module path for a tasks dir, preferring compiled index.js
+ * and falling back to index.ts (Bun runs .ts natively). Returns index.js path
+ * when neither exists so the loader's error names the canonical file.
+ */
+export function resolveTaskRegistryPath(tasksDir: string): string {
+  const jsPath = join(tasksDir, "index.js");
+  const tsPath = join(tasksDir, "index.ts");
+  if (existsSync(jsPath)) return jsPath;
+  if (existsSync(tsPath)) return tsPath;
+  return jsPath;
+}
+
+let tempCounter = 0;
+
+/**
+ * Atomic temp-file + rename write. Serializes the full JSON before opening any
+ * file, writes to a same-directory temp, fsyncs the temp, renames over the
+ * target, fsyncs the directory. On any error before the rename completes, the
+ * temp file is removed (if present) and the original error is rethrown.
+ */
+export async function writeRegistry(
+  root: string,
+  reg: PipelineRegistry,
+): Promise<void> {
+  const serialized = JSON.stringify(reg);
+  const target = registryFilePath(root);
+  const dir = dirname(target);
+  await mkdir(dir, { recursive: true });
+  const temp = join(
+    dir,
+    `.registry.json.tmp.${process.pid}.${tempCounter++}`,
+  );
+  try {
+    await writeFile(temp, serialized);
+    const fd = openSync(temp, "r");
+    try {
+      fsyncSync(fd);
+    } finally {
+      closeSync(fd);
+    }
+    await rename(temp, target);
+  } catch (err) {
+    if (existsSync(temp)) {
+      try {
+        unlinkSync(temp);
+      } catch {
+      }
+    }
+    throw err;
+  }
+
+  // Content is durable once rename succeeds. Fsync the directory so the new
+  // entry survives a crash, but treat failure as best-effort: the write has
+  // already committed, so a dir-fsync error must not propagate as a failure.
+  try {
+    const dirFd = openSync(dir, "r");
+    try {
+      fsyncSync(dirFd);
+    } finally {
+      closeSync(dirFd);
+    }
+  } catch {
+  }
+}
+
+/**
+ * Read-modify-write a single entry. Reads the current registry (or a fresh v1
+ * document if absent), sets `pipelines[slug] = entry`, and writes back
+ * atomically through {@link writeRegistry}.
+ */
+export async function registerPipeline(
+  root: string,
+  slug: string,
+  entry: PipelineRegistryEntry,
+): Promise<void> {
+  const reg = readRegistrySync(root);
+  reg.pipelines[slug] = entry;
+  await writeRegistry(root, reg);
+}

package/src/config/sse-events.ts

@@ -0,0 +1,122 @@
+import { z } from "zod";
+import { JobStatus } from "./statuses.ts";
+
+const GateInfoSchema = z
+  .object({
+    afterTask: z.string(),
+    message: z.string(),
+    artifacts: z.array(z.string()).optional(),
+    requestedAt: z.string(),
+    kind: z.enum(["approval", "control_invalid"]).optional(),
+  })
+  .passthrough();
+
+const CostsSummarySchema = z
+  .object({
+    totalTokens: z.number(),
+    totalInputTokens: z.number(),
+    totalOutputTokens: z.number(),
+    totalCost: z.number(),
+    totalInputCost: z.number(),
+    totalOutputCost: z.number(),
+    estimatedCost: z.number().default(0),
+    hasEstimated: z.boolean().default(false),
+    unavailableCost: z.number().default(0),
+  })
+  .passthrough();
+
+const JobErrorSchema = z
+  .object({
+    code: z.string(),
+    message: z.string(),
+    path: z.string().optional(),
+  })
+  .passthrough();
+
+export const JobWireSchema = z
+  .object({
+    jobId: z.string(),
+    title: z.string(),
+    status: z.enum([
+      JobStatus.PENDING,
+      JobStatus.RUNNING,
+      JobStatus.FAILED,
+      JobStatus.COMPLETE,
+      JobStatus.WAITING,
+    ]),
+    progress: z.number(),
+    createdAt: z.string().nullable(),
+    updatedAt: z.string().nullable(),
+    location: z.string().nullable(),
+    tasks: z.record(z.string(), z.unknown()),
+    costsSummary: CostsSummarySchema,
+    files: z.record(z.string(), z.unknown()).optional(),
+    current: z.unknown().optional(),
+    currentStage: z.unknown().optional(),
+    gate: GateInfoSchema.nullable().optional(),
+    pipelineSlug: z.string().optional(),
+    pipeline: z.string().optional(),
+    pipelineLabel: z.string().optional(),
+    pipelineConfig: z.record(z.string(), z.unknown()).optional(),
+    readable: z.boolean().optional(),
+    error: JobErrorSchema.optional(),
+  })
+  .passthrough();
+
+export type JobWire = z.infer<typeof JobWireSchema>;
+
+export const SseEvent = z.discriminatedUnion("type", [
+  z.object({
+    type: z.literal("state:change"),
+    data: z.object({ jobId: z.string(), path: z.string() }).passthrough(),
+  }),
+  z.object({
+    type: z.literal("state:summary"),
+    data: z.object({ changeCount: z.number() }).passthrough(),
+  }),
+  z.object({ type: z.literal("job:created"), data: JobWireSchema }),
+  z.object({ type: z.literal("job:updated"), data: JobWireSchema }),
+  z.object({
+    type: z.literal("job:removed"),
+    data: z.object({ jobId: z.string() }).passthrough(),
+  }),
+  z.object({
+    type: z.literal("heartbeat"),
+    data: z.object({ ok: z.boolean(), timestamp: z.string() }).passthrough(),
+  }),
+  z.object({
+    type: z.literal("seed:uploaded"),
+    data: z.object({}).passthrough(),
+  }),
+]);
+
+export type SseEvent = z.infer<typeof SseEvent>;
+export type SseEventType = SseEvent["type"];
+
+export const DOCUMENTED_SSE_EVENTS = [
+  "state:change",
+  "state:summary",
+  "job:created",
+  "job:updated",
+  "job:removed",
+  "heartbeat",
+  "seed:uploaded",
+] as const satisfies ReadonlyArray<SseEventType>;
+
+export function parseSseEvent(type: string, data: unknown): SseEvent | null {
+  const result = SseEvent.safeParse({ type, data });
+  return result.success ? result.data : null;
+}
+
+export function emitSseEvent(
+  registry: { broadcast: (type: string, data: unknown) => void },
+  event: SseEvent,
+): void {
+  const result = SseEvent.safeParse(event);
+  if (!result.success) {
+    throw new Error(
+      `emitSseEvent: invalid event of type "${(event as { type?: string }).type ?? "unknown"}" failed schema validation: ${result.error.message}`,
+    );
+  }
+  registry.broadcast(event.type, event.data);
+}

package/src/core/__tests__/agent-step.test.ts

@@ -763,3 +763,118 @@ describe("runAgentStep", () => {
     });
   });
 });
+
+function createContentCapturingIO(): TaskFileIO & {
+  logWrites: Array<{ name: string; content: string }>;
+} {
+  const logWrites: Array<{ name: string; content: string }> = [];
+  return {
+    logWrites,
+    async writeArtifact(_name: string, _content: string, _options?: WriteOptions) {},
+    async writeLog(name: string, content: string, _options?: WriteOptions) {
+      logWrites.push({ name, content });
+    },
+    async writeTmp(_name: string, _content: string, _options?: WriteOptions) {},
+    async readArtifact(_name: string) {
+      return "";
+    },
+    async readLog(_name: string) {
+      return "";
+    },
+    async readTmp(_name: string) {
+      return "";
+    },
+    getTaskDir() {
+      return join(tmpdir(), "pop-fake-task-dir");
+    },
+    writeLogSync() {},
+    getCurrentStage() {
+      return "test";
+    },
+    getDB() {
+      throw new Error("not implemented");
+    },
+    async runBatch() {
+      throw new Error("not implemented");
+    },
+  };
+}
+
+describe("agent event log redaction", () => {
+  it("with an MCP handle, onEvent redacts the token from Authorization fields and embedded strings while preserving non-secret fields", async () => {
+    const token = "s3cret-tok-abc123xyz";
+    const io = createContentCapturingIO();
+    const startMcpIoServer = mock(async () =>
+      createFakeMcpHandle([], { url: "http://127.0.0.1:9990/mcp", token }),
+    );
+    const run = mock((opts: { onEvent?: (event: HarnessEvent) => void }) => {
+      opts.onEvent?.({
+        harness: "claude",
+        seq: 0,
+        at: Date.now(),
+        type: "raw",
+        raw: {
+          model: "gpt",
+          sessionId: "abc",
+          Authorization: `Bearer ${token}`,
+          command: `run --token ${token} --verbose`,
+          nested: {
+            headers: { authorization: `Bearer ${token}` },
+            keep: "preserved",
+          },
+        },
+      });
+      return makeFakeHarnessRun(makeSuccessResult());
+    });
+
+    await executeAgent(
+      { io, entry: { name: "a", harness: "claude", prompt: "p" } },
+      { run, startMcpIoServer },
+    );
+
+    expect(io.logWrites).toHaveLength(1);
+    const content = io.logWrites[0]!.content;
+    expect(content).not.toContain(token);
+    expect(content).not.toContain(`Bearer ${token}`);
+    expect(content).toContain("[REDACTED]");
+    expect(content).toContain('"model":"gpt"');
+    expect(content).toContain('"sessionId":"abc"');
+    expect(content).toContain('"keep":"preserved"');
+  });
+
+  it("with no MCP handle, onEvent still runs key-based redaction and preserves non-secret content", async () => {
+    const io = createContentCapturingIO();
+    const startMcpIoServer = mock(async () => {
+      throw new Error("startMcpIoServer must not be called when entry.io === false");
+    });
+    const run = mock((opts: { onEvent?: (event: HarnessEvent) => void }) => {
+      opts.onEvent?.({
+        harness: "claude",
+        seq: 0,
+        at: Date.now(),
+        type: "raw",
+        raw: {
+          model: "gpt",
+          Authorization: "Bearer xyz",
+          sessionId: "abc",
+          message: "got nothing sensitive from server",
+        },
+      });
+      return makeFakeHarnessRun(makeSuccessResult());
+    });
+
+    await executeAgent(
+      { io, entry: { name: "a", harness: "claude", prompt: "p", io: false } },
+      { run, startMcpIoServer },
+    );
+
+    expect(startMcpIoServer).not.toHaveBeenCalled();
+    expect(io.logWrites).toHaveLength(1);
+    const content = io.logWrites[0]!.content;
+    expect(content).not.toContain("[Circular]");
+    expect(content).toContain('"Authorization":"[REDACTED]"');
+    expect(content).toContain('"model":"gpt"');
+    expect(content).toContain('"sessionId":"abc"');
+    expect(content).toContain('"message":"got nothing sensitive from server"');
+  });
+});