@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

package/src/core/task-telemetry.ts

@@ -0,0 +1,107 @@
+import type { EventEmitter } from "node:events";
+import { mkdir } from "node:fs/promises";
+import { dirname } from "node:path";
+
+/**
+ * INVARIANT: correct attribution + console capture assume ONE runTask at a time
+ * per process (sequential stages, one runner per job). Lifetime-scoped listener
+ * attribution and global console capture are valid only under it; if in-process
+ * task concurrency is introduced, switch to an event correlation id
+ * (jobId/taskId/requestId) the handlers filter on.
+ *
+ * Attach the two LLM telemetry listeners, run `body`, then in `finally` remove
+ * both listeners and flush the token-write queue with best-effort swallow
+ * semantics so a flush failure cannot mask the original task result/error.
+ *
+ * `flush` is a thunk: it is evaluated at finally-time, not captured eagerly, so
+ * it awaits the latest token-write queue rather than a stale snapshot.
+ */
+export async function withTaskTelemetry<T>(
+  events: EventEmitter,
+  handlers: {
+    onComplete: (m: Record<string, unknown>) => void;
+    onError: (m: Record<string, unknown>) => void;
+  },
+  flush: () => Promise<void>,
+  body: () => Promise<T>,
+): Promise<T> {
+  const { onComplete, onError } = handlers;
+
+  events.on("llm:request:complete", onComplete);
+  events.on("llm:request:error", onError);
+
+  try {
+    return await body();
+  } finally {
+    events.removeListener("llm:request:complete", onComplete);
+    events.removeListener("llm:request:error", onError);
+    await flush().catch(() => {});
+  }
+}
+
+/**
+ * Replaces console.log/error/warn/info/debug with functions that write to a
+ * string buffer with appropriate prefixes. Returns a restore function that
+ * flushes the buffer to `logPath` via Bun.write() and restores originals.
+ * Creates the log directory before capturing.
+ *
+ * The restore is idempotent: a `restored` flag guards it so a double-call is a
+ * no-op and cannot reinstate stale originals over methods reassigned since.
+ */
+export function captureConsoleOutput(logPath: string): () => Promise<void> {
+  const origLog = console.log;
+  const origError = console.error;
+  const origWarn = console.warn;
+  const origInfo = console.info;
+  const origDebug = console.debug;
+
+  let buffer = "";
+  let restored = false;
+
+  const append = (prefix: string, args: unknown[]) => {
+    buffer += prefix + args.map(String).join(" ") + "\n";
+  };
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  console.log = (...args: any[]) => append("", args);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  console.error = (...args: any[]) => append("[ERROR] ", args);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  console.warn = (...args: any[]) => append("[WARN] ", args);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  console.info = (...args: any[]) => append("[INFO] ", args);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  console.debug = (...args: any[]) => append("[DEBUG] ", args);
+
+  return async () => {
+    if (restored) return;
+    restored = true;
+
+    console.log = origLog;
+    console.error = origError;
+    console.warn = origWarn;
+    console.info = origInfo;
+    console.debug = origDebug;
+
+    await mkdir(dirname(logPath), { recursive: true });
+    await Bun.write(logPath, buffer);
+  };
+}
+
+/**
+ * Captures console output to `logPath` for the duration of `body`, then in
+ * `finally` restores every console method the capture mutated. The restore is
+ * guarded by `captureConsoleOutput`'s `restored` flag, so it runs exactly once
+ * regardless of how `body` exits.
+ */
+export async function withConsoleCapture<T>(
+  logPath: string,
+  body: () => Promise<T>,
+): Promise<T> {
+  const restore = captureConsoleOutput(logPath);
+  try {
+    return await body();
+  } finally {
+    await restore();
+  }
+}

package/src/harness/__tests__/subprocess.test.ts

@@ -1,5 +1,8 @@
 import { describe, it, expect } from "vitest";
-import { runJsonlSubprocess } from "../subprocess.ts";
+import {
+  runJsonlSubprocess,
+  SUBPROCESS_KILL_GRACE_MS,
+} from "../subprocess.ts";
 
 describe("runJsonlSubprocess", () => {
   it("parses two JSONL lines into two events", async () => {
@@ -58,4 +61,112 @@ describe("runJsonlSubprocess", () => {
     expect(result.stderr).toContain("oops");
     expect(result.timedOut).toBe(false);
   });
+
+  // AC-7: concurrent drain — child floods stderr past the OS pipe buffer
+  // (~64KB) while stdout stays open; sequential reads would deadlock.
+  it("drains >64KB stderr with pending stdout without deadlocking", async () => {
+    const result = await runJsonlSubprocess({
+      argv: [
+        "bash",
+        "-c",
+        // 100KB of stderr, then a stdout JSONL line, then exit.
+        'yes x | head -c 102400 >&2; echo "{\\"done\\":true}"',
+      ],
+      env: {},
+      timeoutMs: 10000,
+    });
+
+    expect(result.stderr.length).toBeGreaterThanOrEqual(102400);
+    expect(result.events).toEqual([{ done: true }]);
+    expect(result.exitCode).toBe(0);
+    expect(result.timedOut).toBe(false);
+  }, 15000);
+
+  // AC-8: SIGTERM-ignoring child is escalated to SIGKILL after the grace
+  // window; timedOut is set and partial output captured.
+  it("escalates to SIGKILL on timeout and captures partial output", async () => {
+    const start = Date.now();
+    const result = await runJsonlSubprocess({
+      argv: [
+        "bash",
+        "-c",
+        // exec replaces bash so the trapping process itself holds the pipe;
+        // tree-kill of grandchildren is out of scope (single-child contract).
+        'trap "" TERM; echo "{\\"partial\\":true}"; echo warn >&2; exec sleep 30',
+      ],
+      env: {},
+      timeoutMs: 150,
+    });
+    const elapsed = Date.now() - start;
+
+    expect(result.timedOut).toBe(true);
+    expect(result.exitCode).not.toBe(0);
+    expect(result.events).toEqual([{ partial: true }]);
+    expect(result.stderr).toContain("warn");
+    // SIGTERM was trapped, so resolution proves SIGKILL fired after the grace.
+    expect(elapsed).toBeGreaterThanOrEqual(150 + SUBPROCESS_KILL_GRACE_MS - 50);
+    expect(elapsed).toBeLessThan(150 + SUBPROCESS_KILL_GRACE_MS + 4000);
+  }, 10000);
+
+  // AC-9: abort during run mirrors the timeout kill + partial capture.
+  it("kills and captures partial output when aborted mid-run", async () => {
+    const controller = new AbortController();
+    setTimeout(() => controller.abort(), 150);
+
+    const result = await runJsonlSubprocess({
+      argv: [
+        "bash",
+        "-c",
+        // exec replaces bash so the trapping process itself holds the pipe;
+        // tree-kill of grandchildren is out of scope (single-child contract).
+        'trap "" TERM; echo "{\\"partial\\":true}"; echo warn >&2; exec sleep 30',
+      ],
+      env: {},
+      timeoutMs: 30000,
+      signal: controller.signal,
+    });
+
+    expect(result.timedOut).toBe(true);
+    expect(result.events).toEqual([{ partial: true }]);
+    expect(result.stderr).toContain("warn");
+    expect(result.exitCode).not.toBe(0);
+  }, 10000);
+
+  // AC-9: an already-aborted signal kills immediately.
+  it("kills immediately when the signal is already aborted", async () => {
+    const result = await runJsonlSubprocess({
+      argv: ["sleep", "30"],
+      env: {},
+      timeoutMs: 30000,
+      signal: AbortSignal.abort(),
+    });
+
+    expect(result.timedOut).toBe(true);
+    expect(result.exitCode).not.toBe(0);
+  }, 10000);
+
+  // AC-10: the abort listener is removed on every exit, so aborting the same
+  // signal after completion does not re-kill (no leak).
+  it("removes the abort listener after completion", async () => {
+    const controller = new AbortController();
+    let abortObserved = false;
+    controller.signal.addEventListener("abort", () => {
+      abortObserved = true;
+    });
+
+    const result = await runJsonlSubprocess({
+      argv: ["bun", "-e", 'console.log(JSON.stringify({ok:true}))'],
+      env: {},
+      timeoutMs: 5000,
+      signal: controller.signal,
+    });
+
+    expect(result.exitCode).toBe(0);
+
+    // The helper's listener must be gone; only our own observer remains.
+    controller.abort();
+    expect(abortObserved).toBe(true);
+    // No throw / no late kill side effects: the resolved result is unchanged.
+    expect(result.events).toEqual([{ ok: true }]);
+  }, 10000);
 });

package/src/harness/subprocess.ts

@@ -28,6 +28,9 @@ function parseJsonl(stdout: string): unknown[] {
   return events;
 }
 
+/** Grace period after SIGTERM before escalating to SIGKILL on timeout/abort. */
+export const SUBPROCESS_KILL_GRACE_MS = 250;
+
 export async function runJsonlSubprocess(
   args: RunJsonlSubprocessArgs,
 ): Promise<RunJsonlSubprocessResult> {
@@ -40,33 +43,61 @@ export async function runJsonlSubprocess(
     ...(cwd !== undefined ? { cwd } : {}),
   });
 
+  // Drain both streams concurrently so neither pipe buffer can deadlock the
+  // other. Capture the latest text so the throw path can still report partial
+  // output; the wrapped drains never reject.
+  const captured = { stdout: "", stderr: "" };
+  const stdoutP = new Response(proc.stdout)
+    .text()
+    .then((text) => {
+      captured.stdout = text;
+      return text;
+    });
+  const stderrP = new Response(proc.stderr)
+    .text()
+    .then((text) => {
+      captured.stderr = text;
+      return text;
+    });
+
+  let exited = false;
+  void proc.exited.then(() => {
+    exited = true;
+  });
+
   let timedOut = false;
+  let graceTimer: ReturnType<typeof setTimeout> | undefined;
+
+  const killEscalate = (): void => {
+    proc.kill("SIGTERM");
+    if (graceTimer === undefined) {
+      graceTimer = setTimeout(() => {
+        if (!exited) proc.kill("SIGKILL");
+      }, SUBPROCESS_KILL_GRACE_MS);
+    }
+  };
+
   const timer = setTimeout(() => {
     timedOut = true;
-    proc.kill();
+    killEscalate();
   }, timeoutMs);
 
+  const onAbort = (): void => {
+    timedOut = true;
+    killEscalate();
+  };
+
   if (signal !== undefined) {
     if (signal.aborted) {
-      timedOut = true;
-      proc.kill();
+      onAbort();
     } else {
-      signal.addEventListener(
-        "abort",
-        () => {
-          proc.kill();
-        },
-        { once: true },
-      );
+      signal.addEventListener("abort", onAbort, { once: true });
     }
   }
 
   try {
-    const stdout = await new Response(proc.stdout).text();
+    const [stdout, stderr] = await Promise.all([stdoutP, stderrP]);
     await proc.exited;
-    clearTimeout(timer);
-
-    const stderr = await new Response(proc.stderr).text();
 
     return {
       events: parseJsonl(stdout),
@@ -76,8 +107,16 @@ export async function runJsonlSubprocess(
       timedOut,
     };
   } catch (err) {
-    clearTimeout(timer);
-    proc.kill();
+    if (typeof err === "object" && err !== null) {
+      Object.assign(err, {
+        stdout: captured.stdout,
+        stderr: captured.stderr,
+      });
+    }
     throw err;
+  } finally {
+    clearTimeout(timer);
+    if (graceTimer !== undefined) clearTimeout(graceTimer);
+    if (signal !== undefined) signal.removeEventListener("abort", onAbort);
   }
 }