@golba98/codexa 1.0.1

package/src/core/providers/codexSubprocess.ts

@@ -0,0 +1,372 @@
+import { spawn } from "child_process";
+import { formatCodexLaunchError, spawnCodexProcess } from "../executables/codexExecutable.js";
+import { prepareCodexExecLaunch } from "../codexLaunch.js";
+import * as perf from "../perf/profiler.js";
+import { buildCodexPrompt } from "../codexPrompt.js";
+import { createTerminalTitleSequenceStripper } from "../terminal/terminalTitle.js";
+import { createCodexJsonStreamParser } from "./codexJsonStream.js";
+import {
+  createCodexTranscriptStreamParser,
+  createStdoutSanitizer,
+  isStderrNoise,
+  sanitizeCodexTranscript,
+  stripAnsi,
+  stripNonPrintableControls,
+} from "./codexTranscript.js";
+import type { BackendProvider } from "./types.js";
+
+// Detects CLI error messages that indicate --experimental-json is not supported.
+// When this fires the provider retries in legacy transcript mode.
+function looksLikeUnsupportedStructuredOutput(raw: string): boolean {
+  return /experimental-json|unknown option|unrecognized option|unexpected argument|unexpected option/i.test(raw);
+}
+
+function isProcessTerminationNoise(line: string): boolean {
+  return /^SUCCESS: The process with PID \d+ .* has been terminated\.$/.test(line.trim());
+}
+
+export const codexSubprocessProvider: BackendProvider = {
+  id: "codex-subprocess",
+  label: "Codexa",
+  description: "Direct connection to the Codex neural network.",
+  authState: "delegated",
+  authLabel: "Authenticated via Codex",
+  statusMessage: "Authentication is managed via Codexa.",
+  supportsModels: () => true,
+  run: (prompt, options, handlers) => {
+    let done = false;
+    let cancelled = false;
+    let proc: ReturnType<typeof spawn> | null = null;
+    let procExited = false;
+    let currentRawOutput = "";
+    let finalAnswerObserved = false;
+
+    const finishError = (message: string) => {
+      if (done) return;
+      done = true;
+      handlers.onError(message, currentRawOutput);
+    };
+
+    const finishSuccess = (response: string) => {
+      if (done) return;
+      done = true;
+      handlers.onResponse(response);
+    };
+
+    const emitFinalAnswerObserved = (response: string) => {
+      if (finalAnswerObserved) return;
+      finalAnswerObserved = true;
+      handlers.onFinalAnswerObserved?.(response);
+    };
+
+    const startAttempt = (structuredOutput: boolean, probeCapabilities = false) => {
+      if (cancelled || done) return;
+      let firstChunkSeen = false;
+      let firstStderrSeen = false;
+      handlers.benchmarkHooks?.onProviderPrepStart?.();
+      void prepareCodexExecLaunch(
+        {
+          runtime: options.runtime,
+          cwd: options.workspaceRoot,
+          structuredOutput,
+          probeCapabilities,
+          codexCommandPath: options.runtime.codexCommandPath,
+        },
+        import.meta.url,
+      )
+        .then((launchPlan) => {
+          if (cancelled || done) return;
+          if (!launchPlan.ok) {
+            finishError(launchPlan.error);
+            return;
+          }
+          if (!launchPlan.executable) {
+            finishError("Codex launch preparation did not return an executable.");
+            return;
+          }
+          handlers.benchmarkHooks?.onProviderPrepComplete?.();
+
+          let rawStdout = "";
+          let rawStderr = "";
+          let stdoutLineBuffer = "";
+          let mode: "undecided" | "json" | "legacy" = structuredOutput ? "undecided" : "legacy";
+          let legacyProgressSequence = 0;
+
+          // Consecutive thinking lines from the transcript parser are coalesced into
+          // a single progress update so the UI shows one accumulating block rather
+          // than a rapid series of individual entries. The sequence resets whenever
+          // an assistant delta or tool activity event breaks the run of thinking lines.
+          let activeTranscriptThinkingId: string | null = null;
+          let activeTranscriptThinkingText = "";
+
+          const resetTranscriptCoalescing = () => {
+            activeTranscriptThinkingId = null;
+            activeTranscriptThinkingText = "";
+          };
+
+          const emitLegacyProgress = (source: "stdout" | "stderr" | "transcript", text: string) => {
+            if (isProcessTerminationNoise(text)) return;
+            const displayText = text.length > 80 ? `${text.slice(0, 77)}...` : text;
+            handlers.onProgress?.({
+              id: `${source}-${++legacyProgressSequence}`,
+              source,
+              text: displayText,
+            });
+          };
+
+          const transcriptParser = createCodexTranscriptStreamParser({
+            onThinkingLine: (line) => {
+              if (activeTranscriptThinkingId === null) {
+                activeTranscriptThinkingId = `transcript-thinking-${++legacyProgressSequence}`;
+                activeTranscriptThinkingText = line;
+              } else {
+                activeTranscriptThinkingText = `${activeTranscriptThinkingText}\n${line}`;
+              }
+              handlers.onProgress?.({
+                id: activeTranscriptThinkingId,
+                source: "transcript",
+                text: activeTranscriptThinkingText,
+              });
+            },
+            onAssistantDelta: (chunk) => {
+              resetTranscriptCoalescing();
+              handlers.onAssistantDelta?.(chunk);
+            },
+            onToolActivity: (activity) => {
+              resetTranscriptCoalescing();
+              handlers.onToolActivity?.(activity);
+            },
+          });
+          const transcriptStdoutSanitizer = createStdoutSanitizer();
+          const transcriptStderrSanitizer = createStdoutSanitizer();
+          const stdoutTitleStripper = createTerminalTitleSequenceStripper({
+            source: "src/core/providers/codexSubprocess.ts:codex.stdout",
+            stream: "stdout",
+            origin: "codex-cli",
+          });
+          const stderrTitleStripper = createTerminalTitleSequenceStripper({
+            source: "src/core/providers/codexSubprocess.ts:codex.stderr",
+            stream: "stderr",
+            origin: "codex-cli",
+          });
+          const jsonParser = createCodexJsonStreamParser({
+            onProgress: (update) => handlers.onProgress?.(update),
+            onAssistantDelta: (chunk) => handlers.onAssistantDelta?.(chunk),
+            onFinalAnswerObserved: emitFinalAnswerObserved,
+            onToolActivity: (activity) => handlers.onToolActivity?.(activity),
+          });
+
+          const feedTranscript = (text: string, stream: "stdout" | "stderr") => {
+            const sanitizer = stream === "stdout" ? transcriptStdoutSanitizer : transcriptStderrSanitizer;
+            const clean = sanitizer.process(text);
+            if (clean) {
+              transcriptParser.feed(clean);
+            }
+          };
+
+          const switchToTranscriptFallback = () => {
+            if (mode === "legacy") return;
+            mode = "legacy";
+            if (rawStdout) {
+              feedTranscript(rawStdout, "stdout");
+            }
+            if (rawStderr) {
+              feedTranscript(rawStderr, "stderr");
+            }
+            stdoutLineBuffer = "";
+          };
+
+          const processStructuredLines = (flush: boolean) => {
+            while (true) {
+              const newlineIndex = stdoutLineBuffer.indexOf("\n");
+              if (newlineIndex === -1) {
+                if (!flush) return;
+                if (!stdoutLineBuffer) return;
+              }
+
+              const line = newlineIndex === -1
+                ? stdoutLineBuffer
+                : stdoutLineBuffer.slice(0, newlineIndex);
+              stdoutLineBuffer = newlineIndex === -1 ? "" : stdoutLineBuffer.slice(newlineIndex + 1);
+              const normalizedLine = line.replace(/\r$/, "");
+              if (!normalizedLine.trim()) {
+                continue;
+              }
+
+              const parsed = jsonParser.feedLine(normalizedLine);
+              if (!parsed) {
+                if (mode === "json") {
+                  emitLegacyProgress("stdout", normalizedLine);
+                  continue;
+                }
+                switchToTranscriptFallback();
+                return;
+              }
+              mode = "json";
+            }
+          };
+
+          const ingestStdoutText = (text: string) => {
+            if (!text) return;
+            currentRawOutput += text;
+            rawStdout += text;
+
+            if (mode === "legacy") {
+              feedTranscript(text, "stdout");
+              return;
+            }
+
+            stdoutLineBuffer += text;
+            processStructuredLines(false);
+          };
+
+          const ingestStderrText = (text: string) => {
+            if (!text) return;
+            currentRawOutput += text;
+            rawStderr += text;
+
+            if (mode === "legacy") {
+              feedTranscript(text, "stderr");
+              return;
+            }
+
+            const lines = stripNonPrintableControls(stripAnsi(text))
+              .replace(/\r\n/g, "\n")
+              .replace(/\r/g, "\n")
+              .split("\n");
+            for (const line of lines) {
+              const trimmed = line.trim();
+              if (!trimmed || isStderrNoise(trimmed) || isProcessTerminationNoise(trimmed)) continue;
+              emitLegacyProgress("stderr", trimmed);
+            }
+          };
+
+          handlers.onProcessLifecycle?.("before-spawn");
+          proc = spawnCodexProcess(launchPlan.executable, launchPlan.args, { stdio: ["pipe", "pipe", "pipe"] });
+          procExited = false;
+          handlers.onProcessLifecycle?.("spawned");
+          handlers.benchmarkHooks?.onCodexProcessSpawned?.({
+            executable: launchPlan.executable,
+            argv: launchPlan.args,
+          });
+          perf.mark("spawn_done");
+
+          proc.stdout?.on("data", (chunk: Buffer) => {
+            if (cancelled || done) return;
+            if (!firstChunkSeen) {
+              firstChunkSeen = true;
+              perf.mark("first_chunk");
+              handlers.benchmarkHooks?.onFirstStdout?.();
+            }
+            perf.mark("last_chunk");
+            ingestStdoutText(stdoutTitleStripper.process(chunk));
+          });
+
+          proc.stderr?.on("data", (chunk: Buffer) => {
+            if (cancelled || done) return;
+            if (!firstStderrSeen) {
+              firstStderrSeen = true;
+              handlers.benchmarkHooks?.onFirstStderr?.();
+            }
+            ingestStderrText(stderrTitleStripper.process(chunk));
+          });
+
+          proc.on("close", (code) => {
+            procExited = true;
+            handlers.onProcessLifecycle?.("exit");
+            if (cancelled || done) return;
+            ingestStdoutText(stdoutTitleStripper.flush());
+            ingestStderrText(stderrTitleStripper.flush());
+            if (!firstChunkSeen) {
+              handlers.benchmarkHooks?.onFirstStdout?.(false);
+            }
+            if (!firstStderrSeen) {
+              handlers.benchmarkHooks?.onFirstStderr?.(false);
+            }
+
+            if (mode !== "legacy") {
+              processStructuredLines(true);
+            }
+
+            if (mode === "legacy") {
+              const remainingStdout = transcriptStdoutSanitizer.flush();
+              if (remainingStdout) transcriptParser.feed(remainingStdout);
+              const remainingStderr = transcriptStderrSanitizer.flush();
+              if (remainingStderr) transcriptParser.feed(remainingStderr);
+              transcriptParser.flush();
+            }
+
+            handlers.benchmarkHooks?.onCodexProcessExit?.(code);
+
+            if (
+              structuredOutput
+              && code !== 0
+              && !jsonParser.hasStructuredEvents()
+              && looksLikeUnsupportedStructuredOutput(`${rawStdout}\n${rawStderr}`)
+            ) {
+              currentRawOutput = "";
+              startAttempt(false, true);
+              return;
+            }
+
+            const structuredFailure = jsonParser.getFailureMessage();
+            if (structuredFailure) {
+              finishError(structuredFailure);
+              return;
+            }
+
+            if (code === 0) {
+              const finalResponse = mode === "json"
+                ? jsonParser.getFinalResponse().trim() || sanitizeCodexTranscript(currentRawOutput)
+                : sanitizeCodexTranscript(currentRawOutput);
+              emitFinalAnswerObserved(finalResponse);
+              finishSuccess(finalResponse);
+              return;
+            }
+
+            finishError(`Process exited with code ${code}`);
+          });
+
+          proc.on("error", (err) => {
+            handlers.onProcessLifecycle?.("error");
+            if (cancelled || done) return;
+            const errno = err as NodeJS.ErrnoException;
+            finishError(formatCodexLaunchError(errno));
+          });
+
+          const promptPolicy = options.promptPolicy ?? "wrapped";
+          const providerPrompt = promptPolicy === "raw"
+            ? prompt
+            : buildCodexPrompt(prompt, options.runtime, undefined, {
+                projectInstructions: options.projectInstructions,
+              });
+          handlers.benchmarkHooks?.onProviderPromptPrepared?.({
+            policy: promptPolicy,
+            characterCount: providerPrompt.length,
+          });
+          proc.stdin?.write(providerPrompt);
+          proc.stdin?.end();
+        })
+        .catch((error) => {
+          const errno = error as NodeJS.ErrnoException;
+          finishError(formatCodexLaunchError(errno));
+        });
+    };
+
+    currentRawOutput = "";
+    startAttempt(true);
+
+    return () => {
+      cancelled = true;
+      done = true;
+      handlers.benchmarkHooks?.onCleanupStart?.();
+      handlers.onProcessLifecycle?.("cleanup");
+      if (!proc || procExited || proc.killed) {
+        handlers.benchmarkHooks?.onCleanupComplete?.({ skipped: true });
+        return;
+      }
+      proc.kill();
+      handlers.benchmarkHooks?.onCleanupComplete?.({ skipped: false });
+    };
+  },
+};

package/src/core/providers/codexTranscript.test.ts

@@ -0,0 +1,284 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import type { RunToolActivity } from "../../session/types.js";
+import {
+  createCodexTranscriptStreamParser,
+  sanitizeCodexTranscript,
+  stripNonPrintableControls,
+} from "./codexTranscript.js";
+
+test("extracts the assistant reply from noisy codex transcript output", () => {
+  const raw = [
+    "Reading additional input from stdin...",
+    "OpenAI Codex v0.118.0 (research preview)",
+    "--------",
+    "workdir: C:\\Development\\1-JavaScript\\13-Custom CLI",
+    "model: gpt-5.4",
+    "provider: openai",
+    "approval: never",
+    "sandbox: read-only",
+    "reasoning effort: xhigh",
+    "reasoning summaries: none",
+    "session id: 019d4984-90c0-7402-80c9-3d55a8e0373f",
+    "--------",
+    "user",
+    "Hello",
+    "codex",
+    "Hello. What do you need help with?",
+    "tokens used",
+    "1,390",
+    "Hello. What do you need help with?",
+  ].join("\n");
+
+  assert.equal(sanitizeCodexTranscript(raw), "Hello. What do you need help with?");
+});
+
+test("falls back to filtered plain output when no labeled assistant block exists", () => {
+  const raw = [
+    "\u001b[32mOpenAI Codex v0.118.0\u001b[0m",
+    "--------",
+    "workdir: C:\\repo",
+    "",
+    "First useful line",
+    "Second useful line",
+  ].join("\n");
+
+  assert.equal(sanitizeCodexTranscript(raw), "First useful line\nSecond useful line");
+});
+
+test("filters echoed clarification-policy prompt lines from finalized transcripts", () => {
+  const raw = [
+    "default to best-effort continuation instead of stopping for clarification.",
+    "if a detail is missing but non-critical, make the most reasonable assumption and state it briefly.",
+    "if multiple paths are possible, choose one sensible path and continue.",
+    "only ask a blocking follow-up question if proceeding would likely use the wrong file, wrong command, destructive behavior, or produce fundamentally incorrect output.",
+    "if you are truly blocked on one critical missing fact, end the response with exactly one line in this format: [QUESTION]: <your question>",
+    "",
+    "Implemented the fix and assumed src/app.tsx was the target entrypoint.",
+  ].join("\n");
+
+  assert.equal(
+    sanitizeCodexTranscript(raw),
+    "Implemented the fix and assumed src/app.tsx was the target entrypoint.",
+  );
+});
+
+test("returns a readable fallback when only noise is present", () => {
+  const raw = [
+    "Reading additional input from stdin...",
+    "OpenAI Codex v0.118.0",
+    "--------",
+    "model: gpt-5.4",
+    "provider: openai",
+  ].join("\n");
+
+  assert.match(
+    sanitizeCodexTranscript(raw),
+    /no assistant response text was detected/i,
+  );
+});
+
+test("streams thinking lines separately from assistant deltas", () => {
+  const thinking: string[] = [];
+  const assistant: string[] = [];
+  const parser = createCodexTranscriptStreamParser({
+    onThinkingLine: (line) => thinking.push(line),
+    onAssistantDelta: (chunk) => assistant.push(chunk),
+  });
+
+  parser.feed([
+    "OpenAI Codex v0.118.0",
+    "Checking src/app.tsx",
+    "Task:",
+    "Refactor the CLI",
+    "",
+    "assistant",
+    "First line",
+    "Second line",
+  ].join("\n"));
+  parser.flush();
+
+  assert.deepEqual(thinking, ["Checking src/app.tsx"]);
+  assert.deepEqual(assistant, ["First line", "\nSecond line"]);
+});
+
+test("emits fenced code blocks atomically when streaming", () => {
+  const assistant: string[] = [];
+  const parser = createCodexTranscriptStreamParser({
+    onAssistantDelta: (chunk) => assistant.push(chunk),
+  });
+
+  parser.feed("assistant\n```ts\n");
+  parser.feed("const value = 1;\n");
+  parser.feed("```\n");
+  parser.flush();
+
+  // Code fences are buffered and emitted as a single atomic chunk
+  assert.deepEqual(assistant, ["```ts\nconst value = 1;\n```"]);
+});
+
+test("emits tool activity separately from assistant prose while streaming", () => {
+  const assistant: string[] = [];
+  const toolActivity: RunToolActivity[] = [];
+  const parser = createCodexTranscriptStreamParser({
+    onAssistantDelta: (chunk) => assistant.push(chunk),
+    onToolActivity: (activity) => toolActivity.push(activity),
+  });
+
+  parser.feed([
+    "assistant",
+    "$ rg --files",
+    "src/app.tsx",
+    "src/ui/BottomComposer.tsx",
+    "",
+    "I found the relevant files.",
+  ].join("\n"));
+  parser.flush();
+
+  assert.deepEqual(assistant, ["I found the relevant files."]);
+  assert.equal(toolActivity.length, 2);
+  assert.equal(toolActivity[0]?.command, "rg --files");
+  assert.equal(toolActivity[0]?.status, "running");
+  assert.equal(toolActivity[1]?.command, "rg --files");
+  assert.equal(toolActivity[1]?.status, "completed");
+  assert.equal(toolActivity[1]?.summary, "Found 2 files");
+});
+
+test("removes tool execution stdout from the finalized assistant transcript", () => {
+  const raw = [
+    "assistant",
+    "$ rg --files",
+    "src/app.tsx",
+    "src/ui/BottomComposer.tsx",
+    "",
+    "I found the relevant files and updated the composer.",
+  ].join("\n");
+
+  assert.equal(
+    sanitizeCodexTranscript(raw),
+    "I found the relevant files and updated the composer.",
+  );
+});
+
+// ─── Progressive streaming tests ──────────────────────────────────────────────
+
+test("emits partial lines after timeout when no newline arrives", async () => {
+  const assistant: string[] = [];
+  const parser = createCodexTranscriptStreamParser({
+    onAssistantDelta: (chunk) => assistant.push(chunk),
+  });
+
+  parser.feed("assistant\n");
+  parser.feed("Hello wor");
+  // No newline yet — partial flush timer should fire after ~100ms
+  assert.equal(assistant.length, 0);
+
+  await new Promise((resolve) => setTimeout(resolve, 150));
+  assert.equal(assistant.length, 1);
+  assert.equal(assistant[0], "Hello wor");
+
+  parser.flush();
+  // flush should not re-emit the already-emitted partial
+  assert.equal(assistant.length, 1);
+});
+
+test("does not duplicate content when partial is followed by complete line", async () => {
+  const assistant: string[] = [];
+  const parser = createCodexTranscriptStreamParser({
+    onAssistantDelta: (chunk) => assistant.push(chunk),
+  });
+
+  parser.feed("assistant\n");
+  parser.feed("Start of line");
+  // Wait for partial flush
+  await new Promise((resolve) => setTimeout(resolve, 150));
+  assert.equal(assistant.length, 1);
+  assert.equal(assistant[0], "Start of line");
+
+  // Now the rest of the line arrives with newline
+  parser.feed(" and the rest\n");
+  // The partial was already emitted; the remaining text appends naturally
+  // The feed should process the complete line that includes both parts
+  assert.ok(assistant.length >= 1);
+
+  parser.flush();
+});
+
+test("code fence with prose before and after emits prose immediately and fence atomically", () => {
+  const assistant: string[] = [];
+  const parser = createCodexTranscriptStreamParser({
+    onAssistantDelta: (chunk) => assistant.push(chunk),
+  });
+
+  parser.feed("assistant\nHere is the code:\n```ts\nconst x = 1;\n```\nDone.\n");
+  parser.flush();
+
+  assert.equal(assistant.length, 3);
+  assert.equal(assistant[0], "Here is the code:");
+  assert.equal(assistant[1], "\n```ts\nconst x = 1;\n```");
+  assert.equal(assistant[2], "\nDone.");
+});
+
+test("code fence safety timeout emits buffered content after 3 seconds", async () => {
+  const assistant: string[] = [];
+  const parser = createCodexTranscriptStreamParser({
+    onAssistantDelta: (chunk) => assistant.push(chunk),
+  });
+
+  parser.feed("assistant\n```ts\nconst x = 1;\n");
+  // No closing fence — wait for safety timeout (3s)
+  // We'll test that flush() force-emits instead of waiting the full 3s
+  assert.equal(assistant.length, 0);
+
+  parser.flush();
+  assert.equal(assistant.length, 1);
+  assert.equal(assistant[0], "```ts\nconst x = 1;");
+});
+
+test("auto-promotes long prose lines from preamble to assistant section", () => {
+  const thinking: string[] = [];
+  const assistant: string[] = [];
+  const parser = createCodexTranscriptStreamParser({
+    onThinkingLine: (line) => thinking.push(line),
+    onAssistantDelta: (chunk) => assistant.push(chunk),
+  });
+
+  parser.feed([
+    "I have analyzed the codebase and here is a comprehensive summary of all the changes that need to be made to fix this issue.",
+  ].join("\n"));
+  parser.flush();
+
+  // Long prose with many words should auto-promote to assistant
+  assert.equal(thinking.length, 0);
+  assert.equal(assistant.length, 1);
+  assert.match(assistant[0]!, /analyzed the codebase/);
+});
+
+test("does not auto-promote short status lines from preamble", () => {
+  const thinking: string[] = [];
+  const assistant: string[] = [];
+  const parser = createCodexTranscriptStreamParser({
+    onThinkingLine: (line) => thinking.push(line),
+    onAssistantDelta: (chunk) => assistant.push(chunk),
+  });
+
+  parser.feed("Checking src/app.tsx\n");
+  parser.flush();
+
+  assert.equal(thinking.length, 1);
+  assert.equal(assistant.length, 0);
+});
+
+test("strips control characters from streamed and finalized assistant text", () => {
+  const assistant: string[] = [];
+  const parser = createCodexTranscriptStreamParser({
+    onAssistantDelta: (chunk) => assistant.push(chunk),
+  });
+
+  parser.feed("assistant\nHello\u0007 world\u0001\n");
+  parser.flush();
+
+  assert.deepEqual(assistant, ["Hello world"]);
+  assert.equal(stripNonPrintableControls("A\u0000B\u001fC"), "ABC");
+  assert.equal(sanitizeCodexTranscript("assistant\nDone\u0007\n"), "Done");
+});