@elench/testkit 0.1.82 → 0.1.83

package/lib/cli/agents/providers/codex.mjs

@@ -0,0 +1,83 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { execa } from "execa";
+import {
+  buildErrorEvent,
+  buildStatusEvent,
+  buildToolEvent,
+  createHostedSessionRunner,
+  extractTextFragments,
+  readTextFileIfPresent,
+} from "./shared.mjs";
+
+export function startCodexHostedSession({ cwd, prompt, onEvent, purpose = "investigate" } = {}) {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-codex-"));
+  const outputFile = path.join(tempDir, "final-message.txt");
+  const args = ["exec", "--json", "-o", outputFile];
+
+  if (purpose === "investigate") {
+    args.push("-s", "read-only", "-a", "never");
+  }
+
+  args.push(prompt);
+
+  const child = execa("codex", args, {
+    cwd,
+    stdout: "pipe",
+    stderr: "pipe",
+    reject: false,
+  });
+
+  const session = createHostedSessionRunner({
+    provider: "codex",
+    child,
+    onEvent,
+    parsePayload: parseCodexPayload,
+    readFinalText(result) {
+      return readTextFileIfPresent(outputFile) || result.stdout || null;
+    },
+  });
+
+  const completion = session.completion.finally(() => {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  return {
+    completion,
+    cancel: session.cancel,
+  };
+}
+
+function parseCodexPayload(payload) {
+  const events = [];
+  if (!payload || typeof payload !== "object") return events;
+  const type = payload.type || payload.event || payload.kind || null;
+  const errorMessage = payload.error?.message || payload.error || null;
+  if (errorMessage) {
+    const event = buildErrorEvent(errorMessage);
+    if (event) events.push(event);
+    return events;
+  }
+
+  if (type && /(tool|command|patch|exec)/i.test(type)) {
+    const event = buildToolEvent(
+      payload.name || payload.tool_name || payload.command || type,
+      payload.detail || payload.summary || payload.status || null
+    );
+    if (event) events.push(event);
+    return events;
+  }
+
+  const fragments = [...new Set(extractTextFragments(payload, []))];
+  if (fragments.length > 0) {
+    for (const fragment of fragments) {
+      events.push({ type: "delta", text: fragment });
+    }
+    return events;
+  }
+
+  const statusEvent = buildStatusEvent(type ? `Codex event: ${type}` : JSON.stringify(payload));
+  if (statusEvent) events.push(statusEvent);
+  return events;
+}

package/lib/cli/agents/providers/codex.test.mjs

@@ -0,0 +1,93 @@
+import { PassThrough } from "stream";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const execaMock = vi.fn();
+
+vi.mock("execa", () => ({
+  execa: execaMock,
+}));
+
+function createChildHandle() {
+  const stdout = new PassThrough();
+  const stderr = new PassThrough();
+  let resolveChild;
+  const promise = new Promise((resolve) => {
+    resolveChild = resolve;
+  });
+  promise.stdout = stdout;
+  promise.stderr = stderr;
+  promise.kill = vi.fn(() => {
+    stdout.end();
+    stderr.end();
+  });
+  return {
+    child: promise,
+    finish(result = {}) {
+      stdout.end();
+      stderr.end();
+      resolveChild({
+        exitCode: 0,
+        stdout: "",
+        stderr: "",
+        ...result,
+      });
+    },
+  };
+}
+
+afterEach(() => {
+  execaMock.mockReset();
+});
+
+describe("Codex hosted session", () => {
+  it("normalizes streamed tool, delta, and final output", async () => {
+    const { startCodexHostedSession } = await import("./codex.mjs");
+    const handle = createChildHandle();
+    execaMock.mockReturnValue(handle.child);
+    const events = [];
+
+    const session = startCodexHostedSession({
+      cwd: "/tmp/project",
+      prompt: "Investigate this failure",
+      onEvent(event) {
+        events.push(event);
+      },
+    });
+
+    handle.child.stdout.write(`${JSON.stringify({ type: "command", command: "rg", status: "done" })}\n`);
+    handle.child.stdout.write(`${JSON.stringify({ type: "assistant_delta", text: "Check the migration step." })}\n`);
+    handle.finish({ exitCode: 0, stdout: "Check the migration step." });
+
+    const result = await session.completion;
+
+    expect(execaMock).toHaveBeenCalledWith(
+      "codex",
+      expect.arrayContaining(["exec", "--json", "-s", "read-only", "-a", "never"]),
+      expect.objectContaining({ cwd: "/tmp/project", reject: false })
+    );
+    expect(events[0]).toMatchObject({ provider: "codex", type: "start" });
+    expect(events).toContainEqual(expect.objectContaining({ provider: "codex", type: "tool", name: "rg" }));
+    expect(events).toContainEqual(expect.objectContaining({ provider: "codex", type: "delta", text: "Check the migration step." }));
+    expect(events.at(-2)).toMatchObject({ provider: "codex", type: "final", text: "Check the migration step." });
+    expect(events.at(-1)).toMatchObject({ provider: "codex", type: "exit", code: 0 });
+    expect(result.finalText).toBe("Check the migration step.");
+  });
+
+  it("cancels the child process", async () => {
+    const { startCodexHostedSession } = await import("./codex.mjs");
+    const handle = createChildHandle();
+    execaMock.mockReturnValue(handle.child);
+
+    const session = startCodexHostedSession({
+      cwd: "/tmp/project",
+      prompt: "Investigate this failure",
+    });
+
+    session.cancel();
+    handle.finish({ exitCode: 130 });
+    const result = await session.completion;
+
+    expect(handle.child.kill).toHaveBeenCalledWith("SIGTERM");
+    expect(result.cancelled).toBe(true);
+  });
+});

package/lib/cli/agents/providers/shared.mjs

@@ -0,0 +1,134 @@
+import fs from "fs";
+import readline from "readline";
+
+export function createHostedSessionRunner({ provider, child, onEvent, parsePayload, readFinalText } = {}) {
+  let cancelled = false;
+  let settled = false;
+  let assistantText = "";
+
+  const emit = (event) => {
+    if (event?.type === "delta" || event?.type === "final") {
+      assistantText += event.text || "";
+    }
+    if (typeof onEvent === "function" && event) onEvent({ provider, ...event });
+  };
+
+  emit({ type: "start" });
+
+  const stdoutReader = readline.createInterface({ input: child.stdout });
+  stdoutReader.on("line", (line) => {
+    const parsed = tryParseJson(line);
+    if (parsed == null) {
+      emit({ type: "status", message: line });
+      return;
+    }
+    const events = parsePayload ? parsePayload(parsed) : [];
+    if (!Array.isArray(events) || events.length === 0) return;
+    for (const event of events) emit(event);
+  });
+
+  const stderrReader = readline.createInterface({ input: child.stderr });
+  stderrReader.on("line", (line) => {
+    emit({ type: "status", message: line });
+  });
+
+  const completion = (async () => {
+    const result = await child;
+    const finalText = (readFinalText ? readFinalText(result) : null) || assistantText.trim() || null;
+    if (finalText) emit({ type: "final", text: finalText });
+    emit({ type: "exit", code: result.exitCode ?? 0 });
+    settled = true;
+    return {
+      provider,
+      exitCode: result.exitCode ?? 0,
+      stdout: result.stdout || "",
+      stderr: result.stderr || "",
+      finalText: finalText || result.stdout || "",
+      cancelled,
+    };
+  })();
+
+  return {
+    completion,
+    cancel() {
+      cancelled = true;
+      if (settled) return;
+      try {
+        child.kill("SIGTERM");
+      } catch {
+        // Ignore cancellation races.
+      }
+    },
+  };
+}
+
+export function tryParseJson(line) {
+  const normalized = String(line || "").trim();
+  if (!normalized) return null;
+  try {
+    return JSON.parse(normalized);
+  } catch {
+    return null;
+  }
+}
+
+export function buildToolEvent(name, detail = null) {
+  if (!name) return null;
+  return {
+    type: "tool",
+    name: String(name),
+    ...(detail ? { detail: String(detail) } : {}),
+  };
+}
+
+export function buildStatusEvent(message) {
+  if (!message) return null;
+  return { type: "status", message: String(message) };
+}
+
+export function buildErrorEvent(message) {
+  if (!message) return null;
+  return { type: "error", message: String(message) };
+}
+
+export function extractTextFragments(payload, fragments = [], depth = 0) {
+  if (payload == null || depth > 5 || fragments.length >= 12) return fragments;
+  if (typeof payload === "string") {
+    const normalized = payload.trim();
+    if (normalized) fragments.push(normalized);
+    return fragments;
+  }
+  if (Array.isArray(payload)) {
+    for (const entry of payload) {
+      extractTextFragments(entry, fragments, depth + 1);
+      if (fragments.length >= 12) break;
+    }
+    return fragments;
+  }
+  if (typeof payload !== "object") return fragments;
+
+  const preferredKeys = ["delta", "text", "content", "message", "output_text", "summary"];
+  let matchedPreferredKey = false;
+  for (const key of preferredKeys) {
+    if (!(key in payload)) continue;
+    matchedPreferredKey = true;
+    extractTextFragments(payload[key], fragments, depth + 1);
+    if (fragments.length >= 12) return fragments;
+  }
+  if (matchedPreferredKey && fragments.length > 0) return fragments;
+
+  for (const value of Object.values(payload)) {
+    extractTextFragments(value, fragments, depth + 1);
+    if (fragments.length >= 12) break;
+  }
+  return fragments;
+}
+
+export function readTextFileIfPresent(filePath) {
+  if (!filePath) return null;
+  try {
+    return fs.readFileSync(filePath, "utf8").trim() || null;
+  } catch {
+    return null;
+  }
+}

package/lib/cli/command-helpers.mjs

@@ -11,6 +11,8 @@ import {
 } from "./args.mjs";
 import * as runner from "../runner/index.mjs";
 import { createRunReporter } from "./presentation/run-reporter.mjs";
+import { createTreeReporter } from "./presentation/tree-reporter.mjs";
+import { createRunEventsReporter } from "./presentation/events-reporter.mjs";
 
 export const sharedFlags = {
   dir: Flags.string({
@@ -64,7 +66,7 @@ export const runFlags = {
   }),
   "output-mode": Flags.string({
     description: "Reporter mode",
-    options: ["compact", "debug"],
+    options: ["compact", "debug", "events"],
   }),
   debug: Flags.boolean({
     description: "Alias for --output-mode debug",
@@ -94,31 +96,57 @@ export async function executeRunCommand(command, flags, positionalType = null) {
     : flags.debug
       ? "debug"
       : flags["output-mode"] || "compact";
-  const reporter = createRunReporter({ outputMode });
-  const result = await runner.runAll(
-    configs,
-    typeValues,
-    suiteSelectors,
-    {
-      ...flags,
+
+  let reporter;
+  let finalize = Promise.resolve();
+  let close = () => {};
+
+  if (outputMode === "compact" && process.stdout.isTTY) {
+    const tree = createTreeReporter({
+      stdout: process.stdout,
+      stderr: process.stderr,
+      productDir,
+    });
+    reporter = tree.reporter;
+    finalize = tree.finalize;
+    close = tree.close;
+  } else if (outputMode === "events") {
+    reporter = createRunEventsReporter({ stdout: process.stdout, stderr: process.stderr });
+  } else {
+    reporter = createRunReporter({ outputMode });
+  }
+
+  try {
+    const result = await runner.runAll(
+      configs,
       typeValues,
-      fileNames,
-      workers,
-      fileTimeoutSeconds,
-      shard,
-      scenarioSeed: flags.seed || null,
-      serviceFilter: flags.service || null,
-      reporter,
-      writeStatus: flags["write-status"],
-      allowPartialStatus: flags["allow-partial-status"],
-      ignoreSkipRules: flags["ignore-skip-rules"],
-    },
-    allConfigs
-  );
-  return {
-    outputMode,
-    ...result,
-  };
+      suiteSelectors,
+      {
+        ...flags,
+        typeValues,
+        fileNames,
+        workers,
+        fileTimeoutSeconds,
+        shard,
+        scenarioSeed: flags.seed || null,
+        serviceFilter: flags.service || null,
+        reporter,
+        writeStatus: flags["write-status"],
+        allowPartialStatus: flags["allow-partial-status"],
+        ignoreSkipRules: flags["ignore-skip-rules"],
+      },
+      allConfigs
+    );
+    await finalize;
+    return {
+      outputMode,
+      ...result,
+    };
+  } catch (error) {
+    close();
+    await finalize.catch(() => {});
+    throw error;
+  }
 }
 
 export async function runStatusLike(commandName, flags) {

package/lib/cli/command-helpers.test.mjs

@@ -0,0 +1,122 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const loadManagedConfigsMock = vi.fn();
+const runAllMock = vi.fn();
+const createRunReporterMock = vi.fn();
+const createTreeReporterMock = vi.fn();
+const createRunEventsReporterMock = vi.fn();
+
+vi.mock("../app/configs.mjs", () => ({
+  loadManagedConfigs: loadManagedConfigsMock,
+}));
+
+vi.mock("../runner/index.mjs", () => ({
+  runAll: runAllMock,
+}));
+
+vi.mock("./presentation/run-reporter.mjs", () => ({
+  createRunReporter: createRunReporterMock,
+}));
+
+vi.mock("./presentation/tree-reporter.mjs", () => ({
+  createTreeReporter: createTreeReporterMock,
+}));
+
+vi.mock("./presentation/events-reporter.mjs", () => ({
+  createRunEventsReporter: createRunEventsReporterMock,
+}));
+
+const originalIsTTY = process.stdout.isTTY;
+const originalStdoutGetWindowSize = process.stdout.getWindowSize;
+const originalStderrGetWindowSize = process.stderr.getWindowSize;
+
+beforeEach(() => {
+  loadManagedConfigsMock.mockResolvedValue({
+    allConfigs: [{ name: "api", productDir: "/tmp/product" }],
+    configs: [{ name: "api", productDir: "/tmp/product" }],
+  });
+  runAllMock.mockResolvedValue({ ok: true });
+  createRunReporterMock.mockReturnValue({ outputMode: "compact" });
+  createRunEventsReporterMock.mockReturnValue({ outputMode: "events" });
+  createTreeReporterMock.mockReturnValue({
+    reporter: { outputMode: "compact" },
+    finalize: Promise.resolve(),
+    close: vi.fn(),
+  });
+  Object.defineProperty(process.stdout, "isTTY", {
+    configurable: true,
+    value: true,
+  });
+  process.stdout.getWindowSize = vi.fn(() => [100, 40]);
+  process.stderr.getWindowSize = vi.fn(() => [100, 40]);
+});
+
+afterEach(() => {
+  loadManagedConfigsMock.mockReset();
+  runAllMock.mockReset();
+  createRunReporterMock.mockReset();
+  createTreeReporterMock.mockReset();
+  createRunEventsReporterMock.mockReset();
+  Object.defineProperty(process.stdout, "isTTY", {
+    configurable: true,
+    value: originalIsTTY,
+  });
+  process.stdout.getWindowSize = originalStdoutGetWindowSize;
+  process.stderr.getWindowSize = originalStderrGetWindowSize;
+});
+
+describe("executeRunCommand", () => {
+  it("uses the tree reporter for compact TTY runs and awaits finalization", async () => {
+    let finalizeResolved = false;
+    createTreeReporterMock.mockReturnValueOnce({
+      reporter: { outputMode: "compact" },
+      finalize: Promise.resolve().then(() => {
+        finalizeResolved = true;
+      }),
+      close: vi.fn(),
+    });
+
+    const { executeRunCommand } = await import("./command-helpers.mjs");
+    const result = await executeRunCommand({ jsonEnabled: () => false }, {}, null);
+
+    expect(createTreeReporterMock).toHaveBeenCalledWith(
+      expect.objectContaining({ productDir: "/tmp/product" })
+    );
+    expect(runAllMock).toHaveBeenCalledWith(
+      expect.any(Array),
+      expect.any(Array),
+      expect.any(Array),
+      expect.objectContaining({ reporter: { outputMode: "compact" } }),
+      expect.any(Array)
+    );
+    expect(finalizeResolved).toBe(true);
+    expect(result.outputMode).toBe("compact");
+  });
+
+  it("uses the events reporter when requested", async () => {
+    const { executeRunCommand } = await import("./command-helpers.mjs");
+    await executeRunCommand(
+      { jsonEnabled: () => false },
+      { "output-mode": "events" },
+      null
+    );
+
+    expect(createRunEventsReporterMock).toHaveBeenCalled();
+    expect(createRunReporterMock).not.toHaveBeenCalled();
+  });
+
+  it("closes the tree reporter when runner.runAll throws", async () => {
+    const close = vi.fn();
+    createTreeReporterMock.mockReturnValueOnce({
+      reporter: { outputMode: "compact" },
+      finalize: Promise.resolve(),
+      close,
+    });
+    runAllMock.mockRejectedValueOnce(new Error("boom"));
+
+    const { executeRunCommand } = await import("./command-helpers.mjs");
+
+    await expect(executeRunCommand({ jsonEnabled: () => false }, {}, null)).rejects.toThrow("boom");
+    expect(close).toHaveBeenCalled();
+  });
+});

package/lib/cli/commands/investigate.mjs

@@ -0,0 +1,87 @@
+import { Args, Command, Flags } from "@oclif/core";
+import { sharedFlags } from "../command-helpers.mjs";
+import { loadCurrentRunArtifact, resolveFileSubject } from "../viewer.mjs";
+import { runInteractiveInvestigation, startHostedInvestigation } from "../agents/investigate.mjs";
+
+export default class InvestigateCommand extends Command {
+  static summary = "Investigate a failed file from the latest run with Codex or Claude";
+
+  static enableJsonFlag = true;
+
+  static args = {
+    file: Args.string({
+      description: "Optional file path; defaults to the first failed file",
+      required: false,
+    }),
+  };
+
+  static flags = {
+    ...sharedFlags,
+    provider: Flags.string({
+      description: "Agent provider to use",
+      options: ["auto", "claude", "codex"],
+      default: "auto",
+    }),
+    message: Flags.string({
+      description: "Additional user instruction for the investigation prompt",
+    }),
+    handoff: Flags.boolean({
+      description: "Launch the provider's native interactive TUI instead of hosted output",
+      default: false,
+    }),
+  };
+
+  async run() {
+    const { args, flags } = await this.parse(InvestigateCommand);
+    const productDir = flags.dir || process.cwd();
+    const runArtifact = loadCurrentRunArtifact(productDir);
+    const subject = resolveFileSubject(runArtifact, args.file || null, flags.service || null);
+
+    if (flags.handoff) {
+      const result = await runInteractiveInvestigation({
+        productDir,
+        serviceName: subject.service.name,
+        filePath: subject.file.path,
+        provider: flags.provider,
+        userMessage: flags.message || null,
+      });
+      if (!this.jsonEnabled()) {
+        this.log(`${result.provider} exited with code ${result.exitCode}`);
+      }
+      return result;
+    }
+
+    let finalText = "";
+    const session = startHostedInvestigation({
+      productDir,
+      serviceName: subject.service.name,
+      filePath: subject.file.path,
+      provider: flags.provider,
+      userMessage: flags.message || null,
+      onEvent: this.jsonEnabled()
+        ? null
+        : (event) => {
+            if (event.type === "status" || event.type === "tool") {
+              this.log(event.type === "tool" ? `[tool] ${event.name}${event.detail ? `: ${event.detail}` : ""}` : `[status] ${event.message}`);
+            } else if (event.type === "error") {
+              this.error(event.message);
+            }
+          },
+    });
+    const result = await session.completion;
+    finalText = result.finalText || "";
+
+    if (!this.jsonEnabled() && finalText.trim()) {
+      this.log("");
+      this.log(finalText.trim());
+    }
+
+    return {
+      provider: result.provider,
+      exitCode: result.exitCode,
+      file: subject.file.path,
+      service: subject.service.name,
+      finalText,
+    };
+  }
+}

package/lib/cli/commands/investigate.test.mjs

@@ -0,0 +1,83 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const loadCurrentRunArtifactMock = vi.fn();
+const resolveFileSubjectMock = vi.fn();
+const startHostedInvestigationMock = vi.fn();
+const runInteractiveInvestigationMock = vi.fn();
+
+vi.mock("../viewer.mjs", () => ({
+  loadCurrentRunArtifact: loadCurrentRunArtifactMock,
+  resolveFileSubject: resolveFileSubjectMock,
+}));
+
+vi.mock("../agents/investigate.mjs", () => ({
+  startHostedInvestigation: startHostedInvestigationMock,
+  runInteractiveInvestigation: runInteractiveInvestigationMock,
+}));
+
+afterEach(() => {
+  loadCurrentRunArtifactMock.mockReset();
+  resolveFileSubjectMock.mockReset();
+  startHostedInvestigationMock.mockReset();
+  runInteractiveInvestigationMock.mockReset();
+});
+
+function seedSubject() {
+  loadCurrentRunArtifactMock.mockReturnValue({});
+  resolveFileSubjectMock.mockReturnValue({
+    service: { name: "api" },
+    suite: { name: "users", type: "integration" },
+    file: { path: "tests/api/users.int.testkit.ts" },
+  });
+}
+
+describe("InvestigateCommand", () => {
+  it("runs hosted investigation and returns the final text", async () => {
+    seedSubject();
+    startHostedInvestigationMock.mockReturnValue({
+      completion: Promise.resolve({
+        provider: "codex",
+        exitCode: 0,
+        finalText: "Likely root cause.",
+      }),
+    });
+
+    const { default: InvestigateCommand } = await import("./investigate.mjs");
+    const result = await InvestigateCommand.run(["--json"]);
+
+    expect(startHostedInvestigationMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        productDir: process.cwd(),
+        serviceName: "api",
+        filePath: "tests/api/users.int.testkit.ts",
+      })
+    );
+    expect(result).toMatchObject({
+      provider: "codex",
+      finalText: "Likely root cause.",
+      file: "tests/api/users.int.testkit.ts",
+      service: "api",
+    });
+  });
+
+  it("runs handoff investigation when requested", async () => {
+    seedSubject();
+    runInteractiveInvestigationMock.mockResolvedValue({
+      provider: "claude",
+      exitCode: 0,
+    });
+
+    const { default: InvestigateCommand } = await import("./investigate.mjs");
+    const result = await InvestigateCommand.run(["--json", "--handoff", "--provider", "claude"]);
+
+    expect(runInteractiveInvestigationMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        productDir: process.cwd(),
+        serviceName: "api",
+        filePath: "tests/api/users.int.testkit.ts",
+        provider: "claude",
+      })
+    );
+    expect(result).toMatchObject({ provider: "claude", exitCode: 0 });
+  });
+});