@elench/testkit 0.1.82 → 0.1.83

package/README.md

@@ -365,9 +365,11 @@ HTTP suites:
 
 ```ts
 import { defineHttpSuite } from "@elench/testkit";
+import { expect } from "@elench/testkit/runtime";
 
 const suite = defineHttpSuite(({ rawReq }) => {
-  rawReq("GET", "/health");
+  const response = rawReq.get("/health");
+  expect.status(response, 200, "health returns 200");
 });
 
 export default suite;
@@ -410,9 +412,11 @@ export default defineConfig({
   },
 });
 
-const suite = defineHttpSuite({ profile: "defaultAuth" }, ({ actor, req }) => {
+const suite = defineHttpSuite({ profile: "defaultAuth" }, ({ actor, actors, req }) => {
   req.get("/api/auth/session");
   actor?.req.get("/api/auth/session");
+  req.as("outsider").get("/api/auth/session");
+  actors.get("reviewer").rawReq.get("/api/auth/session");
 });
 ```
 
@@ -439,7 +443,7 @@ const suite = defineScenarioSuite(({ rawReq, scenario }) => {
     includeHealthCheck: scenario.maybe("includeHealthCheck", 1),
   });
 
-  const selected = scenario.resource("selected-endpoint", () => rawReq("GET", plan.endpoint));
+  const selected = scenario.resource("selected-endpoint", () => rawReq.get(plan.endpoint));
 
   scenario.step("fetch selected endpoint", () => {
     selected.get();
@@ -449,21 +453,47 @@ const suite = defineScenarioSuite(({ rawReq, scenario }) => {
 export default suite;
 ```
 
-Low-level runtime primitives remain available:
+First-class runtime namespaces:
 
 ```ts
-import { check, group, http, waitFor } from "@elench/testkit/runtime";
+import { checks, expect, parse, waitFor } from "@elench/testkit/runtime";
+
+const suite = defineHttpSuite(({ actor, rawReq, req }) => {
+  const me = req.get("/api/v1/me");
+  expect.status(me, 200, "authenticated session loads");
+  expect.json(me, (body) => typeof body.data?.id === "string", "response includes user id");
+
+  checks.authGate(rawReq, "sessions", {
+    get: ["/api/v1/sessions"],
+  });
+
+  actor?.rawHeaders();
+
+  const upload = req.multipart.post("/api/v1/uploads", {
+    fields: { name: "fixture" },
+    files: [{ field: "file", data: "hello", filename: "fixture.txt", contentType: "text/plain" }],
+  });
+
+  expect.status(upload, 201, "upload is accepted");
+  parse.safeJson(upload);
+});
+```
+
+Low-level runtime primitives still exist when you genuinely need them:
+
+```ts
+import { check, group, http } from "@elench/testkit/runtime";
 ```
 
 `waitFor()` consumes the file budget configured by `execution.fileTimeoutSeconds`.
 Consumers should not set local timeout values in test files.
 
 ```ts
-import { waitFor } from "@elench/testkit/runtime";
+import { parse, waitFor } from "@elench/testkit/runtime";
 
 const response = waitFor(
   () => req.get("/api/v1/jobs/123"),
-  (res) => JSON.parse(res.body).data?.status === "completed",
+  (res) => parse.json(res).data?.status === "completed",
   { description: "job 123 to complete" }
 );
 ```

package/lib/cli/agents/index.mjs

@@ -0,0 +1,64 @@
+import fs from "fs";
+import path from "path";
+import { spawn } from "child_process";
+import { startClaudeHostedSession } from "./providers/claude.mjs";
+import { startCodexHostedSession } from "./providers/codex.mjs";
+
+const PROVIDERS = ["codex", "claude"];
+
+export function resolvePreferredProvider(preferred = null, env = process.env) {
+  if (preferred && preferred !== "auto") {
+    if (!isProviderInstalled(preferred, env)) {
+      throw new Error(`${preferred} is not installed or not available on PATH`);
+    }
+    return preferred;
+  }
+
+  for (const provider of PROVIDERS) {
+    if (isProviderInstalled(provider, env)) return provider;
+  }
+  throw new Error("Neither codex nor claude was found on PATH");
+}
+
+export function isProviderInstalled(provider, env = process.env) {
+  const pathValue = env.PATH || "";
+  const candidates = pathValue.split(path.delimiter).filter(Boolean);
+  const fileNames =
+    process.platform === "win32"
+      ? [`${provider}.exe`, `${provider}.cmd`, `${provider}.bat`, provider]
+      : [provider];
+  for (const directory of candidates) {
+    for (const fileName of fileNames) {
+      const candidate = path.join(directory, fileName);
+      try {
+        fs.accessSync(candidate, fs.constants.X_OK);
+        return true;
+      } catch {
+        // Continue searching PATH entries.
+      }
+    }
+  }
+  return false;
+}
+
+export function startAgentSession({ provider = "auto", cwd, prompt, onEvent, purpose = "investigate" } = {}) {
+  const resolvedProvider = resolvePreferredProvider(provider);
+  if (resolvedProvider === "claude") {
+    return startClaudeHostedSession({ cwd, prompt, onEvent, purpose });
+  }
+  return startCodexHostedSession({ cwd, prompt, onEvent, purpose });
+}
+
+export function startInteractiveAgentHandoff({ provider = "auto", cwd, prompt } = {}) {
+  const resolvedProvider = resolvePreferredProvider(provider);
+  const command = resolvedProvider;
+  const args = prompt ? [prompt] : [];
+  const child = spawn(command, args, {
+    cwd,
+    stdio: "inherit",
+  });
+  return new Promise((resolve, reject) => {
+    child.on("error", reject);
+    child.on("close", (code) => resolve({ provider: resolvedProvider, exitCode: code ?? 0 }));
+  });
+}

package/lib/cli/agents/investigate.mjs

@@ -0,0 +1,75 @@
+import { loadInvestigationContext } from "./investigation-context.mjs";
+import { buildInvestigationPrompt } from "./prompt-builder.mjs";
+import { startAgentSession, startInteractiveAgentHandoff } from "./index.mjs";
+
+export function createInvestigationRequest({
+  productDir,
+  serviceName,
+  filePath,
+  provider = "auto",
+  userMessage,
+} = {}) {
+  const context = loadInvestigationContext({
+    productDir,
+    serviceName,
+    filePath,
+  });
+  const prompt = buildInvestigationPrompt({ context, userMessage });
+  return {
+    provider,
+    context,
+    prompt,
+  };
+}
+
+export function startHostedInvestigation({
+  productDir,
+  serviceName,
+  filePath,
+  provider = "auto",
+  userMessage,
+  onEvent,
+} = {}) {
+  const request = createInvestigationRequest({
+    productDir,
+    serviceName,
+    filePath,
+    provider,
+    userMessage,
+  });
+  const session = startAgentSession({
+    provider: request.provider,
+    cwd: productDir,
+    prompt: request.prompt,
+    onEvent,
+  });
+  return {
+    ...session,
+    request,
+  };
+}
+
+export async function runInteractiveInvestigation({
+  productDir,
+  serviceName,
+  filePath,
+  provider = "auto",
+  userMessage,
+} = {}) {
+  const request = createInvestigationRequest({
+    productDir,
+    serviceName,
+    filePath,
+    provider,
+    userMessage,
+  });
+  const result = await startInteractiveAgentHandoff({
+    provider: request.provider,
+    cwd: productDir,
+    prompt: request.prompt,
+  });
+  return {
+    ...result,
+    request,
+  };
+}

package/lib/cli/agents/investigation-context.mjs

@@ -0,0 +1,102 @@
+import path from "path";
+import {
+  collectArtifactEntries,
+  formatArtifactPreview,
+  formatFileDetail,
+  getServiceLogRefs,
+  getSetupOperationsForService,
+  loadCurrentRunArtifact,
+  resolveFileSubject,
+} from "../viewer.mjs";
+import { readLogTail } from "../../runner/logs.mjs";
+
+export function loadInvestigationContext({
+  productDir,
+  serviceName,
+  filePath,
+  logTail = 20,
+  failureLimit = 5,
+  previewLength = 6,
+} = {}) {
+  if (!productDir) throw new Error("productDir is required");
+  if (!serviceName) throw new Error("serviceName is required");
+  if (!filePath) throw new Error("filePath is required");
+
+  const runArtifact = loadCurrentRunArtifact(productDir);
+  const subject = resolveFileSubject(runArtifact, filePath, serviceName);
+  const detailLines = formatFileDetail(productDir, runArtifact, subject, {
+    logTail,
+    failureLimit,
+    previewLength,
+  });
+  const artifacts = collectArtifactEntries(productDir, runArtifact, subject.file.path, subject.service.name).map((entry) => ({
+    name: entry.artifactRef.name,
+    kind: entry.artifactRef.kind || null,
+    summary: entry.artifactRef.summary || null,
+    path: entry.artifactRef.path,
+    previewLines: formatArtifactPreview(entry.payload, previewLength),
+  }));
+  const setupOperations = getSetupOperationsForService(runArtifact, subject.service.name).slice(0, 8).map((operation) => ({
+    stage: operation.stage,
+    status: operation.status,
+    summary: operation.summary || null,
+    durationMs: operation.durationMs ?? null,
+    logPath: operation.logRef?.path || null,
+    error: operation.error || null,
+  }));
+  const backendLogs = getServiceLogRefs(runArtifact, subject.service.name).map((logRef) => ({
+    runtimeLabel: logRef.runtimeLabel,
+    path: logRef.path,
+    lines: readLogTail(path.join(productDir, logRef.path), logTail),
+  }));
+
+  return {
+    productDir,
+    runArtifact,
+    subject,
+    detailLines,
+    summary: {
+      service: subject.service.name,
+      suite: `${subject.suite.type}:${subject.suite.name}`,
+      file: subject.file.path,
+      status: subject.file.status,
+      durationMs: subject.file.durationMs || 0,
+      error: subject.file.error || null,
+      failureCount: Array.isArray(subject.file.failureDetails) ? subject.file.failureDetails.length : 0,
+    },
+    setupOperations,
+    artifacts,
+    backendLogs,
+  };
+}
+
+export function formatInvestigationTranscriptContext(context) {
+  if (!context) return "";
+  const artifactSummary = context.artifacts.length === 0
+    ? "none"
+    : context.artifacts
+        .map((entry) => `${entry.name}${entry.kind ? ` [${entry.kind}]` : ""}: ${entry.path}`)
+        .join("\n");
+  const logSummary = context.backendLogs.length === 0
+    ? "none"
+    : context.backendLogs.map((entry) => `${entry.runtimeLabel}: ${entry.path}`).join("\n");
+
+  return [
+    `Service: ${context.summary.service}`,
+    `Suite: ${context.summary.suite}`,
+    `File: ${context.summary.file}`,
+    `Status: ${context.summary.status}`,
+    context.summary.error ? `Error: ${context.summary.error}` : null,
+    "",
+    "Artifacts:",
+    artifactSummary,
+    "",
+    "Backend Logs:",
+    logSummary,
+    "",
+    "Detailed Failure View:",
+    ...context.detailLines,
+  ]
+    .filter(Boolean)
+    .join("\n");
+}

package/lib/cli/agents/investigation-context.test.mjs

@@ -0,0 +1,144 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  formatInvestigationTranscriptContext,
+  loadInvestigationContext,
+} from "./investigation-context.mjs";
+import { buildInvestigationPrompt, defaultInvestigationMessage } from "./prompt-builder.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
+  }
+});
+
+function makeTempProduct() {
+  const productDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-investigation-"));
+  tempDirs.push(productDir);
+  fs.mkdirSync(path.join(productDir, ".testkit", "results"), { recursive: true });
+  return productDir;
+}
+
+function writeJson(filePath, value) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+describe("investigation context", () => {
+  it("loads the selected failure, artifacts, logs, and formatted detail lines", () => {
+    const productDir = makeTempProduct();
+    writeJson(path.join(productDir, "artifacts", "failure.json"), {
+      source: "testkit-runtime-artifact",
+      data: { message: "artifact payload" },
+    });
+    fs.mkdirSync(path.join(productDir, "logs"), { recursive: true });
+    fs.writeFileSync(path.join(productDir, "logs", "api.log"), "first line\nsecond line\n");
+
+    writeJson(path.join(productDir, ".testkit", "results", "latest.json"), {
+      services: [
+        {
+          name: "api",
+          suites: [
+            {
+              name: "users",
+              type: "integration",
+              files: [
+                {
+                  path: "tests/api/users.int.testkit.ts",
+                  status: "failed",
+                  durationMs: 1000,
+                  error: "status 500",
+                  failureDetails: [
+                    {
+                      kind: "http-assertion",
+                      title: "status is 200",
+                      message: "GET /users expected 200, got 500",
+                      response: { status: 500, bodyPreview: "{\"error\":\"boom\"}" },
+                    },
+                  ],
+                  artifacts: [
+                    {
+                      name: "failure-artifact",
+                      kind: "example",
+                      summary: "captured payload",
+                      path: "artifacts/failure.json",
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+      logs: {
+        services: [
+          {
+            serviceName: "api",
+            runtimeLabel: "api",
+            path: "logs/api.log",
+          },
+        ],
+      },
+      setup: {
+        operations: [
+          {
+            serviceName: "api",
+            stage: "runtime:prepare",
+            status: "passed",
+            summary: "prepared runtime",
+            durationMs: 1200,
+            logRef: { path: "logs/api.log" },
+          },
+        ],
+      },
+    });
+
+    const context = loadInvestigationContext({
+      productDir,
+      serviceName: "api",
+      filePath: "tests/api/users.int.testkit.ts",
+    });
+
+    expect(context.summary).toMatchObject({
+      service: "api",
+      suite: "integration:users",
+      file: "tests/api/users.int.testkit.ts",
+      status: "failed",
+      error: "status 500",
+    });
+    expect(context.artifacts[0]).toMatchObject({
+      name: "failure-artifact",
+      kind: "example",
+      path: "artifacts/failure.json",
+    });
+    expect(context.backendLogs[0].lines).toContain("second line");
+    expect(context.detailLines.join("\n")).toContain("GET /users expected 200, got 500");
+    expect(formatInvestigationTranscriptContext(context)).toContain("Artifacts:");
+  });
+
+  it("builds the default investigation prompt with contextual evidence", () => {
+    const context = {
+      summary: {
+        service: "api",
+        suite: "integration:users",
+        file: "tests/api/users.int.testkit.ts",
+        status: "failed",
+        error: "status 500",
+      },
+      artifacts: [],
+      backendLogs: [],
+      detailLines: ["File: tests/api/users.int.testkit.ts", "Error: status 500"],
+    };
+
+    const prompt = buildInvestigationPrompt({ context });
+
+    expect(defaultInvestigationMessage()).toContain("Investigate");
+    expect(prompt).toContain("User request:");
+    expect(prompt).toContain("Service: api");
+    expect(prompt).toContain("Detailed Failure View:");
+  });
+});

package/lib/cli/agents/prompt-builder.mjs

@@ -0,0 +1,25 @@
+import { formatInvestigationTranscriptContext } from "./investigation-context.mjs";
+
+const DEFAULT_INVESTIGATION_MESSAGE =
+  "Investigate the selected testkit failure. Explain the most likely root cause, cite concrete evidence from the provided context, and suggest the next debugging steps or code changes.";
+
+export function defaultInvestigationMessage() {
+  return DEFAULT_INVESTIGATION_MESSAGE;
+}
+
+export function buildInvestigationPrompt({ context, userMessage } = {}) {
+  const message = userMessage || DEFAULT_INVESTIGATION_MESSAGE;
+  const transcriptContext = formatInvestigationTranscriptContext(context);
+
+  return [
+    "You are investigating a failed testkit run.",
+    "Use the provided run context as the primary evidence.",
+    "If the evidence is incomplete, say what additional command or file you would inspect next.",
+    "Keep the response actionable and focused on root cause, evidence, and next steps.",
+    "",
+    `User request: ${message}`,
+    "",
+    "Run context:",
+    transcriptContext,
+  ].join("\n");
+}

package/lib/cli/agents/providers/claude.mjs

@@ -0,0 +1,74 @@
+import { execa } from "execa";
+import {
+  buildErrorEvent,
+  buildStatusEvent,
+  buildToolEvent,
+  createHostedSessionRunner,
+  extractTextFragments,
+} from "./shared.mjs";
+
+export function startClaudeHostedSession({ cwd, prompt, onEvent, purpose = "investigate" } = {}) {
+  const args = [
+    "-p",
+    "--output-format",
+    "stream-json",
+    "--include-partial-messages",
+  ];
+
+  if (purpose === "investigate") {
+    args.push("--permission-mode", "plan");
+  }
+
+  args.push(prompt);
+
+  const child = execa("claude", args, {
+    cwd,
+    stdout: "pipe",
+    stderr: "pipe",
+    reject: false,
+  });
+
+  return createHostedSessionRunner({
+    provider: "claude",
+    child,
+    onEvent,
+    parsePayload: parseClaudePayload,
+    readFinalText() {
+      return null;
+    },
+  });
+}
+
+function parseClaudePayload(payload) {
+  const events = [];
+  if (!payload || typeof payload !== "object") return events;
+
+  const type = payload.type || payload.event || payload.kind || null;
+  const errorMessage = payload.error?.message || payload.error || null;
+  if (errorMessage) {
+    const event = buildErrorEvent(errorMessage);
+    if (event) events.push(event);
+    return events;
+  }
+
+  if (type && /tool/i.test(type)) {
+    const event = buildToolEvent(
+      payload.name || payload.tool_name || payload.tool || type,
+      payload.detail || payload.summary || null
+    );
+    if (event) events.push(event);
+    return events;
+  }
+
+  const fragments = [...new Set(extractTextFragments(payload, []))];
+  if (fragments.length > 0) {
+    for (const fragment of fragments) {
+      events.push({ type: "delta", text: fragment });
+    }
+    return events;
+  }
+
+  const statusEvent = buildStatusEvent(type ? `Claude event: ${type}` : JSON.stringify(payload));
+  if (statusEvent) events.push(statusEvent);
+  return events;
+}

package/lib/cli/agents/providers/claude.test.mjs

@@ -0,0 +1,95 @@
+import { PassThrough } from "stream";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const execaMock = vi.fn();
+
+vi.mock("execa", () => ({
+  execa: execaMock,
+}));
+
+function createChildHandle() {
+  const stdout = new PassThrough();
+  const stderr = new PassThrough();
+  let resolveChild;
+  const promise = new Promise((resolve) => {
+    resolveChild = resolve;
+  });
+  promise.stdout = stdout;
+  promise.stderr = stderr;
+  promise.kill = vi.fn(() => {
+    stdout.end();
+    stderr.end();
+  });
+  return {
+    child: promise,
+    finish(result = {}) {
+      stdout.end();
+      stderr.end();
+      resolveChild({
+        exitCode: 0,
+        stdout: "",
+        stderr: "",
+        ...result,
+      });
+    },
+  };
+}
+
+afterEach(() => {
+  execaMock.mockReset();
+});
+
+describe("Claude hosted session", () => {
+  it("normalizes streamed tool, delta, status, and final events", async () => {
+    const { startClaudeHostedSession } = await import("./claude.mjs");
+    const handle = createChildHandle();
+    execaMock.mockReturnValue(handle.child);
+    const events = [];
+
+    const session = startClaudeHostedSession({
+      cwd: "/tmp/project",
+      prompt: "Investigate this failure",
+      onEvent(event) {
+        events.push(event);
+      },
+    });
+
+    handle.child.stdout.write(`${JSON.stringify({ type: "tool_use", name: "Bash", detail: "rg failure" })}\n`);
+    handle.child.stdout.write(`${JSON.stringify({ type: "message_delta", delta: "Likely root cause." })}\n`);
+    handle.child.stderr.write("stderr note\n");
+    handle.finish({ exitCode: 0 });
+
+    const result = await session.completion;
+
+    expect(execaMock).toHaveBeenCalledWith(
+      "claude",
+      expect.arrayContaining(["-p", "--output-format", "stream-json", "--include-partial-messages"]),
+      expect.objectContaining({ cwd: "/tmp/project", reject: false })
+    );
+    expect(events[0]).toMatchObject({ provider: "claude", type: "start" });
+    expect(events).toContainEqual(expect.objectContaining({ provider: "claude", type: "tool", name: "Bash" }));
+    expect(events).toContainEqual(expect.objectContaining({ provider: "claude", type: "delta", text: "Likely root cause." }));
+    expect(events).toContainEqual(expect.objectContaining({ provider: "claude", type: "status", message: "stderr note" }));
+    expect(events.at(-2)).toMatchObject({ provider: "claude", type: "final", text: "Likely root cause." });
+    expect(events.at(-1)).toMatchObject({ provider: "claude", type: "exit", code: 0 });
+    expect(result.finalText).toBe("Likely root cause.");
+  });
+
+  it("cancels the child process", async () => {
+    const { startClaudeHostedSession } = await import("./claude.mjs");
+    const handle = createChildHandle();
+    execaMock.mockReturnValue(handle.child);
+
+    const session = startClaudeHostedSession({
+      cwd: "/tmp/project",
+      prompt: "Investigate this failure",
+    });
+
+    session.cancel();
+    handle.finish({ exitCode: 130 });
+    const result = await session.completion;
+
+    expect(handle.child.kill).toHaveBeenCalledWith("SIGTERM");
+    expect(result.cancelled).toBe(true);
+  });
+});