@elench/testkit 0.1.82 → 0.1.84

package/README.md

@@ -365,9 +365,11 @@ HTTP suites:
 
 ```ts
 import { defineHttpSuite } from "@elench/testkit";
+import { expect } from "@elench/testkit/runtime";
 
 const suite = defineHttpSuite(({ rawReq }) => {
-  rawReq("GET", "/health");
+  const response = rawReq.get("/health");
+  expect.status(response, 200, "health returns 200");
 });
 
 export default suite;
@@ -410,9 +412,11 @@ export default defineConfig({
   },
 });
 
-const suite = defineHttpSuite({ profile: "defaultAuth" }, ({ actor, req }) => {
+const suite = defineHttpSuite({ profile: "defaultAuth" }, ({ actor, actors, req }) => {
   req.get("/api/auth/session");
   actor?.req.get("/api/auth/session");
+  req.as("outsider").get("/api/auth/session");
+  actors.get("reviewer").rawReq.get("/api/auth/session");
 });
 ```
 
@@ -439,7 +443,7 @@ const suite = defineScenarioSuite(({ rawReq, scenario }) => {
     includeHealthCheck: scenario.maybe("includeHealthCheck", 1),
   });
 
-  const selected = scenario.resource("selected-endpoint", () => rawReq("GET", plan.endpoint));
+  const selected = scenario.resource("selected-endpoint", () => rawReq.get(plan.endpoint));
 
   scenario.step("fetch selected endpoint", () => {
     selected.get();
@@ -449,21 +453,47 @@ const suite = defineScenarioSuite(({ rawReq, scenario }) => {
 export default suite;
 ```
 
-Low-level runtime primitives remain available:
+First-class runtime namespaces:
 
 ```ts
-import { check, group, http, waitFor } from "@elench/testkit/runtime";
+import { checks, expect, parse, waitFor } from "@elench/testkit/runtime";
+
+const suite = defineHttpSuite(({ actor, rawReq, req }) => {
+  const me = req.get("/api/v1/me");
+  expect.status(me, 200, "authenticated session loads");
+  expect.json(me, (body) => typeof body.data?.id === "string", "response includes user id");
+
+  checks.authGate(rawReq, "sessions", {
+    get: ["/api/v1/sessions"],
+  });
+
+  actor?.rawHeaders();
+
+  const upload = req.multipart.post("/api/v1/uploads", {
+    fields: { name: "fixture" },
+    files: [{ field: "file", data: "hello", filename: "fixture.txt", contentType: "text/plain" }],
+  });
+
+  expect.status(upload, 201, "upload is accepted");
+  parse.safeJson(upload);
+});
+```
+
+Low-level runtime primitives still exist when you genuinely need them:
+
+```ts
+import { check, group, http } from "@elench/testkit/runtime";
 ```
 
 `waitFor()` consumes the file budget configured by `execution.fileTimeoutSeconds`.
 Consumers should not set local timeout values in test files.
 
 ```ts
-import { waitFor } from "@elench/testkit/runtime";
+import { parse, waitFor } from "@elench/testkit/runtime";
 
 const response = waitFor(
   () => req.get("/api/v1/jobs/123"),
-  (res) => JSON.parse(res.body).data?.status === "completed",
+  (res) => parse.json(res).data?.status === "completed",
   { description: "job 123 to complete" }
 );
 ```

package/lib/cli/agents/index.mjs

@@ -0,0 +1,64 @@
+import fs from "fs";
+import path from "path";
+import { spawn } from "child_process";
+import { startClaudeHostedSession } from "./providers/claude.mjs";
+import { startCodexHostedSession } from "./providers/codex.mjs";
+
+const PROVIDERS = ["codex", "claude"];
+
+export function resolvePreferredProvider(preferred = null, env = process.env) {
+  if (preferred && preferred !== "auto") {
+    if (!isProviderInstalled(preferred, env)) {
+      throw new Error(`${preferred} is not installed or not available on PATH`);
+    }
+    return preferred;
+  }
+
+  for (const provider of PROVIDERS) {
+    if (isProviderInstalled(provider, env)) return provider;
+  }
+  throw new Error("Neither codex nor claude was found on PATH");
+}
+
+export function isProviderInstalled(provider, env = process.env) {
+  const pathValue = env.PATH || "";
+  const candidates = pathValue.split(path.delimiter).filter(Boolean);
+  const fileNames =
+    process.platform === "win32"
+      ? [`${provider}.exe`, `${provider}.cmd`, `${provider}.bat`, provider]
+      : [provider];
+  for (const directory of candidates) {
+    for (const fileName of fileNames) {
+      const candidate = path.join(directory, fileName);
+      try {
+        fs.accessSync(candidate, fs.constants.X_OK);
+        return true;
+      } catch {
+        // Continue searching PATH entries.
+      }
+    }
+  }
+  return false;
+}
+
+export function startAgentSession({ provider = "auto", cwd, prompt, onEvent, purpose = "investigate" } = {}) {
+  const resolvedProvider = resolvePreferredProvider(provider);
+  if (resolvedProvider === "claude") {
+    return startClaudeHostedSession({ cwd, prompt, onEvent, purpose });
+  }
+  return startCodexHostedSession({ cwd, prompt, onEvent, purpose });
+}
+
+export function startInteractiveAgentHandoff({ provider = "auto", cwd, prompt } = {}) {
+  const resolvedProvider = resolvePreferredProvider(provider);
+  const command = resolvedProvider;
+  const args = prompt ? [prompt] : [];
+  const child = spawn(command, args, {
+    cwd,
+    stdio: "inherit",
+  });
+  return new Promise((resolve, reject) => {
+    child.on("error", reject);
+    child.on("close", (code) => resolve({ provider: resolvedProvider, exitCode: code ?? 0 }));
+  });
+}

package/lib/cli/agents/investigate.mjs

@@ -0,0 +1,75 @@
+import { loadInvestigationContext } from "./investigation-context.mjs";
+import { buildInvestigationPrompt } from "./prompt-builder.mjs";
+import { startAgentSession, startInteractiveAgentHandoff } from "./index.mjs";
+
+export function createInvestigationRequest({
+  productDir,
+  serviceName,
+  filePath,
+  provider = "auto",
+  userMessage,
+} = {}) {
+  const context = loadInvestigationContext({
+    productDir,
+    serviceName,
+    filePath,
+  });
+  const prompt = buildInvestigationPrompt({ context, userMessage });
+  return {
+    provider,
+    context,
+    prompt,
+  };
+}
+
+export function startHostedInvestigation({
+  productDir,
+  serviceName,
+  filePath,
+  provider = "auto",
+  userMessage,
+  onEvent,
+} = {}) {
+  const request = createInvestigationRequest({
+    productDir,
+    serviceName,
+    filePath,
+    provider,
+    userMessage,
+  });
+  const session = startAgentSession({
+    provider: request.provider,
+    cwd: productDir,
+    prompt: request.prompt,
+    onEvent,
+  });
+  return {
+    ...session,
+    request,
+  };
+}
+
+export async function runInteractiveInvestigation({
+  productDir,
+  serviceName,
+  filePath,
+  provider = "auto",
+  userMessage,
+} = {}) {
+  const request = createInvestigationRequest({
+    productDir,
+    serviceName,
+    filePath,
+    provider,
+    userMessage,
+  });
+  const result = await startInteractiveAgentHandoff({
+    provider: request.provider,
+    cwd: productDir,
+    prompt: request.prompt,
+  });
+  return {
+    ...result,
+    request,
+  };
+}

package/lib/cli/agents/investigation-context.mjs

@@ -0,0 +1,102 @@
+import path from "path";
+import {
+  collectArtifactEntries,
+  formatArtifactPreview,
+  formatFileDetail,
+  getServiceLogRefs,
+  getSetupOperationsForService,
+  loadCurrentRunArtifact,
+  resolveFileSubject,
+} from "../viewer.mjs";
+import { readLogTail } from "../../runner/logs.mjs";
+
+export function loadInvestigationContext({
+  productDir,
+  serviceName,
+  filePath,
+  logTail = 20,
+  failureLimit = 5,
+  previewLength = 6,
+} = {}) {
+  if (!productDir) throw new Error("productDir is required");
+  if (!serviceName) throw new Error("serviceName is required");
+  if (!filePath) throw new Error("filePath is required");
+
+  const runArtifact = loadCurrentRunArtifact(productDir);
+  const subject = resolveFileSubject(runArtifact, filePath, serviceName);
+  const detailLines = formatFileDetail(productDir, runArtifact, subject, {
+    logTail,
+    failureLimit,
+    previewLength,
+  });
+  const artifacts = collectArtifactEntries(productDir, runArtifact, subject.file.path, subject.service.name).map((entry) => ({
+    name: entry.artifactRef.name,
+    kind: entry.artifactRef.kind || null,
+    summary: entry.artifactRef.summary || null,
+    path: entry.artifactRef.path,
+    previewLines: formatArtifactPreview(entry.payload, previewLength),
+  }));
+  const setupOperations = getSetupOperationsForService(runArtifact, subject.service.name).slice(0, 8).map((operation) => ({
+    stage: operation.stage,
+    status: operation.status,
+    summary: operation.summary || null,
+    durationMs: operation.durationMs ?? null,
+    logPath: operation.logRef?.path || null,
+    error: operation.error || null,
+  }));
+  const backendLogs = getServiceLogRefs(runArtifact, subject.service.name).map((logRef) => ({
+    runtimeLabel: logRef.runtimeLabel,
+    path: logRef.path,
+    lines: readLogTail(path.join(productDir, logRef.path), logTail),
+  }));
+
+  return {
+    productDir,
+    runArtifact,
+    subject,
+    detailLines,
+    summary: {
+      service: subject.service.name,
+      suite: `${subject.suite.type}:${subject.suite.name}`,
+      file: subject.file.path,
+      status: subject.file.status,
+      durationMs: subject.file.durationMs || 0,
+      error: subject.file.error || null,
+      failureCount: Array.isArray(subject.file.failureDetails) ? subject.file.failureDetails.length : 0,
+    },
+    setupOperations,
+    artifacts,
+    backendLogs,
+  };
+}
+
+export function formatInvestigationTranscriptContext(context) {
+  if (!context) return "";
+  const artifactSummary = context.artifacts.length === 0
+    ? "none"
+    : context.artifacts
+        .map((entry) => `${entry.name}${entry.kind ? ` [${entry.kind}]` : ""}: ${entry.path}`)
+        .join("\n");
+  const logSummary = context.backendLogs.length === 0
+    ? "none"
+    : context.backendLogs.map((entry) => `${entry.runtimeLabel}: ${entry.path}`).join("\n");
+
+  return [
+    `Service: ${context.summary.service}`,
+    `Suite: ${context.summary.suite}`,
+    `File: ${context.summary.file}`,
+    `Status: ${context.summary.status}`,
+    context.summary.error ? `Error: ${context.summary.error}` : null,
+    "",
+    "Artifacts:",
+    artifactSummary,
+    "",
+    "Backend Logs:",
+    logSummary,
+    "",
+    "Detailed Failure View:",
+    ...context.detailLines,
+  ]
+    .filter(Boolean)
+    .join("\n");
+}

package/lib/cli/agents/prompt-builder.mjs

@@ -0,0 +1,25 @@
+import { formatInvestigationTranscriptContext } from "./investigation-context.mjs";
+
+const DEFAULT_INVESTIGATION_MESSAGE =
+  "Investigate the selected testkit failure. Explain the most likely root cause, cite concrete evidence from the provided context, and suggest the next debugging steps or code changes.";
+
+export function defaultInvestigationMessage() {
+  return DEFAULT_INVESTIGATION_MESSAGE;
+}
+
+export function buildInvestigationPrompt({ context, userMessage } = {}) {
+  const message = userMessage || DEFAULT_INVESTIGATION_MESSAGE;
+  const transcriptContext = formatInvestigationTranscriptContext(context);
+
+  return [
+    "You are investigating a failed testkit run.",
+    "Use the provided run context as the primary evidence.",
+    "If the evidence is incomplete, say what additional command or file you would inspect next.",
+    "Keep the response actionable and focused on root cause, evidence, and next steps.",
+    "",
+    `User request: ${message}`,
+    "",
+    "Run context:",
+    transcriptContext,
+  ].join("\n");
+}

package/lib/cli/agents/providers/claude.mjs

@@ -0,0 +1,74 @@
+import { execa } from "execa";
+import {
+  buildErrorEvent,
+  buildStatusEvent,
+  buildToolEvent,
+  createHostedSessionRunner,
+  extractTextFragments,
+} from "./shared.mjs";
+
+export function startClaudeHostedSession({ cwd, prompt, onEvent, purpose = "investigate" } = {}) {
+  const args = [
+    "-p",
+    "--output-format",
+    "stream-json",
+    "--include-partial-messages",
+  ];
+
+  if (purpose === "investigate") {
+    args.push("--permission-mode", "plan");
+  }
+
+  args.push(prompt);
+
+  const child = execa("claude", args, {
+    cwd,
+    stdout: "pipe",
+    stderr: "pipe",
+    reject: false,
+  });
+
+  return createHostedSessionRunner({
+    provider: "claude",
+    child,
+    onEvent,
+    parsePayload: parseClaudePayload,
+    readFinalText() {
+      return null;
+    },
+  });
+}
+
+function parseClaudePayload(payload) {
+  const events = [];
+  if (!payload || typeof payload !== "object") return events;
+
+  const type = payload.type || payload.event || payload.kind || null;
+  const errorMessage = payload.error?.message || payload.error || null;
+  if (errorMessage) {
+    const event = buildErrorEvent(errorMessage);
+    if (event) events.push(event);
+    return events;
+  }
+
+  if (type && /tool/i.test(type)) {
+    const event = buildToolEvent(
+      payload.name || payload.tool_name || payload.tool || type,
+      payload.detail || payload.summary || null
+    );
+    if (event) events.push(event);
+    return events;
+  }
+
+  const fragments = [...new Set(extractTextFragments(payload, []))];
+  if (fragments.length > 0) {
+    for (const fragment of fragments) {
+      events.push({ type: "delta", text: fragment });
+    }
+    return events;
+  }
+
+  const statusEvent = buildStatusEvent(type ? `Claude event: ${type}` : JSON.stringify(payload));
+  if (statusEvent) events.push(statusEvent);
+  return events;
+}

package/lib/cli/agents/providers/codex.mjs

@@ -0,0 +1,83 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { execa } from "execa";
+import {
+  buildErrorEvent,
+  buildStatusEvent,
+  buildToolEvent,
+  createHostedSessionRunner,
+  extractTextFragments,
+  readTextFileIfPresent,
+} from "./shared.mjs";
+
+export function startCodexHostedSession({ cwd, prompt, onEvent, purpose = "investigate" } = {}) {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-codex-"));
+  const outputFile = path.join(tempDir, "final-message.txt");
+  const args = ["exec", "--json", "-o", outputFile];
+
+  if (purpose === "investigate") {
+    args.push("-s", "read-only");
+  }
+
+  args.push(prompt);
+
+  const child = execa("codex", args, {
+    cwd,
+    stdout: "pipe",
+    stderr: "pipe",
+    reject: false,
+  });
+
+  const session = createHostedSessionRunner({
+    provider: "codex",
+    child,
+    onEvent,
+    parsePayload: parseCodexPayload,
+    readFinalText(result) {
+      return readTextFileIfPresent(outputFile) || result.stdout || null;
+    },
+  });
+
+  const completion = session.completion.finally(() => {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  return {
+    completion,
+    cancel: session.cancel,
+  };
+}
+
+function parseCodexPayload(payload) {
+  const events = [];
+  if (!payload || typeof payload !== "object") return events;
+  const type = payload.type || payload.event || payload.kind || null;
+  const errorMessage = payload.error?.message || payload.error || null;
+  if (errorMessage) {
+    const event = buildErrorEvent(errorMessage);
+    if (event) events.push(event);
+    return events;
+  }
+
+  if (type && /(tool|command|patch|exec)/i.test(type)) {
+    const event = buildToolEvent(
+      payload.name || payload.tool_name || payload.command || type,
+      payload.detail || payload.summary || payload.status || null
+    );
+    if (event) events.push(event);
+    return events;
+  }
+
+  const fragments = [...new Set(extractTextFragments(payload, []))];
+  if (fragments.length > 0) {
+    for (const fragment of fragments) {
+      events.push({ type: "delta", text: fragment });
+    }
+    return events;
+  }
+
+  const statusEvent = buildStatusEvent(type ? `Codex event: ${type}` : JSON.stringify(payload));
+  if (statusEvent) events.push(statusEvent);
+  return events;
+}

package/lib/cli/agents/providers/shared.mjs

@@ -0,0 +1,134 @@
+import fs from "fs";
+import readline from "readline";
+
+export function createHostedSessionRunner({ provider, child, onEvent, parsePayload, readFinalText } = {}) {
+  let cancelled = false;
+  let settled = false;
+  let assistantText = "";
+
+  const emit = (event) => {
+    if (event?.type === "delta" || event?.type === "final") {
+      assistantText += event.text || "";
+    }
+    if (typeof onEvent === "function" && event) onEvent({ provider, ...event });
+  };
+
+  emit({ type: "start" });
+
+  const stdoutReader = readline.createInterface({ input: child.stdout });
+  stdoutReader.on("line", (line) => {
+    const parsed = tryParseJson(line);
+    if (parsed == null) {
+      emit({ type: "status", message: line });
+      return;
+    }
+    const events = parsePayload ? parsePayload(parsed) : [];
+    if (!Array.isArray(events) || events.length === 0) return;
+    for (const event of events) emit(event);
+  });
+
+  const stderrReader = readline.createInterface({ input: child.stderr });
+  stderrReader.on("line", (line) => {
+    emit({ type: "status", message: line });
+  });
+
+  const completion = (async () => {
+    const result = await child;
+    const finalText = (readFinalText ? readFinalText(result) : null) || assistantText.trim() || null;
+    if (finalText) emit({ type: "final", text: finalText });
+    emit({ type: "exit", code: result.exitCode ?? 0 });
+    settled = true;
+    return {
+      provider,
+      exitCode: result.exitCode ?? 0,
+      stdout: result.stdout || "",
+      stderr: result.stderr || "",
+      finalText: finalText || result.stdout || "",
+      cancelled,
+    };
+  })();
+
+  return {
+    completion,
+    cancel() {
+      cancelled = true;
+      if (settled) return;
+      try {
+        child.kill("SIGTERM");
+      } catch {
+        // Ignore cancellation races.
+      }
+    },
+  };
+}
+
+export function tryParseJson(line) {
+  const normalized = String(line || "").trim();
+  if (!normalized) return null;
+  try {
+    return JSON.parse(normalized);
+  } catch {
+    return null;
+  }
+}
+
+export function buildToolEvent(name, detail = null) {
+  if (!name) return null;
+  return {
+    type: "tool",
+    name: String(name),
+    ...(detail ? { detail: String(detail) } : {}),
+  };
+}
+
+export function buildStatusEvent(message) {
+  if (!message) return null;
+  return { type: "status", message: String(message) };
+}
+
+export function buildErrorEvent(message) {
+  if (!message) return null;
+  return { type: "error", message: String(message) };
+}
+
+export function extractTextFragments(payload, fragments = [], depth = 0) {
+  if (payload == null || depth > 5 || fragments.length >= 12) return fragments;
+  if (typeof payload === "string") {
+    const normalized = payload.trim();
+    if (normalized) fragments.push(normalized);
+    return fragments;
+  }
+  if (Array.isArray(payload)) {
+    for (const entry of payload) {
+      extractTextFragments(entry, fragments, depth + 1);
+      if (fragments.length >= 12) break;
+    }
+    return fragments;
+  }
+  if (typeof payload !== "object") return fragments;
+
+  const preferredKeys = ["delta", "text", "content", "message", "output_text", "summary"];
+  let matchedPreferredKey = false;
+  for (const key of preferredKeys) {
+    if (!(key in payload)) continue;
+    matchedPreferredKey = true;
+    extractTextFragments(payload[key], fragments, depth + 1);
+    if (fragments.length >= 12) return fragments;
+  }
+  if (matchedPreferredKey && fragments.length > 0) return fragments;
+
+  for (const value of Object.values(payload)) {
+    extractTextFragments(value, fragments, depth + 1);
+    if (fragments.length >= 12) break;
+  }
+  return fragments;
+}
+
+export function readTextFileIfPresent(filePath) {
+  if (!filePath) return null;
+  try {
+    return fs.readFileSync(filePath, "utf8").trim() || null;
+  } catch {
+    return null;
+  }
+}