@elench/testkit 0.1.96 → 0.1.97

package/lib/cli/agents/providers/claude.mjs

@@ -4,6 +4,7 @@ import {
   buildStatusEvent,
   buildToolEvent,
   createHostedSessionRunner,
+  extractTextFragments,
 } from "./shared.mjs";
 
 export function startClaudeHostedSession({
@@ -50,8 +51,8 @@ export function startClaudeHostedSession({
     child,
     onEvent,
     parsePayload: parseClaudePayload,
-    readFinalText() {
-      return null;
+    readFinalText(result) {
+      return readClaudeFinalText(result?.stdout || "") || null;
     },
   });
 }
@@ -61,7 +62,7 @@ function normalizeProviderArgs(providerArgs) {
   return providerArgs.flatMap((arg) => String(arg || "").split(/\s+/).filter(Boolean));
 }
 
-function parseClaudePayload(payload) {
+export function parseClaudePayload(payload) {
   const events = [];
   if (!payload || typeof payload !== "object") return events;
 
@@ -102,6 +103,10 @@ function parseClaudePayload(payload) {
   }
 
   if (type === "assistant") {
+    const fragments = [...new Set(extractTextFragments(payload.message?.content || payload.content || [], []))];
+    for (const fragment of fragments) {
+      events.push({ type: "delta", text: fragment });
+    }
     return events;
   }
 
@@ -113,7 +118,40 @@ function parseClaudePayload(payload) {
     return events;
   }
 
+  if (type === "system" || type === "rate_limit_event") {
+    return events;
+  }
+
   const statusEvent = buildStatusEvent(type ? `Claude event: ${type}` : JSON.stringify(payload));
   if (statusEvent) events.push(statusEvent);
   return events;
 }
+
+export function readClaudeFinalText(stdout) {
+  const lines = String(stdout || "")
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean);
+
+  let fallback = null;
+  for (const line of lines) {
+    let payload = null;
+    try {
+      payload = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (!payload || typeof payload !== "object") continue;
+
+    if (payload.type === "result" && payload.subtype !== "error" && typeof payload.result === "string") {
+      return payload.result.trim() || null;
+    }
+
+    if (payload.type === "assistant") {
+      const fragments = [...new Set(extractTextFragments(payload.message?.content || payload.content || [], []))];
+      if (fragments.length > 0) fallback = fragments.join("");
+    }
+  }
+
+  return fallback;
+}

package/lib/cli/agents/providers/codex.mjs

@@ -22,17 +22,14 @@ export function startCodexHostedSession({
 } = {}) {
   const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-codex-"));
   const outputFile = path.join(tempDir, "final-message.txt");
-  const args = ["exec", "--json", "-o", outputFile];
-
-  if (purpose === "assistant") {
-    args.push("-s", "read-only");
-  }
-  if (model) {
-    args.push("--model", String(model));
-  }
-  args.push(...normalizeProviderArgs(providerArgs));
-
-  args.push(prompt);
+  const args = buildCodexArgs({
+    outputFile,
+    purpose,
+    model,
+    providerArgs,
+    prompt,
+    sandbox: process.env.TESTKIT_CODEX_SANDBOX,
+  });
 
   const child = execa(command, args, {
     cwd,
@@ -51,7 +48,7 @@ export function startCodexHostedSession({
       return String(message || "").trim() === "Reading additional input from stdin...";
     },
     readFinalText(result) {
-      return readTextFileIfPresent(outputFile) || result.stdout || null;
+      return readTextFileIfPresent(outputFile) || null;
     },
   });
 
@@ -65,16 +62,38 @@ export function startCodexHostedSession({
   };
 }
 
+export function buildCodexArgs({
+  outputFile,
+  purpose = "assistant",
+  model = null,
+  providerArgs = [],
+  prompt = "",
+  sandbox = null,
+} = {}) {
+  const args = ["exec", "--json"];
+  if (outputFile) args.push("-o", outputFile);
+
+  if (purpose === "assistant") {
+    args.push("-s", String(sandbox || "workspace-write"));
+  }
+  if (model) {
+    args.push("--model", String(model));
+  }
+  args.push(...normalizeProviderArgs(providerArgs));
+  args.push(prompt);
+  return args;
+}
+
 function normalizeProviderArgs(providerArgs) {
   if (!Array.isArray(providerArgs)) return [];
   return providerArgs.flatMap((arg) => String(arg || "").split(/\s+/).filter(Boolean));
 }
 
-function parseCodexPayload(payload) {
+export function parseCodexPayload(payload) {
   const events = [];
   if (!payload || typeof payload !== "object") return events;
   const type = payload.type || payload.event || payload.kind || null;
-  const errorMessage = payload.error?.message || payload.error || null;
+  const errorMessage = payload.error?.message || payload.error || (type === "error" ? payload.message : null) || null;
   if (errorMessage) {
     const event = buildErrorEvent(errorMessage);
     if (event) events.push(event);

package/lib/cli/agents/providers/shared.mjs

@@ -5,11 +5,15 @@ export function createHostedSessionRunner({ provider, child, onEvent, parsePaylo
   let cancelled = false;
   let settled = false;
   let assistantText = "";
+  let lastErrorMessage = null;
 
   const emit = (event) => {
     if (event?.type === "delta" || event?.type === "final") {
       assistantText += event.text || "";
     }
+    if (event?.type === "error") {
+      lastErrorMessage = event.message || lastErrorMessage;
+    }
     if (typeof onEvent === "function" && event) onEvent({ provider, ...event });
   };
 
@@ -36,8 +40,8 @@ export function createHostedSessionRunner({ provider, child, onEvent, parsePaylo
   const completion = (async () => {
     const result = await child;
     const finalText = (readFinalText ? readFinalText(result) : null) || assistantText.trim() || null;
-    if ((result.exitCode ?? 0) !== 0 && !finalText) {
-      const message = result.stderr || result.stdout || `${provider} exited with code ${result.exitCode ?? 1}`;
+    if ((result.exitCode ?? 0) !== 0) {
+      const message = lastErrorMessage || result.stderr || `${provider} exited with code ${result.exitCode ?? 1}`;
       emit({ type: "error", message });
       throw new Error(message);
     }

package/lib/cli/assistant/app.mjs

@@ -1,7 +1,7 @@
-import React, { createElement, useEffect, useMemo, useState } from "react";
-import { Box, Text, useApp, useInput, useStdout } from "ink";
+import React, { createElement, useEffect, useMemo, useRef, useState } from "react";
+import { Box, Text, useApp, useBoxMetrics, useCursor, useInput, useStdout } from "ink";
 import { bold, cyan, dim, green, red, yellow } from "../presentation/colors.mjs";
-import { getComposerRenderParts } from "./composer.mjs";
+import { getComposerDisplayModel } from "./composer.mjs";
 import { buildAssistantViewModel } from "./view-model.mjs";
 
 const MAX_BLOCK_LINES = 18;
@@ -177,15 +177,30 @@ function renderBlock(block) {
 }
 
 function ComposerBar({ view, busy }) {
-  const { before, current, after, empty } = getComposerRenderParts({
-    text: view.composer.text,
-    cursor: view.composer.cursor,
-  });
+  const ref = useRef(null);
+  const metrics = useBoxMetrics(ref);
+  const { setCursorPosition } = useCursor();
+  const display = getComposerDisplayModel(
+    {
+      text: view.composer.text,
+      cursor: view.composer.cursor,
+    },
+    { placeholder: view.composer.placeholder }
+  );
+  setCursorPosition(
+    metrics.hasMeasured
+      ? {
+          x: 2 + display.cursorColumn,
+          y: metrics.top + 1,
+        }
+      : undefined
+  );
+
   const prompt = cyan("❯");
-  const promptText = empty ? dim(`${view.composer.placeholder} `) : before;
   return createElement(
     Box,
     {
+      ref,
       borderStyle: "single",
       borderLeft: false,
       borderRight: false,
@@ -196,9 +211,7 @@ function ComposerBar({ view, busy }) {
       Text,
       null,
       `${prompt} `,
-      promptText,
-      createElement(Text, { inverse: true }, current),
-      after,
+      display.empty ? dim(display.placeholder) : display.text,
       busy ? dim("  provider responding") : ""
     )
   );

package/lib/cli/assistant/composer.mjs

@@ -1,3 +1,5 @@
+import { measureWidth } from "../presentation/terminal-layout.mjs";
+
 const segmenter =
   typeof Intl !== "undefined" && typeof Intl.Segmenter === "function"
     ? new Intl.Segmenter(undefined, { granularity: "grapheme" })
@@ -81,14 +83,30 @@ export function moveComposerCursorToEnd(state) {
 export function getComposerRenderParts(state) {
   const parts = splitGraphemes(state?.text || "");
   const cursor = clampCursor(state?.cursor ?? parts.length, parts.length);
+  const before = parts.slice(0, cursor).join("");
   return {
-    before: parts.slice(0, cursor).join(""),
+    before,
     current: parts[cursor] || " ",
     after: parts.slice(cursor + (parts[cursor] ? 1 : 0)).join(""),
     empty: parts.length === 0,
   };
 }
 
+export function getComposerDisplayModel(state, { placeholder = "" } = {}) {
+  const parts = splitGraphemes(state?.text || "");
+  const cursor = clampCursor(state?.cursor ?? parts.length, parts.length);
+  const before = parts.slice(0, cursor).join("");
+  const text = parts.join("");
+  return {
+    text,
+    before,
+    after: parts.slice(cursor).join(""),
+    cursorColumn: measureWidth(before),
+    placeholder: String(placeholder || ""),
+    empty: parts.length === 0,
+  };
+}
+
 function normalizeComposerState(state) {
   const text = String(state?.text || "");
   const parts = splitGraphemes(text);

package/lib/cli/assistant/context-pack.mjs

@@ -160,9 +160,8 @@ function buildContextMarkdown(productDir, snapshot, paths) {
   lines.push(
     "",
     "## Guidance",
-    "- Use the local `testkit` command directly when you need to execute or inspect tests.",
-    "- Preferred commands: `testkit run --dir . --type <type>`, `testkit discover --dir .`, `testkit status --dir .`, and `testkit doctor --dir .`.",
-    "- Do not launch testkit through pnpm, npm, yarn, bun, or npx unless the user explicitly asks for that exact package-manager command.",
+    "- Use shell commands like `npm run testkit`, `npx testkit`, or `testkit run <type> --dir .` when you need to execute tests.",
+    "- Do not reinterpret CLI syntax after an execution failure unless `testkit run --help` confirms a syntax problem.",
     "- Use the command log and focused context files before rereading artifacts manually.",
     "- Prefer repo-local commands over guessing project-specific wrappers.",
     ""
@@ -175,13 +174,15 @@ function buildCommandsMarkdown() {
   return [
     "# Testkit Commands",
     "",
-    "- `testkit run --dir . --type int`",
-    "- `testkit run --dir . --type e2e`",
+    "- `testkit run int --dir .`",
+    "- `testkit run e2e --dir .`",
     "- `testkit run --dir . --file path/to/file.testkit.ts`",
     "- `testkit discover --dir .`",
     "- `testkit status --dir .`",
     "- `testkit doctor --dir .`",
     "- `testkit destroy --dir .`",
+    "- `npm run testkit`",
+    "- `npx testkit run e2e --dir .`",
     "",
   ].join("\n");
 }

package/lib/cli/assistant/model-discovery.mjs

@@ -0,0 +1,243 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { execaCommand } from "execa";
+
+const CLAUDE_ALIASES = ["default", "best", "sonnet", "opus", "haiku", "opusplan", "sonnet[1m]", "opus[1m]"];
+const CACHE_MAX_AGE_MS = 5 * 60 * 1000;
+
+export async function discoverAssistantModels({
+  provider,
+  productDir = process.cwd(),
+  env = process.env,
+} = {}) {
+  const resolvedProvider = String(provider || "").trim();
+  if (resolvedProvider === "codex") {
+    return discoverCodexModels({ productDir, env });
+  }
+  if (resolvedProvider === "claude") {
+    return discoverClaudeModels({ productDir, env });
+  }
+  return {
+    provider: resolvedProvider || "unknown",
+    source: "none",
+    models: [providerDefaultModel()],
+    warnings: ["No provider is resolved yet."],
+  };
+}
+
+export async function discoverCodexModels({ productDir = process.cwd(), env = process.env } = {}) {
+  const cachePath = path.join(productDir, ".testkit", "assistant", "model-cache-codex.json");
+  const cached = readFreshCache(cachePath);
+  if (cached) return cached;
+
+  const command = env.TESTKIT_CODEX_BIN || "codex";
+  let catalog = null;
+  let warning = null;
+  try {
+    const result = await execaCommand(`${shellQuote(command)} debug models`, {
+      cwd: productDir,
+      reject: false,
+      shell: true,
+      env: { ...process.env, ...env },
+    });
+    if ((result.exitCode ?? 1) === 0) {
+      catalog = JSON.parse(result.stdout || "{}");
+    } else {
+      warning = (result.stderr || result.stdout || "codex debug models failed").trim();
+    }
+  } catch (error) {
+    warning = error instanceof Error ? error.message : String(error);
+  }
+
+  if (!catalog) {
+    catalog = readJson(path.join(os.homedir(), ".codex", "models_cache.json"));
+  }
+
+  const models = normalizeCodexModels(catalog);
+  const discovery = {
+    provider: "codex",
+    source: catalog ? "codex debug models" : "fallback",
+    models: [providerDefaultModel(), ...models],
+    warnings: warning && models.length === 0 ? [warning] : [],
+  };
+  writeCache(cachePath, discovery);
+  return discovery;
+}
+
+export async function discoverClaudeModels({ productDir = process.cwd(), env = process.env } = {}) {
+  const configured = readClaudeAvailableModels({ productDir });
+  const apiModels = await fetchAnthropicModels({ env });
+  const dynamicModels = apiModels.map((model) => ({
+    id: model.id,
+    label: model.displayName || model.id,
+    description: model.description || "",
+    source: "anthropic api",
+    concrete: true,
+  }));
+  const aliasModels = CLAUDE_ALIASES.map((id) => ({
+    id: id === "default" ? null : id,
+    label: id === "default" ? "provider default" : id,
+    description: id === "default" ? "Use Claude Code's selected default model." : "Claude Code model alias.",
+    source: "claude alias",
+    concrete: false,
+  }));
+
+  const merged = dedupeModels([...aliasModels, ...dynamicModels]);
+  const restricted = configured.length > 0
+    ? merged.filter((model) => model.id == null || configured.includes(model.id))
+    : merged;
+
+  return {
+    provider: "claude",
+    source: apiModels.length > 0 ? "anthropic api" : "claude aliases",
+    models: restricted,
+    warnings: apiModels.length > 0
+      ? []
+      : ["Claude Code does not expose a scriptable model catalog; showing stable aliases."],
+  };
+}
+
+export function normalizeCodexModels(catalog) {
+  const rawModels = Array.isArray(catalog?.models) ? catalog.models : [];
+  return rawModels
+    .filter((model) => model?.slug && (model.visibility == null || model.visibility === "list"))
+    .sort((a, b) => Number(a.priority ?? 999) - Number(b.priority ?? 999))
+    .map((model) => ({
+      id: String(model.slug),
+      label: String(model.display_name || model.slug),
+      description: String(model.description || ""),
+      source: "codex catalog",
+      concrete: true,
+      defaultEffort: model.default_reasoning_level || null,
+      efforts: Array.isArray(model.supported_reasoning_levels)
+        ? model.supported_reasoning_levels.map((entry) => entry.effort).filter(Boolean)
+        : [],
+    }));
+}
+
+export function getModelProviderMismatch(provider, model) {
+  const normalizedModel = String(model || "").trim().toLowerCase();
+  if (!provider || !normalizedModel) return null;
+
+  const looksClaude = /\b(?:opus|sonnet|haiku|claude|opusplan|best)\b/.test(normalizedModel);
+  const looksCodex = /\b(?:gpt|codex|o[1-9]|chatgpt)\b/.test(normalizedModel);
+
+  if (provider === "codex" && looksClaude) {
+    return `Model "${model}" looks like a Claude model, but the assistant is using Codex. Run /provider claude or /model default.`;
+  }
+  if (provider === "claude" && looksCodex) {
+    return `Model "${model}" looks like a Codex/OpenAI model, but the assistant is using Claude. Run /provider codex or /model default.`;
+  }
+  return null;
+}
+
+export function formatModelChoices(discovery, { currentModel = null } = {}) {
+  const current = currentModel || "provider default";
+  const lines = [
+    `Models for ${discovery.provider}`,
+    `Current: ${current}`,
+    "",
+    ...discovery.models.map((model) => {
+      const command = model.id ? `/model ${model.id}` : "/model default";
+      const marker = (model.id || null) === (currentModel || null) ? "*" : " ";
+      return `${marker} ${model.label}  ${command}`;
+    }),
+    "  custom...  /model custom <model>",
+  ];
+  for (const warning of discovery.warnings || []) {
+    lines.push("", `Note: ${warning}`);
+  }
+  return lines.join("\n");
+}
+
+export function providerDefaultModel() {
+  return {
+    id: null,
+    label: "provider default",
+    description: "Use the provider CLI default model.",
+    source: "provider default",
+    concrete: false,
+  };
+}
+
+function readFreshCache(filePath) {
+  const value = readJson(filePath);
+  if (!value?.fetchedAt || !Array.isArray(value.models)) return null;
+  if (Date.now() - Date.parse(value.fetchedAt) > CACHE_MAX_AGE_MS) return null;
+  return value;
+}
+
+function writeCache(filePath, discovery) {
+  try {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, JSON.stringify({ ...discovery, fetchedAt: new Date().toISOString() }, null, 2), "utf8");
+  } catch {
+    // Model discovery is best-effort.
+  }
+}
+
+function readJson(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function readClaudeAvailableModels({ productDir }) {
+  const paths = [
+    path.join(os.homedir(), ".claude", "settings.json"),
+    path.join(productDir, ".claude", "settings.json"),
+  ];
+  const models = [];
+  for (const filePath of paths) {
+    const settings = readJson(filePath);
+    const available = settings?.availableModels || settings?.model?.availableModels;
+    if (Array.isArray(available)) models.push(...available.map((entry) => String(entry).trim()).filter(Boolean));
+  }
+  return [...new Set(models)];
+}
+
+async function fetchAnthropicModels({ env }) {
+  const apiKey = env.ANTHROPIC_API_KEY;
+  if (!apiKey || typeof fetch !== "function") return [];
+  try {
+    const response = await fetch("https://api.anthropic.com/v1/models", {
+      headers: {
+        "x-api-key": apiKey,
+        "anthropic-version": "2023-06-01",
+      },
+    });
+    if (!response.ok) return [];
+    const body = await response.json();
+    const data = Array.isArray(body?.data) ? body.data : [];
+    return data
+      .filter((entry) => entry?.id)
+      .map((entry) => ({
+        id: String(entry.id),
+        displayName: String(entry.display_name || entry.id),
+        description: entry.created_at ? `Created ${entry.created_at}` : "",
+      }));
+  } catch {
+    return [];
+  }
+}
+
+function dedupeModels(models) {
+  const seen = new Set();
+  const result = [];
+  for (const model of models) {
+    const key = model.id || "__default__";
+    if (seen.has(key)) continue;
+    seen.add(key);
+    result.push(model);
+  }
+  return result;
+}
+
+function shellQuote(value) {
+  const text = String(value);
+  if (/^[a-zA-Z0-9._:/-]+$/.test(text)) return text;
+  return `'${text.replace(/'/g, `'\\''`)}'`;
+}

package/lib/cli/assistant/prompt-builder.mjs

@@ -16,11 +16,8 @@ export function buildAssistantPrompt({
     "You are Testkit Assistant.",
     "You help users run tests, inspect failures, read logs and artifacts, and navigate the current local test state.",
     "All user natural-language requests must be handled through your own reasoning plus the available tools.",
-    "Use shell_exec when the user asks to run tests or inspect the working repo.",
-    "For testkit work, invoke the local `testkit` command directly, for example `testkit run --dir . --type e2e` or `testkit discover --dir .`.",
-    "Do not wrap testkit with pnpm, npm, yarn, bun, or npx unless the user explicitly asks for that exact package-manager command.",
+    "Prefer real repository commands through shell_exec when the user asks to run tests or inspect the working repo.",
     "Use read_context before repeating artifact/log inspection work, and use read_file/search_repo when you need codebase context.",
-    "After a tool result, describe only what the tool result actually says. Do not invent filesystem, sandbox, package-manager, or permission errors.",
     buildAssistantResponseContract({ tools }),
     "",
     "Current run summary:",

package/lib/cli/assistant/session.mjs

@@ -74,7 +74,24 @@ export async function runAssistantConversationTurn({
         emitted.push({ role: "assistant", text: envelope.commentary });
         currentTranscript.push({ role: "assistant", text: envelope.commentary });
       }
-      const toolResult = await executeAssistantTool(envelope.tool, envelope.arguments, toolContext);
+      let toolResult;
+      try {
+        toolResult = await executeAssistantTool(envelope.tool, envelope.arguments, toolContext);
+      } catch (error) {
+        const toolText = formatToolError(envelope.tool, error);
+        emitted.push({
+          role: "tool",
+          text: toolText,
+          toolName: envelope.tool,
+          title: `${envelope.tool} error`,
+          data: { ok: false, error: toolText },
+        });
+        currentTranscript.push({
+          role: "tool",
+          text: `${envelope.tool}: ${toolText}`,
+        });
+        continue;
+      }
       const toolText = toolResult.text || `${envelope.tool} completed`;
       emitted.push({
         role: "tool",
@@ -105,6 +122,14 @@ export async function runAssistantConversationTurn({
   return emitted;
 }
 
+export function formatToolError(tool, error) {
+  const message = error instanceof Error ? error.message : String(error);
+  if (tool === "shell_exec" && /command string/.test(message)) {
+    return "The assistant requested shell_exec without a command. Retry with arguments.command set to the exact shell command.";
+  }
+  return `Tool failed: ${message}`;
+}
+
 function formatProviderEvent(event) {
   if (event.type === "tool") {
     return `${event.provider}: ${event.name}${event.detail ? ` (${event.detail})` : ""}`;

package/lib/cli/assistant/slash-commands.mjs

@@ -23,8 +23,14 @@ export function parseSlashCommand(input) {
   }
 
   if (command === "model") {
+    if (tokens.length === 0) return { type: "model-list" };
+    if (tokens[0] === "list") return { type: "model-list" };
+    if (tokens[0] === "custom") {
+      const customModel = tokens.slice(1).join(" ").trim();
+      if (!customModel) throw new Error("/model custom expects a model name");
+      return { type: "model", model: customModel, custom: true };
+    }
     const model = tokens.join(" ").trim();
-    if (!model) throw new Error("/model expects a model name or default");
     return { type: "model", model: model === "default" ? null : model };
   }
 
@@ -121,7 +127,7 @@ export function formatSlashHelpLines() {
     "/status",
     "/doctor",
     "/provider <auto|claude|codex>",
-    "/model <model|default>",
+    "/model [list|default|custom <model>|model]",
     "/effort <low|medium|high|xhigh|max|default>",
     "/provider-arg add <arg>",
     "/provider-arg list",