@elench/testkit 0.1.100 → 0.1.102

package/lib/cli/assistant/providers/shared.mjs

@@ -1,29 +1,50 @@
 import fs from "fs";
 import readline from "readline";
+import {
+  providerError,
+  providerEvent,
+  providerSessionEnd,
+  providerStatus,
+  providerToolStart,
+} from "./events.mjs";
 
-export function createHostedSessionRunner({ provider, child, onEvent, parsePayload, readFinalText, shouldIgnoreStatus } = {}) {
+export function createHostedSessionRunner({
+  provider,
+  child,
+  onEvent,
+  onRawLine,
+  parsePayload,
+  readFinalText,
+  shouldIgnoreStatus,
+} = {}) {
   let cancelled = false;
   let settled = false;
   let assistantText = "";
+  let finalText = null;
   let lastErrorMessage = null;
 
   const emit = (event) => {
-    if (event?.type === "delta" || event?.type === "final") {
+    if (!event) return;
+    if (event.type === "assistant-delta") {
       assistantText += event.text || "";
     }
-    if (event?.type === "error") {
-      lastErrorMessage = event.message || lastErrorMessage;
+    if (event.type === "assistant-final") {
+      finalText = event.text || finalText;
+    }
+    if (event.type === "error") {
+      lastErrorMessage = event.text || lastErrorMessage;
     }
     if (typeof onEvent === "function" && event) onEvent({ provider, ...event });
   };
 
-  emit({ type: "start" });
+  emit(providerEvent("session-start"));
 
   const stdoutReader = readline.createInterface({ input: child.stdout });
   stdoutReader.on("line", (line) => {
+    onRawLine?.({ provider, stream: "stdout", line });
     const parsed = tryParseJson(line);
     if (parsed == null) {
-      emit({ type: "status", message: line });
+      emit(providerStatus(line));
       return;
     }
     const events = parsePayload ? parsePayload(parsed) : [];
@@ -33,27 +54,31 @@ export function createHostedSessionRunner({ provider, child, onEvent, parsePaylo
 
   const stderrReader = readline.createInterface({ input: child.stderr });
   stderrReader.on("line", (line) => {
+    onRawLine?.({ provider, stream: "stderr", line });
     if (shouldIgnoreStatus?.(line)) return;
-    emit({ type: "status", message: line });
+    emit(providerStatus(line, { stream: "stderr" }));
   });
 
   const completion = (async () => {
     const result = await child;
-    const finalText = (readFinalText ? readFinalText(result) : null) || assistantText.trim() || null;
+    const fileFinalText = readFinalText ? readFinalText(result) : null;
+    const resolvedFinalText = fileFinalText || finalText || assistantText.trim() || null;
     if ((result.exitCode ?? 0) !== 0) {
       const message = lastErrorMessage || result.stderr || `${provider} exited with code ${result.exitCode ?? 1}`;
-      emit({ type: "error", message });
+      emit(providerError(message));
       throw new Error(message);
     }
-    if (finalText) emit({ type: "final", text: finalText });
-    emit({ type: "exit", code: result.exitCode ?? 0 });
+    if (resolvedFinalText && resolvedFinalText !== finalText) {
+      emit(providerEvent("assistant-final", { text: resolvedFinalText }));
+    }
+    emit(providerSessionEnd({ exitCode: result.exitCode ?? 0 }));
     settled = true;
     return {
       provider,
       exitCode: result.exitCode ?? 0,
       stdout: result.stdout || "",
       stderr: result.stderr || "",
-      finalText: finalText || result.stdout || "",
+      finalText: resolvedFinalText || result.stdout || "",
       cancelled,
     };
   })();
@@ -84,21 +109,15 @@ export function tryParseJson(line) {
 
 export function buildToolEvent(name, detail = null) {
   if (!name) return null;
-  return {
-    type: "tool",
-    name: String(name),
-    ...(detail ? { detail: String(detail) } : {}),
-  };
+  return providerToolStart(name, detail ? { detail: String(detail) } : {});
 }
 
 export function buildStatusEvent(message) {
-  if (!message) return null;
-  return { type: "status", message: String(message) };
+  return providerStatus(message);
 }
 
 export function buildErrorEvent(message) {
-  if (!message) return null;
-  return { type: "error", message: String(message) };
+  return providerError(message);
 }
 
 export function extractTextFragments(payload, fragments = [], depth = 0) {

package/lib/cli/assistant/session.mjs

@@ -1,3 +1,5 @@
+import fs from "fs";
+import path from "path";
 import { startProviderSession, resolvePreferredProvider } from "./providers/index.mjs";
 import { buildAssistantPrompt } from "./prompt-builder.mjs";
 import { createAssistantCommandObserver } from "./command-observer.mjs";
@@ -14,6 +16,7 @@ export async function runAssistantConversationTurn({
   commandLog,
   onStatus,
   onToolEvent,
+  onProviderEvent,
   onResolvedProvider,
   onPrompt,
 } = {}) {
@@ -36,6 +39,12 @@ export async function runAssistantConversationTurn({
   const runtimeSettings = settings || { provider };
   const resolvedProvider = resolvePreferredProvider(runtimeSettings.provider || provider, env);
   const providerEnv = commandLog?.providerEnv?.(env) || env;
+  const tracePath = shouldTraceProviderEvents(env, providerEnv)
+    ? path.join(commandLog?.contextDir || path.join(productDir, ".testkit", "assistant"), "provider-events.jsonl")
+    : null;
+  const rawTracePath = shouldTraceProviderEvents(env, providerEnv)
+    ? path.join(commandLog?.contextDir || path.join(productDir, ".testkit", "assistant"), "provider-raw.jsonl")
+    : null;
   onResolvedProvider?.(resolvedProvider);
   onPrompt?.({
     prompt,
@@ -44,6 +53,7 @@ export async function runAssistantConversationTurn({
     effort: runtimeSettings.effort || null,
   });
   onStatus?.(`Thinking with ${resolvedProvider}...`);
+  onProviderEvent?.({ type: "status", provider: resolvedProvider, text: `Thinking with ${resolvedProvider}...` });
 
   observer.start();
   try {
@@ -56,25 +66,53 @@ export async function runAssistantConversationTurn({
       prompt,
       purpose: "assistant",
       env: providerEnv,
+      onRawLine(line) {
+        appendProviderTrace(rawTracePath, line);
+      },
       onEvent(event) {
-        if (event.type === "status" || event.type === "tool") onStatus?.(formatProviderEvent(event));
+        appendProviderTrace(tracePath, event);
+        onProviderEvent?.(event);
+        if (event.type === "status" || event.type === "tool-start" || event.type === "tool-update" || event.type === "tool-end") {
+          onStatus?.(formatProviderEvent(event));
+        }
       },
     });
-    const result = await session.completion;
+    await session.completion;
     observer.scan();
     commandLog?.refresh?.();
-    return [{
-      role: "assistant",
-      text: result.finalText || "",
-    }];
+    return { provider: resolvedProvider };
   } finally {
     observer.stop();
   }
 }
 
 function formatProviderEvent(event) {
-  if (event.type === "tool") {
+  if (event.type === "tool-start") {
     return `${event.provider}: ${event.name}${event.detail ? ` (${event.detail})` : ""}`;
   }
-  return `${event.provider}: ${event.message}`;
+  if (event.type === "tool-update") {
+    return `${event.provider}: ${event.name || "tool"}${event.text ? ` (${event.text})` : ""}`;
+  }
+  if (event.type === "tool-end") {
+    return `${event.provider}: ${event.name || "tool"} ${event.status || "done"}`;
+  }
+  return `${event.provider}: ${event.text || "working"}`;
+}
+
+function shouldTraceProviderEvents(...envs) {
+  return envs.some((entry) => String(entry?.TESTKIT_PROVIDER_TRACE || "").trim() === "1");
+}
+
+function appendProviderTrace(tracePath, event) {
+  if (!tracePath || !event) return;
+  try {
+    fs.mkdirSync(path.dirname(tracePath), { recursive: true });
+    fs.appendFileSync(
+      tracePath,
+      `${JSON.stringify({ timestamp: new Date().toISOString(), ...event })}\n`,
+      "utf8"
+    );
+  } catch {
+    // Provider tracing must never interfere with the assistant turn.
+  }
 }

package/lib/cli/assistant/settings.mjs

@@ -1,7 +1,8 @@
 import fs from "fs";
 import path from "path";
+import { HOSTED_ASSISTANT_PROVIDERS } from "./providers/index.mjs";
 
-export const ASSISTANT_PROVIDERS = ["auto", "codex", "claude"];
+export const ASSISTANT_PROVIDERS = ["auto", ...HOSTED_ASSISTANT_PROVIDERS];
 export const ASSISTANT_EFFORTS = ["low", "medium", "high", "xhigh", "max"];
 
 export const DEFAULT_ASSISTANT_SETTINGS = Object.freeze({
@@ -19,7 +20,11 @@ export function loadAssistantSettings(productDir) {
   const filePath = assistantSettingsPath(productDir);
   try {
     const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
-    return normalizeAssistantSettings(parsed);
+    const normalized = normalizeAssistantSettings(parsed);
+    if (JSON.stringify(parsed) !== JSON.stringify(normalized)) {
+      writeAssistantSettingsFile(filePath, normalized);
+    }
+    return normalized;
   } catch {
     return { ...DEFAULT_ASSISTANT_SETTINGS };
   }
@@ -27,8 +32,7 @@ export function loadAssistantSettings(productDir) {
 
 export function saveAssistantSettings(productDir, settings) {
   const filePath = assistantSettingsPath(productDir);
-  fs.mkdirSync(path.dirname(filePath), { recursive: true });
-  fs.writeFileSync(filePath, `${JSON.stringify(normalizeAssistantSettings(settings), null, 2)}\n`);
+  writeAssistantSettingsFile(filePath, normalizeAssistantSettings(settings));
 }
 
 export function resetAssistantSettings(productDir) {
@@ -52,8 +56,9 @@ export function mergeAssistantSettings(...settingsObjects) {
 
 export function normalizeAssistantSettings(value = {}) {
   const provider = normalizeProvider(value.provider);
-  const effort = normalizeEffort(value.effort);
-  const model = normalizeOptionalString(value.model);
+  const extracted = extractEffortFromModel(value.model);
+  const effort = normalizeEffort(value.effort || extracted.effort);
+  const model = extracted.model;
   const providerArgs = Array.isArray(value.providerArgs)
     ? value.providerArgs.map((entry) => normalizeOptionalString(entry)).filter(Boolean)
     : [];
@@ -89,6 +94,19 @@ export function normalizeOptionalString(value) {
   return stringValue || null;
 }
 
+export function extractEffortFromModel(value) {
+  const model = normalizeOptionalString(value);
+  if (!model) return { model: null, effort: null };
+  const tokens = model.split(/\s+/);
+  if (tokens.length < 2) return { model, effort: null };
+  const trailingEffort = tokens.at(-1);
+  if (!ASSISTANT_EFFORTS.includes(trailingEffort)) return { model, effort: null };
+  return {
+    model: tokens.slice(0, -1).join(" ").trim() || null,
+    effort: trailingEffort,
+  };
+}
+
 function dropNullishSettings(settings) {
   const result = {};
   for (const [key, value] of Object.entries(settings)) {
@@ -96,3 +114,8 @@ function dropNullishSettings(settings) {
   }
   return result;
 }
+
+function writeAssistantSettingsFile(filePath, settings) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, `${JSON.stringify(settings, null, 2)}\n`);
+}

package/lib/cli/assistant/state.mjs

@@ -99,11 +99,25 @@ export function createAssistantState({
   }
 
   function appendMessage(message) {
-    messages.push({
+    const entry = {
       id: `msg-${messages.length + 1}`,
       ...message,
-    });
+    };
+    messages.push(entry);
+    notify();
+    return entry.id;
+  }
+
+  function updateMessage(id, updater) {
+    const index = messages.findIndex((message) => message.id === id);
+    if (index < 0) return false;
+    const next = updater(messages[index]);
+    messages[index] = {
+      ...messages[index],
+      ...next,
+    };
     notify();
+    return true;
   }
 
   function setBusy(nextBusy, status = null) {
@@ -354,10 +368,11 @@ export function createAssistantState({
 
       try {
         setBusy(true, `Thinking with ${settings.provider === "auto" ? "provider" : settings.provider}...`);
-        const emitted = await runAssistantConversationTurn({
+        const providerTurn = createProviderTurnState();
+        await runAssistantConversationTurn({
           productDir,
           runState,
-          transcript: messages.map((entry) => ({ role: entry.role, text: entry.text })),
+          transcript: buildConversationTranscript(messages),
           userMessage: trimmed,
           settings,
           env,
@@ -379,11 +394,20 @@ export function createAssistantState({
             });
             notify();
           },
+          onProviderEvent(event) {
+            handleProviderEvent(providerTurn, event, {
+              appendMessage,
+              updateMessage,
+              setStatus(status) {
+                activeStatus = status;
+                notify();
+              },
+            });
+          },
           onToolEvent(event) {
             handleAssistantToolEvent(state, event, appendMessage);
           },
         });
-        for (const message of emitted) appendMessage(message);
       } catch (error) {
         appendMessage({
           role: "system",
@@ -470,7 +494,13 @@ async function executeSlashCommand({
   }
   if (slash.type === "model") {
     state.setModel(slash.model, { custom: slash.custom });
-    appendMessage({ role: "assistant", text: slash.model ? `Model set to ${slash.model}.` : "Model reset to provider default." });
+    const snapshot = state.getSnapshot();
+    appendMessage({
+      role: "assistant",
+      text: slash.model
+        ? `Model set to ${snapshot.model}${snapshot.effort ? ` with ${snapshot.effort} effort` : ""}.`
+        : "Model reset to provider default.",
+    });
     return;
   }
   if (slash.type === "model-list") {
@@ -602,6 +632,145 @@ function handleAssistantToolEvent(state, event, appendMessage) {
   }
 }
 
+function createProviderTurnState() {
+  return {
+    assistantMessageId: null,
+    lastActivityText: null,
+  };
+}
+
+function handleProviderEvent(turn, event, { appendMessage, updateMessage, setStatus } = {}) {
+  if (!event) return;
+  if (event.type === "assistant-delta") {
+    appendAssistantDelta(turn, event, { appendMessage, updateMessage });
+    setStatus?.(`${event.provider || "provider"} responding`);
+    return;
+  }
+  if (event.type === "assistant-final") {
+    finalizeAssistantMessage(turn, event, { appendMessage, updateMessage });
+    setStatus?.(`${event.provider || "provider"} complete`);
+    return;
+  }
+  if (event.type === "session-start") {
+    appendProviderActivity(turn, {
+      role: "provider-activity",
+      title: formatProviderName(event.provider),
+      text: "Session started",
+      data: event,
+    }, { appendMessage, setStatus });
+    return;
+  }
+  if (event.type === "session-end") {
+    appendProviderActivity(turn, {
+      role: "provider-activity",
+      title: formatProviderName(event.provider),
+      text: Number.isInteger(event.exitCode) ? `Session ended with exit code ${event.exitCode}` : "Session ended",
+      data: event,
+    }, { appendMessage, setStatus });
+    return;
+  }
+  if (event.type === "status") {
+    appendProviderActivity(turn, {
+      role: "provider-activity",
+      title: formatProviderName(event.provider),
+      text: event.text || "Working",
+      data: event,
+    }, { appendMessage, setStatus });
+    return;
+  }
+  if (event.type === "tool-start" || event.type === "tool-update" || event.type === "tool-end") {
+    appendProviderActivity(turn, {
+      role: "provider-tool",
+      title: formatProviderToolTitle(event),
+      text: formatProviderToolText(event),
+      data: event,
+    }, { appendMessage, setStatus });
+    return;
+  }
+  if (event.type === "error") {
+    appendMessage({
+      role: "provider-error",
+      title: formatProviderName(event.provider),
+      text: event.text || "Provider error",
+      data: event,
+    });
+    setStatus?.(`${event.provider || "provider"} error`);
+  }
+}
+
+function appendAssistantDelta(turn, event, { appendMessage, updateMessage }) {
+  if (!turn.assistantMessageId) {
+    turn.assistantMessageId = appendMessage({
+      role: "assistant",
+      status: "streaming",
+      provider: event.provider || null,
+      text: "",
+    });
+  }
+  updateMessage(turn.assistantMessageId, (message) => ({
+    text: `${message.text || ""}${event.text || ""}`,
+    status: "streaming",
+  }));
+}
+
+function finalizeAssistantMessage(turn, event, { appendMessage, updateMessage }) {
+  const finalText = event.text || "";
+  if (!turn.assistantMessageId) {
+    turn.assistantMessageId = appendMessage({
+      role: "assistant",
+      provider: event.provider || null,
+      text: finalText,
+    });
+    return;
+  }
+  updateMessage(turn.assistantMessageId, (message) => ({
+    text: finalText || message.text || "",
+    status: null,
+  }));
+}
+
+function appendProviderActivity(turn, message, { appendMessage, setStatus }) {
+  const text = String(message.text || "").trim();
+  if (!text) return;
+  const signature = `${message.role}:${message.title || ""}:${text}`;
+  if (turn.lastActivityText === signature) return;
+  turn.lastActivityText = signature;
+  appendMessage(message);
+  setStatus?.(message.title ? `${message.title}: ${text}` : text);
+}
+
+function formatProviderName(provider) {
+  return provider ? String(provider) : "provider";
+}
+
+function formatProviderToolTitle(event) {
+  const provider = formatProviderName(event.provider);
+  const name = event.name || "tool";
+  if (event.type === "tool-end") return `${provider} finished ${name}`;
+  if (event.type === "tool-update") return `${provider} updated ${name}`;
+  return `${provider} started ${name}`;
+}
+
+function formatProviderToolText(event) {
+  const lines = [];
+  if (event.detail) lines.push(String(event.detail));
+  if (event.text) lines.push(String(event.text));
+  if (event.input) lines.push(formatProviderData("input", event.input));
+  if (event.output) lines.push(formatProviderData("output", event.output));
+  if (event.status) lines.push(`status: ${event.status}`);
+  return lines.filter(Boolean).join("\n") || event.name || "Provider tool activity";
+}
+
+function formatProviderData(label, value) {
+  if (value == null) return null;
+  if (typeof value === "string") return `${label}: ${value}`;
+  try {
+    return `${label}: ${JSON.stringify(value)}`;
+  } catch {
+    return `${label}: ${String(value)}`;
+  }
+}
+
 function formatSettings(snapshot) {
   const rows = [
     ["Provider", snapshot.provider || "auto"],
@@ -648,6 +817,12 @@ function serializeRunSession(session) {
   };
 }
 
+function buildConversationTranscript(messages) {
+  return (messages || [])
+    .filter((entry) => !["provider-activity", "provider-tool", "provider-error"].includes(entry.role))
+    .map((entry) => ({ role: entry.role, text: entry.text }));
+}
+
 function formatObservedCommandTitle(command) {
   const kind = command?.kind || "command";
   return `testkit ${kind}`;

package/lib/cli/assistant/transcript-text.mjs

@@ -0,0 +1,35 @@
+import stripAnsi from "strip-ansi";
+import { buildAssistantViewModel } from "./view-model.mjs";
+import { renderMarkdownToAnsi } from "./markdown-block.mjs";
+
+export function renderAssistantSnapshotText(snapshot, { cwd = process.cwd(), ansi = false } = {}) {
+  const view = buildAssistantViewModel(snapshot || {}, { cwd });
+  const lines = [view.title, view.welcome.rows.find(([label]) => label === "Provider")?.[1] || ""]
+    .filter(Boolean);
+  for (const block of view.blocks || []) {
+    lines.push("");
+    lines.push(...renderBlockLines(block, { ansi }));
+  }
+  lines.push("");
+  lines.push(view.statusLine);
+  const text = `${lines.join("\n").trimEnd()}\n`;
+  return ansi ? text : stripAnsi(text);
+}
+
+function renderBlockLines(block, { ansi = false } = {}) {
+  const marker = block.marker || "";
+  const title = block.title ? ` ${block.title}` : "";
+  const text = String(block.text || "").trimEnd();
+  if (block.format === "markdown") {
+    const rendered = text ? renderMarkdownToAnsi(text) : "";
+    const normalized = ansi ? rendered : stripAnsi(rendered);
+    return prefixLines(`${marker}${title}`, normalized);
+  }
+  return prefixLines(`${marker}${title}`, text);
+}
+
+function prefixLines(prefix, text) {
+  const lines = String(text || "").split(/\r?\n/);
+  if (lines.length === 0 || (lines.length === 1 && lines[0] === "")) return [prefix];
+  return lines.map((line, index) => (index === 0 ? `${prefix}${prefix && line ? " " : ""}${line}` : `  ${line}`));
+}

package/lib/cli/assistant/view-model.mjs

@@ -65,6 +65,17 @@ export function buildTranscriptBlocks(messages) {
         exitCode: message.data?.exitCode ?? null,
       };
     }
+    if (role === "provider-activity" || role === "provider-tool" || role === "provider-error") {
+      return {
+        id: message.id,
+        kind: role,
+        format: "plain",
+        marker: role === "provider-error" ? "!" : "◌",
+        title: message.title || null,
+        text: message.text || "",
+        status: message.status || null,
+      };
+    }
     if (role === "user") {
       return {
         id: message.id,

package/lib/cli/command-flags.mjs

@@ -32,9 +32,6 @@ export const runFlags = {
   "file-timeout-seconds": Flags.string({
     description: "Per-file wall-clock timeout in seconds",
   }),
-  shard: Flags.string({
-    description: "Run only shard i of n at suite granularity",
-  }),
   seed: Flags.string({
     description: "Deterministic seed for scenario suites",
   }),

package/lib/cli/entrypoint.mjs

@@ -23,7 +23,6 @@ export function normalizeCliArgs(argv) {
     "--file",
     "--workers",
     "--file-timeout-seconds",
-    "--shard",
     "--seed",
     "--input",
     "--output",
@@ -54,7 +53,6 @@ export function normalizeCliArgs(argv) {
       "-f",
       "--workers",
       "--file-timeout-seconds",
-      "--shard",
       "--seed",
       "--write-status",
       "--allow-partial-status",

package/lib/cli/operations/run/operation.mjs

@@ -2,7 +2,6 @@ import * as runner from "../../../runner/index.mjs";
 import { loadManagedConfigs } from "../../../app/configs.mjs";
 import {
   parseFileTimeoutOption,
-  parseShardOption,
   parseSuiteOption,
   parseTypeOption,
   parseWorkersOption,
@@ -25,7 +24,6 @@ export async function buildRunRequest(flags, positionalType = null, cwd = proces
     flags["file-timeout-seconds"] == null
       ? null
       : parseFileTimeoutOption(flags["file-timeout-seconds"]);
-  const shard = parseShardOption(flags.shard);
   const typeValues = parseTypeOption(flags.type, positionalType);
   const suiteSelectors = parseSuiteOption(flags.suite);
   const rawFileNames = Array.isArray(flags.file) ? flags.file : [flags.file].filter(Boolean);
@@ -44,7 +42,6 @@ export async function buildRunRequest(flags, positionalType = null, cwd = proces
       fileNames,
       workers,
       fileTimeoutSeconds,
-      shard,
       scenarioSeed: flags.seed || null,
       serviceFilter: flags.service || null,
       writeStatus: flags["write-status"],

package/lib/runner/live-run.mjs

@@ -4,12 +4,14 @@ import { writeLiveRunArtifact } from "./artifacts.mjs";
 
 export function createLiveSnapshotWriter({
   productDir,
+  runId,
   configs,
   trackers,
   startedAt,
   execution,
   workerState,
   selection,
+  provenance = null,
   metadata,
   logRegistry,
   setupRegistry,
@@ -21,6 +23,7 @@ export function createLiveSnapshotWriter({
       productDir,
       buildLiveRunArtifact({
         productDir,
+        runId,
         results: partialResults,
         startedAt,
         updatedAt: now,
@@ -31,9 +34,10 @@ export function createLiveSnapshotWriter({
         typeValues: selection.typeValues,
         suiteSelectors: selection.suiteSelectors,
         fileNames: selection.fileNames,
-        shard: selection.shard,
         serviceFilter: selection.serviceFilter,
         scenarioSeed: selection.scenarioSeed,
+        planning: selection.planning || null,
+        provenance,
         metadata,
         summarizeDbBackend,
         serviceLogs: logRegistry.listServiceLogs(),