@elench/testkit 0.1.86 → 0.1.88

package/README.md

@@ -11,9 +11,17 @@ The package is now driven by `testkit.config.ts`, not `testkit.config.json`.
 ```bash
 cd my-product
 
-# Run every testkit-managed suite
+# Launch the interactive assistant
 npx @elench/testkit
 
+# Ask for one assistant turn non-interactively
+npx @elench/testkit assistant --message "/status"
+npx @elench/testkit assistant --message "/run e2e --service api"
+npx @elench/testkit assistant --message "Why did the latest failure happen?"
+
+# Run every testkit-managed suite in batch mode
+npx @elench/testkit run
+
 # Inspect discovered tests without running them
 npx @elench/testkit discover
 npx @elench/testkit discover --output-mode verbose
@@ -55,12 +63,10 @@ npx @elench/testkit status
 npx @elench/testkit destroy
 npx @elench/testkit cleanup
 
-# Inspect the latest run artifact
-npx @elench/testkit show
-npx @elench/testkit show __testkit__/health/health.int.testkit.ts
-npx @elench/testkit artifacts __testkit__/health/health.int.testkit.ts
-npx @elench/testkit logs __testkit__/health/health.int.testkit.ts
-npx @elench/testkit watch
+# Inspect the latest run artifact through the assistant
+npx @elench/testkit assistant --message '/inspect "__testkit__/health/health.int.testkit.ts"'
+npx @elench/testkit assistant --pane artifacts --message '/inspect "__testkit__/health/health.int.testkit.ts"'
+npx @elench/testkit assistant --pane logs --message '/inspect "__testkit__/health/health.int.testkit.ts"'
 
 # Automatic regression intelligence
 # Configure testkit.regressions.json and testkit classifies new vs known regressions automatically during runs
@@ -69,11 +75,12 @@ npx @elench/testkit watch
 npx @elench/testkit db snapshot capture --service api --output scripts/testkit/schema-baseline.sql
 ```
 
-`testkit` now keeps the default terminal output intentionally short: one line
-per completed file, a concise failure block, and a final summary. Service logs,
-captured runtime output, emitted artifacts, and user-visible LLM responses are
-persisted under `.testkit/results/` and inspected on demand with `show`,
-`artifacts`, `logs`, or `watch`.
+`testkit` is now assistant-first in an interactive TTY. The assistant keeps a
+chat transcript and a live workbench around the current run artifact, selected
+file, logs, setup operations, and emitted artifacts. Batch `run` output stays
+intentionally short: one line per completed file, a concise failure block, and
+a final summary. Service logs, captured runtime output, emitted artifacts, and
+assistant-visible run state are persisted under `.testkit/results/`.
 
 `testkit discover` also maintains a small durable per-test history index at
 `.testkit/history/tests.json`. The index tracks first/last seen timestamps,

package/lib/cli/agents/providers/claude.mjs

@@ -15,7 +15,7 @@ export function startClaudeHostedSession({ cwd, prompt, onEvent, purpose = "inve
     "--include-partial-messages",
   ];
 
-  if (purpose === "investigate") {
+  if (purpose === "investigate" || purpose === "assistant") {
     args.push("--permission-mode", "plan");
   }
 

package/lib/cli/agents/providers/codex.mjs

@@ -16,7 +16,7 @@ export function startCodexHostedSession({ cwd, prompt, onEvent, purpose = "inves
   const outputFile = path.join(tempDir, "final-message.txt");
   const args = ["exec", "--json", "-o", outputFile];
 
-  if (purpose === "investigate") {
+  if (purpose === "investigate" || purpose === "assistant") {
     args.push("-s", "read-only");
   }
 

package/lib/cli/assistant/prompt-builder.mjs

@@ -0,0 +1,78 @@
+import { buildInspectPaneContent } from "../tui/detail-pane.mjs";
+import { buildAssistantResponseContract } from "./protocol.mjs";
+
+export function buildAssistantPrompt({
+  productDir,
+  snapshot,
+  transcript = [],
+  tools = [],
+  userMessage,
+} = {}) {
+  const selectionSummary = buildSelectionSummary(snapshot);
+  const panePreview = buildPanePreview(productDir, snapshot);
+  const summaryRows = snapshot?.summaryData?.rows || [];
+
+  return [
+    "You are Testkit Assistant.",
+    "You help users run tests, inspect failures, read artifacts/logs, and explain the current local test state.",
+    "Prefer using tools when the user is asking testkit to inspect or run something.",
+    buildAssistantResponseContract({ tools }),
+    "",
+    "Current run summary:",
+    ...(summaryRows.length > 0 ? summaryRows.map(([label, value]) => `- ${label}: ${value}`) : ["- No run artifact is currently loaded."]),
+    "",
+    "Current selection:",
+    selectionSummary,
+    "",
+    "Current pane preview:",
+    ...(panePreview.length > 0 ? panePreview : ["(empty)"]),
+    "",
+    "Recent conversation:",
+    ...formatTranscript(transcript),
+    "",
+    `User message: ${String(userMessage || "").trim()}`,
+  ].join("\n");
+}
+
+function buildSelectionSummary(snapshot) {
+  const entry = snapshot?.selectedEntry;
+  if (!entry) return "No selection.";
+  if (entry.kind === "file") {
+    return `File ${entry.filePath} (${entry.status}) in ${entry.serviceName}/${entry.type}.`;
+  }
+  if (entry.kind === "suite") {
+    return `Suite ${entry.suiteName} (${entry.type}) in service ${entry.serviceName}.`;
+  }
+  if (entry.kind === "type") {
+    return `Type ${entry.type} in service ${entry.serviceName}.`;
+  }
+  if (entry.kind === "service") {
+    return `Service ${entry.serviceName}.`;
+  }
+  return entry.label || "Unknown selection.";
+}
+
+function buildPanePreview(productDir, snapshot) {
+  if (!productDir || !snapshot) return [];
+  try {
+    const pane = buildInspectPaneContent({
+      productDir,
+      snapshot,
+      paneMode: snapshot.paneMode || "detail",
+      logTail: 8,
+    });
+    return (pane.lines || []).slice(0, 24).map((line) => `- ${line}`);
+  } catch (error) {
+    return [`(pane unavailable: ${error instanceof Error ? error.message : String(error)})`];
+  }
+}
+
+function formatTranscript(transcript) {
+  const entries = Array.isArray(transcript) ? transcript.slice(-10) : [];
+  if (entries.length === 0) return ["- No prior turns."];
+  return entries.map((entry) => {
+    const role = entry.role || "system";
+    const text = String(entry.text || "").replace(/\s+/g, " ").trim();
+    return `- ${role}: ${text}`;
+  });
+}

package/lib/cli/assistant/protocol.mjs

@@ -0,0 +1,67 @@
+export function buildAssistantResponseContract({ tools = [] } = {}) {
+  return [
+    "Respond with exactly one JSON object and no surrounding commentary.",
+    'Use {"type":"answer","message":"..."} when you can answer directly.',
+    'Use {"type":"tool","tool":"<name>","arguments":{...},"commentary":"..."} when you need testkit to act before you answer.',
+    "Only request one tool at a time.",
+    `Available tools: ${tools.map((tool) => tool.name).join(", ") || "none"}.`,
+  ].join("\n");
+}
+
+export function parseAssistantEnvelope(text) {
+  const raw = String(text || "").trim();
+  if (!raw) {
+    return {
+      type: "answer",
+      message: "",
+    };
+  }
+
+  const candidate = extractJsonObject(raw);
+  if (!candidate) {
+    return {
+      type: "answer",
+      message: raw,
+    };
+  }
+
+  try {
+    const parsed = JSON.parse(candidate);
+    if (parsed?.type === "tool" && parsed.tool) {
+      return {
+        type: "tool",
+        tool: String(parsed.tool),
+        arguments: normalizePlainObject(parsed.arguments),
+        commentary: parsed.commentary ? String(parsed.commentary) : "",
+      };
+    }
+    if (parsed?.type === "answer") {
+      return {
+        type: "answer",
+        message: parsed.message ? String(parsed.message) : "",
+      };
+    }
+  } catch {
+    // Fall through to raw answer.
+  }
+
+  return {
+    type: "answer",
+    message: raw,
+  };
+}
+
+function extractJsonObject(text) {
+  const fencedMatch = text.match(/```json\s*([\s\S]*?)```/i);
+  if (fencedMatch?.[1]) return fencedMatch[1].trim();
+
+  const start = text.indexOf("{");
+  const end = text.lastIndexOf("}");
+  if (start < 0 || end <= start) return null;
+  return text.slice(start, end + 1);
+}
+
+function normalizePlainObject(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+  return value;
+}

package/lib/cli/assistant/session.mjs

@@ -0,0 +1,92 @@
+import { startAgentSession, resolvePreferredProvider } from "../agents/index.mjs";
+import { buildAssistantPrompt } from "./prompt-builder.mjs";
+import { listAssistantTools, executeAssistantTool } from "./tool-registry.mjs";
+import { parseAssistantEnvelope } from "./protocol.mjs";
+
+export async function runAssistantConversationTurn({
+  productDir,
+  inspectState,
+  transcript,
+  userMessage,
+  provider = "auto",
+  configs,
+  onStatus,
+  onToolEvent,
+} = {}) {
+  const tools = listAssistantTools();
+  const toolContext = {
+    productDir,
+    inspectState,
+    configs,
+    onEvent: onToolEvent,
+  };
+
+  let currentTranscript = [...(transcript || []), { role: "user", text: userMessage }];
+  const emitted = [];
+
+  for (let attempt = 0; attempt < 6; attempt += 1) {
+    const snapshot = inspectState.getSnapshot();
+    const prompt = buildAssistantPrompt({
+      productDir,
+      snapshot,
+      transcript: currentTranscript,
+      tools,
+      userMessage,
+    });
+
+    onStatus?.(`Thinking with ${resolvePreferredProvider(provider)}...`);
+    const events = [];
+    const session = startAgentSession({
+      provider,
+      cwd: productDir,
+      prompt,
+      purpose: "assistant",
+      onEvent(event) {
+        events.push(event);
+        if (event.type === "status" || event.type === "tool") onStatus?.(formatProviderEvent(event));
+      },
+    });
+    const result = await session.completion;
+    const envelope = parseAssistantEnvelope(result.finalText || "");
+
+    if (envelope.type === "tool") {
+      if (envelope.commentary) {
+        emitted.push({ role: "assistant", text: envelope.commentary });
+        currentTranscript.push({ role: "assistant", text: envelope.commentary });
+      }
+      const toolResult = await executeAssistantTool(envelope.tool, envelope.arguments, toolContext);
+      const toolText = toolResult.text || `${envelope.tool} completed`;
+      emitted.push({
+        role: "tool",
+        text: toolText,
+        toolName: envelope.tool,
+        data: toolResult.data || null,
+      });
+      currentTranscript.push({
+        role: "tool",
+        text: `${envelope.tool}: ${toolText}`,
+      });
+      onToolEvent?.({ type: "tool-result", tool: envelope.tool, text: toolText });
+      continue;
+    }
+
+    emitted.push({
+      role: "assistant",
+      text: envelope.message || result.finalText || "",
+    });
+    return emitted;
+  }
+
+  emitted.push({
+    role: "assistant",
+    text: "I hit the assistant tool-call limit before reaching a final answer.",
+  });
+  return emitted;
+}
+
+function formatProviderEvent(event) {
+  if (event.type === "tool") {
+    return `${event.provider}: ${event.name}${event.detail ? ` (${event.detail})` : ""}`;
+  }
+  return `${event.provider}: ${event.message}`;
+}

package/lib/cli/assistant/slash-commands.mjs

@@ -0,0 +1,160 @@
+const RUN_TYPES = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
+const PANES = new Set(["detail", "artifacts", "logs", "setup"]);
+const PROVIDERS = new Set(["auto", "claude", "codex"]);
+
+export function parseSlashCommand(input) {
+  const trimmed = String(input || "").trim();
+  if (!trimmed.startsWith("/")) return null;
+  const tokens = tokenizeShellLike(trimmed.slice(1));
+  const command = tokens.shift() || "";
+
+  if (command === "help") return { type: "help" };
+  if (command === "clear") return { type: "clear" };
+  if (command === "quit" || command === "exit") return { type: "quit" };
+
+  if (command === "provider") {
+    const provider = tokens[0] || "auto";
+    if (!PROVIDERS.has(provider)) {
+      throw new Error(`/provider expects one of: ${[...PROVIDERS].join(", ")}`);
+    }
+    return { type: "provider", provider };
+  }
+
+  if (command === "pane") {
+    const pane = tokens[0] || "detail";
+    if (!PANES.has(pane)) {
+      throw new Error(`/pane expects one of: ${[...PANES].join(", ")}`);
+    }
+    return { type: "pane", pane };
+  }
+
+  if (command === "file" || command === "focus") {
+    if (!tokens[0]) throw new Error(`/${command} expects a file path`);
+    return { type: "file", file: tokens.join(" ") };
+  }
+
+  if (command === "service") {
+    if (!tokens[0]) throw new Error("/service expects a service name");
+    return { type: "service", service: tokens[0] };
+  }
+
+  if (command === "inspect") {
+    return {
+      type: "inspect",
+      file: tokens[0] || null,
+    };
+  }
+
+  if (command === "status") return { type: "status" };
+  if (command === "discover") return { type: "discover" };
+  if (command === "doctor") return { type: "doctor" };
+
+  if (command === "run") {
+    return {
+      type: "run",
+      options: parseRunCommandTokens(tokens),
+    };
+  }
+
+  throw new Error(`Unknown slash command "/${command}"`);
+}
+
+export function formatSlashHelpLines() {
+  return [
+    "/run [type] [--service name] [--file path] [--suite selector]",
+    "/file <path>",
+    "/service <name>",
+    "/pane <detail|artifacts|logs|setup>",
+    "/inspect [path]",
+    "/discover",
+    "/status",
+    "/doctor",
+    "/provider <auto|claude|codex>",
+    "/clear",
+    "/quit",
+  ];
+}
+
+function parseRunCommandTokens(tokens) {
+  const options = {
+    type: [],
+    suite: [],
+    file: [],
+    service: null,
+  };
+
+  for (let index = 0; index < tokens.length; index += 1) {
+    const token = tokens[index];
+    if (!token) continue;
+    if (RUN_TYPES.has(token)) {
+      options.type.push(token);
+      continue;
+    }
+    if (token === "--service") {
+      options.service = tokens[index + 1] || null;
+      index += 1;
+      continue;
+    }
+    if (token === "--file") {
+      if (tokens[index + 1]) options.file.push(tokens[index + 1]);
+      index += 1;
+      continue;
+    }
+    if (token === "--suite") {
+      if (tokens[index + 1]) options.suite.push(tokens[index + 1]);
+      index += 1;
+      continue;
+    }
+    if (token === "--type") {
+      if (tokens[index + 1]) {
+        options.type.push(...String(tokens[index + 1]).split(",").map((value) => value.trim()).filter(Boolean));
+      }
+      index += 1;
+      continue;
+    }
+    throw new Error(`Unsupported /run token "${token}"`);
+  }
+
+  return options;
+}
+
+function tokenizeShellLike(input) {
+  const tokens = [];
+  let current = "";
+  let quote = null;
+
+  for (let index = 0; index < input.length; index += 1) {
+    const char = input[index];
+    if (quote) {
+      if (char === quote) {
+        quote = null;
+      } else if (char === "\\" && index + 1 < input.length) {
+        current += input[index + 1];
+        index += 1;
+      } else {
+        current += char;
+      }
+      continue;
+    }
+    if (char === '"' || char === "'") {
+      quote = char;
+      continue;
+    }
+    if (/\s/.test(char)) {
+      if (current) {
+        tokens.push(current);
+        current = "";
+      }
+      continue;
+    }
+    if (char === "\\" && index + 1 < input.length) {
+      current += input[index + 1];
+      index += 1;
+      continue;
+    }
+    current += char;
+  }
+
+  if (current) tokens.push(current);
+  return tokens;
+}