@elench/testkit 0.1.96 → 0.1.98

package/lib/cli/assistant/session.mjs

@@ -1,11 +1,11 @@
-import { startAgentSession, resolvePreferredProvider } from "../agents/index.mjs";
+import { startProviderSession, resolvePreferredProvider } from "./providers/index.mjs";
 import { buildAssistantPrompt } from "./prompt-builder.mjs";
 import { listAssistantTools, executeAssistantTool } from "./tool-registry.mjs";
 import { parseAssistantEnvelope } from "./protocol.mjs";
 
 export async function runAssistantConversationTurn({
   productDir,
-  inspectState,
+  runState,
   transcript,
   userMessage,
   provider = "auto",
@@ -21,7 +21,7 @@ export async function runAssistantConversationTurn({
   const tools = listAssistantTools();
   const toolContext = {
     productDir,
-    inspectState,
+    runState,
     configs,
     env,
     commandLog,
@@ -32,7 +32,7 @@ export async function runAssistantConversationTurn({
   const emitted = [];
 
   for (let attempt = 0; attempt < 6; attempt += 1) {
-    const snapshot = inspectState.getSnapshot();
+    const snapshot = runState.getSnapshot();
     const prompt = buildAssistantPrompt({
       productDir,
       snapshot,
@@ -52,7 +52,7 @@ export async function runAssistantConversationTurn({
     });
     onStatus?.(`Thinking with ${resolvedProvider}...`);
     const events = [];
-    const session = startAgentSession({
+    const session = startProviderSession({
       provider: runtimeSettings.provider || provider,
       model: runtimeSettings.model || null,
       effort: runtimeSettings.effort || null,
@@ -74,7 +74,24 @@ export async function runAssistantConversationTurn({
         emitted.push({ role: "assistant", text: envelope.commentary });
         currentTranscript.push({ role: "assistant", text: envelope.commentary });
       }
-      const toolResult = await executeAssistantTool(envelope.tool, envelope.arguments, toolContext);
+      let toolResult;
+      try {
+        toolResult = await executeAssistantTool(envelope.tool, envelope.arguments, toolContext);
+      } catch (error) {
+        const toolText = formatToolError(envelope.tool, error);
+        emitted.push({
+          role: "tool",
+          text: toolText,
+          toolName: envelope.tool,
+          title: `${envelope.tool} error`,
+          data: { ok: false, error: toolText },
+        });
+        currentTranscript.push({
+          role: "tool",
+          text: `${envelope.tool}: ${toolText}`,
+        });
+        continue;
+      }
       const toolText = toolResult.text || `${envelope.tool} completed`;
       emitted.push({
         role: "tool",
@@ -105,6 +122,14 @@ export async function runAssistantConversationTurn({
   return emitted;
 }
 
+export function formatToolError(tool, error) {
+  const message = error instanceof Error ? error.message : String(error);
+  if (tool === "shell_exec" && /command string/.test(message)) {
+    return "The assistant requested shell_exec without a command. Retry with arguments.command set to the exact shell command.";
+  }
+  return `Tool failed: ${message}`;
+}
+
 function formatProviderEvent(event) {
   if (event.type === "tool") {
     return `${event.provider}: ${event.name}${event.detail ? ` (${event.detail})` : ""}`;

package/lib/cli/assistant/slash-commands.mjs

@@ -1,6 +1,6 @@
-const RUN_TYPES = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
 import { ASSISTANT_EFFORTS, ASSISTANT_PROVIDERS } from "./settings.mjs";
 
+const RUN_TYPES = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
 const PROVIDERS = new Set(ASSISTANT_PROVIDERS);
 const EFFORTS = new Set(ASSISTANT_EFFORTS);
 
@@ -23,8 +23,14 @@ export function parseSlashCommand(input) {
   }
 
   if (command === "model") {
+    if (tokens.length === 0) return { type: "model-list" };
+    if (tokens[0] === "list") return { type: "model-list" };
+    if (tokens[0] === "custom") {
+      const customModel = tokens.slice(1).join(" ").trim();
+      if (!customModel) throw new Error("/model custom expects a model name");
+      return { type: "model", model: customModel, custom: true };
+    }
     const model = tokens.join(" ").trim();
-    if (!model) throw new Error("/model expects a model name or default");
     return { type: "model", model: model === "default" ? null : model };
   }
 
@@ -56,6 +62,17 @@ export function parseSlashCommand(input) {
     throw new Error('/settings expects "show" or "reset"');
   }
 
+  if (command === "config") {
+    const action = tokens[0] || "show";
+    if (action === "show") return { type: "config-show" };
+    if (action === "reset") return { type: "config-reset" };
+    if (action === "auto-collapse-passed" || action === "autoCollapsePassedTreeBranches") {
+      const value = parseBooleanToken(tokens[1], "/config auto-collapse-passed");
+      return { type: "config-set-auto-collapse", value };
+    }
+    throw new Error('/config expects "show", "reset", or "auto-collapse-passed <on|off>"');
+  }
+
   if (command === "file" || command === "focus") {
     if (!tokens[0]) throw new Error(`/${command} expects a file path`);
     return { type: "file", file: tokens.join(" ") };
@@ -121,18 +138,28 @@ export function formatSlashHelpLines() {
     "/status",
     "/doctor",
     "/provider <auto|claude|codex>",
-    "/model <model|default>",
+    "/model [list|default|custom <model>|model]",
     "/effort <low|medium|high|xhigh|max|default>",
     "/provider-arg add <arg>",
     "/provider-arg list",
     "/provider-arg clear",
     "/settings",
     "/settings reset",
+    "/config",
+    "/config auto-collapse-passed <on|off>",
+    "/config reset",
     "/clear",
     "/quit",
   ];
 }
 
+function parseBooleanToken(value, commandName) {
+  const normalized = String(value || "").trim().toLowerCase();
+  if (["on", "true", "yes", "1"].includes(normalized)) return true;
+  if (["off", "false", "no", "0"].includes(normalized)) return false;
+  throw new Error(`${commandName} expects on or off`);
+}
+
 function parseRunCommandTokens(tokens) {
   const options = {
     type: [],

package/lib/cli/assistant/state.mjs

@@ -1,11 +1,23 @@
-import { loadCurrentRunArtifact, loadLatestRunArtifact } from "../viewer.mjs";
-import { createInspectState } from "../tui/inspect-state.mjs";
-import { buildContextSelection } from "../context-resources.mjs";
-import { isProviderInstalled } from "../agents/index.mjs";
+import { loadCurrentRunArtifact, loadLatestRunArtifact } from "../../results/artifacts.mjs";
+import {
+  formatCliConfig,
+  loadCliConfig,
+  mergeCliConfig,
+  resetCliConfig,
+  saveCliConfig,
+} from "../config.mjs";
+import { createRunState } from "../state/run/state.mjs";
+import { buildContextSelection } from "../../results/context.mjs";
+import { isProviderInstalled } from "./providers/index.mjs";
 import { parseSlashCommand, formatSlashHelpLines } from "./slash-commands.mjs";
 import { executeAssistantTool } from "./tool-registry.mjs";
 import { runAssistantConversationTurn } from "./session.mjs";
 import { prepareAssistantContextPack } from "./context-pack.mjs";
+import {
+  discoverAssistantModels,
+  formatModelChoices,
+  getModelProviderMismatch,
+} from "./model-discovery.mjs";
 import {
   DEFAULT_ASSISTANT_SETTINGS,
   loadAssistantSettings,
@@ -36,10 +48,12 @@ export function createAssistantState({
   configs = [],
   env = process.env,
 } = {}) {
-  const inspectState = createInspectState({ dataSource });
+  const runState = createRunState({ dataSource });
+  let cliConfig = loadCliConfig(productDir);
+  runState.setAutoCollapsePassedTreeBranches(cliConfig.autoCollapsePassedTreeBranches);
   const commandLog = prepareAssistantContextPack({
     productDir,
-    inspectState,
+    runState,
   });
 
   const listeners = new Set();
@@ -57,14 +71,25 @@ export function createAssistantState({
     }
   );
   let resolvedProviderName = resolveInitialProvider(settings.provider, env);
+  const sanitizedStartup = sanitizeSettingsForResolvedProvider({
+    productDir,
+    settings,
+    resolvedProvider: resolvedProviderName,
+  });
+  settings = sanitizedStartup.settings;
+  if (sanitizedStartup.notice) notice = sanitizedStartup.notice;
   let activeStatus = null;
+  let startupNoticeEmitted = false;
   let contextUsage = buildContextUsage({
     provider: resolvedProviderName || settings.provider,
     model: settings.model,
     prompt: "",
   });
+  let liveRunSession = null;
+  let lastRunSession = null;
+  let liveRunSessionUnsubscribe = null;
 
-  inspectState.subscribe(() => {
+  runState.subscribe(() => {
     commandLog.refresh();
     notify();
   });
@@ -91,13 +116,46 @@ export function createAssistantState({
     commandLog.refresh();
   }
 
+  function attachRunSession(session, { active = true } = {}) {
+    if (liveRunSessionUnsubscribe) {
+      liveRunSessionUnsubscribe();
+      liveRunSessionUnsubscribe = null;
+    }
+    if (!session) {
+      if (!active) notify();
+      return;
+    }
+    lastRunSession = session;
+    if (active) {
+      liveRunSession = session;
+      liveRunSessionUnsubscribe = session.runState.subscribe(() => {
+        notify();
+      });
+    }
+    notify();
+  }
+
+  function completeRunSession(session) {
+    if (liveRunSession === session) {
+      if (liveRunSessionUnsubscribe) {
+        liveRunSessionUnsubscribe();
+        liveRunSessionUnsubscribe = null;
+      }
+      liveRunSession = null;
+    }
+    if (session) lastRunSession = session;
+    notify();
+  }
+
   const state = {
-    inspectState,
+    runState,
     commandLog,
+    attachRunSession,
+    completeRunSession,
 
     async loadLatestArtifact() {
       try {
-        inspectState.hydrateFromArtifact(loadLatestRunArtifact(productDir));
+        runState.hydrateFromArtifact(loadLatestRunArtifact(productDir));
       } catch {
         // No artifact yet.
       }
@@ -106,7 +164,7 @@ export function createAssistantState({
 
     async loadCurrentArtifact() {
       try {
-        inspectState.hydrateFromArtifact(loadCurrentRunArtifact(productDir));
+        runState.hydrateFromArtifact(loadCurrentRunArtifact(productDir));
       } catch {
         // No artifact yet.
       }
@@ -114,13 +172,13 @@ export function createAssistantState({
     },
 
     revealFile(serviceName, filePath) {
-      const revealed = inspectState.revealFile(serviceName, filePath);
+      const revealed = runState.revealFile(serviceName, filePath);
       refreshContextPack();
       return revealed;
     },
 
     revealService(serviceName) {
-      const revealed = inspectState.revealService(serviceName);
+      const revealed = runState.revealService(serviceName);
       refreshContextPack();
       return revealed;
     },
@@ -176,12 +234,20 @@ export function createAssistantState({
 
     setProvider(nextProvider) {
       settings = mergeAssistantSettings(settings, { provider: nextProvider || DEFAULT_ASSISTANT_SETTINGS.provider });
-      resolvedProviderName = null;
+      resolvedProviderName = resolveInitialProvider(settings.provider, env);
+      if (settings.model && getModelProviderMismatch(resolvedProviderName, settings.model)) {
+        settings = mergeAssistantSettings(settings, { model: null });
+      }
       saveAssistantSettings(productDir, settings);
       notify();
     },
 
-    setModel(nextModel) {
+    setModel(nextModel, { custom = false } = {}) {
+      const resolvedProvider = resolveInitialProvider(settings.provider, env);
+      const mismatch = getModelProviderMismatch(resolvedProvider, nextModel);
+      if (mismatch && !custom) {
+        throw new Error(mismatch);
+      }
       settings = mergeAssistantSettings(settings, { model: nextModel || null });
       saveAssistantSettings(productDir, settings);
       notify();
@@ -207,6 +273,19 @@ export function createAssistantState({
       notify();
     },
 
+    setCliConfig(nextConfig) {
+      cliConfig = mergeCliConfig(cliConfig, nextConfig);
+      saveCliConfig(productDir, cliConfig);
+      runState.setAutoCollapsePassedTreeBranches(cliConfig.autoCollapsePassedTreeBranches);
+      notify();
+    },
+
+    resetCliConfig() {
+      cliConfig = resetCliConfig(productDir);
+      runState.setAutoCollapsePassedTreeBranches(cliConfig.autoCollapsePassedTreeBranches);
+      notify();
+    },
+
     resetSettings() {
       settings = resetAssistantSettings(productDir);
       resolvedProviderName = null;
@@ -218,6 +297,14 @@ export function createAssistantState({
       notify();
     },
 
+    getLiveRunSession() {
+      return liveRunSession;
+    },
+
+    getLastRunSession() {
+      return lastRunSession;
+    },
+
     async submitCurrentComposer() {
       const value = composerState.text.trim();
       composerState = createComposerState();
@@ -229,6 +316,10 @@ export function createAssistantState({
     async submitInput(input) {
       const trimmed = String(input || "").trim();
       if (!trimmed) return;
+      if (notice && !startupNoticeEmitted) {
+        startupNoticeEmitted = true;
+        appendMessage({ role: "system", text: notice });
+      }
       appendMessage({ role: "user", text: trimmed });
 
       const slash = parseSlashCommandSafe(trimmed);
@@ -238,6 +329,7 @@ export function createAssistantState({
       }
       if (slash) {
         try {
+          setBusy(true, `Running ${slash.type}...`);
           await executeSlashCommand({
             slash,
             state,
@@ -252,6 +344,34 @@ export function createAssistantState({
             role: "system",
             text: error instanceof Error ? error.message : String(error),
           });
+        } finally {
+          setBusy(false, null);
+        }
+        refreshContextPack();
+        notify();
+        return;
+      }
+
+      const routedSlash = routeLocalIntent(trimmed);
+      if (routedSlash) {
+        try {
+          setBusy(true, `Running ${routedSlash.type}...`);
+          await executeSlashCommand({
+            slash: routedSlash,
+            state,
+            productDir,
+            settings,
+            configs,
+            env,
+            appendMessage,
+          });
+        } catch (error) {
+          appendMessage({
+            role: "system",
+            text: error instanceof Error ? error.message : String(error),
+          });
+        } finally {
+          setBusy(false, null);
         }
         refreshContextPack();
         notify();
@@ -262,7 +382,7 @@ export function createAssistantState({
         setBusy(true, `Thinking with ${settings.provider === "auto" ? "provider" : settings.provider}...`);
         const emitted = await runAssistantConversationTurn({
           productDir,
-          inspectState,
+          runState,
           transcript: messages.map((entry) => ({ role: entry.role, text: entry.text })),
           userMessage: trimmed,
           settings,
@@ -286,7 +406,7 @@ export function createAssistantState({
             notify();
           },
           onToolEvent(event) {
-            handleAssistantToolEvent(event, appendMessage);
+            handleAssistantToolEvent(state, event, appendMessage);
           },
         });
         for (const message of emitted) appendMessage(message);
@@ -308,8 +428,8 @@ export function createAssistantState({
 
     getSnapshot() {
       return {
-        context: buildContextSelection(inspectState.getSnapshot()),
-        inspect: inspectState.getSnapshot(),
+        context: buildContextSelection(runState.getSnapshot()),
+        run: runState.getSnapshot(),
         productDir,
         messages: [...messages],
         composer: composerState.text,
@@ -321,8 +441,11 @@ export function createAssistantState({
         model: settings.model,
         effort: settings.effort,
         providerArgs: [...settings.providerArgs],
+        cliConfig,
         activeStatus,
         contextUsage,
+        liveRunSession: serializeRunSession(liveRunSession),
+        lastRunSession: serializeRunSession(lastRunSession),
         contextPaths: {
           contextPath: commandLog.contextPath,
           summaryPath: commandLog.summaryPath,
@@ -372,10 +495,17 @@ async function executeSlashCommand({
     return;
   }
   if (slash.type === "model") {
-    state.setModel(slash.model);
+    state.setModel(slash.model, { custom: slash.custom });
     appendMessage({ role: "assistant", text: slash.model ? `Model set to ${slash.model}.` : "Model reset to provider default." });
     return;
   }
+  if (slash.type === "model-list") {
+    const snapshot = state.getSnapshot();
+    const provider = snapshot.resolvedProvider || resolveInitialProvider(snapshot.provider, env) || snapshot.provider;
+    const discovery = await discoverAssistantModels({ provider, productDir, env });
+    appendMessage({ role: "assistant", text: formatModelChoices(discovery, { currentModel: snapshot.model }) });
+    return;
+  }
   if (slash.type === "effort") {
     state.setEffort(slash.effort);
     appendMessage({ role: "assistant", text: slash.effort ? `Effort set to ${slash.effort}.` : "Effort reset to provider default." });
@@ -405,28 +535,32 @@ async function executeSlashCommand({
     appendMessage({ role: "assistant", text: "Assistant settings reset." });
     return;
   }
+  if (slash.type === "config-show") {
+    appendMessage({ role: "assistant", text: formatCliConfig(state.getSnapshot().cliConfig) });
+    return;
+  }
+  if (slash.type === "config-reset") {
+    state.resetCliConfig();
+    appendMessage({ role: "assistant", text: "CLI config reset." });
+    return;
+  }
+  if (slash.type === "config-set-auto-collapse") {
+    state.setCliConfig({ autoCollapsePassedTreeBranches: slash.value });
+    appendMessage({
+      role: "assistant",
+      text: `autoCollapsePassedTreeBranches set to ${slash.value}.`,
+    });
+    return;
+  }
 
   const result = await executeSlashTool(slash, {
     productDir,
-    inspectState: state.inspectState,
+    runState: state.runState,
     configs,
     env,
     commandLog: state.commandLog,
     onEvent(event) {
-      if (event.type === "tool-start") {
-        appendMessage({
-          role: "tool",
-          status: "running",
-          title: event.title || event.tool || "Tool",
-          text: event.message,
-          data: {
-            command: event.command || null,
-            testkitRelated: Boolean(event.testkitRelated),
-          },
-        });
-      } else if (event.type === "tool-status") {
-        state.setNotice(event.message);
-      }
+      handleAssistantToolEvent(state, event, appendMessage);
     },
     provider: settings.provider,
   });
@@ -439,18 +573,32 @@ async function executeSlashCommand({
   });
 }
 
-function handleAssistantToolEvent(event, appendMessage) {
-  if (!event || event.type !== "tool-start") return;
-  appendMessage({
-    role: "tool",
-    status: "running",
-    title: event.title || event.tool || "Tool",
-    text: event.message || "Running tool",
-    data: {
-      command: event.command || null,
-      testkitRelated: Boolean(event.testkitRelated),
-    },
-  });
+function handleAssistantToolEvent(state, event, appendMessage) {
+  if (!event) return;
+  if (event.type === "tool-start") {
+    appendMessage({
+      role: "tool",
+      status: "running",
+      title: event.title || event.tool || "Tool",
+      text: event.message || "Running tool",
+      data: {
+        command: event.command || null,
+        testkitRelated: Boolean(event.testkitRelated),
+      },
+    });
+    return;
+  }
+  if (event.type === "tool-status") {
+    state.setNotice(event.message);
+    return;
+  }
+  if (event.type === "run-session-start") {
+    state.attachRunSession(event.session, { active: true });
+    return;
+  }
+  if (event.type === "run-session-end") {
+    state.completeRunSession(event.session);
+  }
 }
 
 function formatSettings(snapshot) {
@@ -483,39 +631,18 @@ async function executeSlashTool(slash, context) {
     case "service":
       return executeAssistantTool("read_context", { service: slash.service, mode: "detail" }, context);
     case "status":
-      return executeAssistantTool("shell_exec", { command: "testkit status --dir ." }, context);
+      return executeAssistantTool("show_status", {}, context);
     case "discover":
-      return executeAssistantTool("shell_exec", { command: "testkit discover --dir ." }, context);
+      return executeAssistantTool("discover_tests", {}, context);
     case "doctor":
-      return executeAssistantTool("shell_exec", { command: "testkit doctor --dir ." }, context);
+      return executeAssistantTool("run_doctor", {}, context);
     case "run":
-      return executeAssistantTool("shell_exec", { command: buildRunSlashCommand(slash.options) }, context);
+      return executeAssistantTool("run_tests", slash.options, context);
     default:
       throw new Error(`Unsupported slash command "${slash.type}"`);
   }
 }
 
-function buildRunSlashCommand(options = {}) {
-  const parts = ["testkit", "run", "--dir", "."];
-  for (const type of options.type || []) {
-    parts.push("--type", type);
-  }
-  for (const suite of options.suite || []) {
-    parts.push("--suite", suite);
-  }
-  for (const file of options.file || []) {
-    parts.push("--file", file);
-  }
-  if (options.service) parts.push("--service", options.service);
-  return parts.map(shellEscapeArg).join(" ");
-}
-
-function shellEscapeArg(value) {
-  const stringValue = String(value);
-  if (/^[a-zA-Z0-9._:/-]+$/.test(stringValue)) return stringValue;
-  return `'${stringValue.replace(/'/g, `'\\''`)}'`;
-}
-
 function parseSlashCommandSafe(input) {
   try {
     return parseSlashCommand(input);
@@ -526,3 +653,42 @@ function parseSlashCommandSafe(input) {
     };
   }
 }
+
+function sanitizeSettingsForResolvedProvider({ productDir, settings, resolvedProvider }) {
+  const mismatch = getModelProviderMismatch(resolvedProvider, settings.model);
+  if (!mismatch) return { settings, notice: null };
+  const previousModel = settings.model;
+  const sanitized = mergeAssistantSettings(settings, { model: null });
+  saveAssistantSettings(productDir, sanitized);
+  return {
+    settings: sanitized,
+    notice: `Cleared incompatible saved model "${previousModel}" for ${resolvedProvider}.`,
+  };
+}
+
+function routeLocalIntent(input) {
+  const normalized = String(input || "").trim().toLowerCase();
+  const runMatch = normalized.match(/^run\s+(int|e2e|scenario|dal|load|pw|all)(?:\s+tests?)?$/);
+  if (runMatch) {
+    return {
+      type: "run",
+      options: {
+        type: [runMatch[1]],
+        suite: [],
+        file: [],
+        service: null,
+      },
+    };
+  }
+  if (/^(show\s+)?latest\s+summary$/.test(normalized)) return { type: "status" };
+  if (/^list\s+test\s+files$/.test(normalized)) return { type: "discover" };
+  return null;
+}
+
+function serializeRunSession(session) {
+  if (!session) return null;
+  return {
+    productDir: session.productDir,
+    snapshot: session.getSnapshot(),
+  };
+}