@elench/testkit 0.1.96 → 0.1.97

package/lib/cli/assistant/state.mjs

@@ -6,6 +6,11 @@ import { parseSlashCommand, formatSlashHelpLines } from "./slash-commands.mjs";
 import { executeAssistantTool } from "./tool-registry.mjs";
 import { runAssistantConversationTurn } from "./session.mjs";
 import { prepareAssistantContextPack } from "./context-pack.mjs";
+import {
+  discoverAssistantModels,
+  formatModelChoices,
+  getModelProviderMismatch,
+} from "./model-discovery.mjs";
 import {
   DEFAULT_ASSISTANT_SETTINGS,
   loadAssistantSettings,
@@ -57,7 +62,15 @@ export function createAssistantState({
     }
   );
   let resolvedProviderName = resolveInitialProvider(settings.provider, env);
+  const sanitizedStartup = sanitizeSettingsForResolvedProvider({
+    productDir,
+    settings,
+    resolvedProvider: resolvedProviderName,
+  });
+  settings = sanitizedStartup.settings;
+  if (sanitizedStartup.notice) notice = sanitizedStartup.notice;
   let activeStatus = null;
+  let startupNoticeEmitted = false;
   let contextUsage = buildContextUsage({
     provider: resolvedProviderName || settings.provider,
     model: settings.model,
@@ -176,12 +189,20 @@ export function createAssistantState({
 
     setProvider(nextProvider) {
       settings = mergeAssistantSettings(settings, { provider: nextProvider || DEFAULT_ASSISTANT_SETTINGS.provider });
-      resolvedProviderName = null;
+      resolvedProviderName = resolveInitialProvider(settings.provider, env);
+      if (settings.model && getModelProviderMismatch(resolvedProviderName, settings.model)) {
+        settings = mergeAssistantSettings(settings, { model: null });
+      }
       saveAssistantSettings(productDir, settings);
       notify();
     },
 
-    setModel(nextModel) {
+    setModel(nextModel, { custom = false } = {}) {
+      const resolvedProvider = resolveInitialProvider(settings.provider, env);
+      const mismatch = getModelProviderMismatch(resolvedProvider, nextModel);
+      if (mismatch && !custom) {
+        throw new Error(mismatch);
+      }
       settings = mergeAssistantSettings(settings, { model: nextModel || null });
       saveAssistantSettings(productDir, settings);
       notify();
@@ -229,6 +250,10 @@ export function createAssistantState({
     async submitInput(input) {
       const trimmed = String(input || "").trim();
       if (!trimmed) return;
+      if (notice && !startupNoticeEmitted) {
+        startupNoticeEmitted = true;
+        appendMessage({ role: "system", text: notice });
+      }
       appendMessage({ role: "user", text: trimmed });
 
       const slash = parseSlashCommandSafe(trimmed);
@@ -258,6 +283,29 @@ export function createAssistantState({
         return;
       }
 
+      const routedSlash = routeLocalIntent(trimmed);
+      if (routedSlash) {
+        try {
+          await executeSlashCommand({
+            slash: routedSlash,
+            state,
+            productDir,
+            settings,
+            configs,
+            env,
+            appendMessage,
+          });
+        } catch (error) {
+          appendMessage({
+            role: "system",
+            text: error instanceof Error ? error.message : String(error),
+          });
+        }
+        refreshContextPack();
+        notify();
+        return;
+      }
+
       try {
         setBusy(true, `Thinking with ${settings.provider === "auto" ? "provider" : settings.provider}...`);
         const emitted = await runAssistantConversationTurn({
@@ -372,10 +420,17 @@ async function executeSlashCommand({
     return;
   }
   if (slash.type === "model") {
-    state.setModel(slash.model);
+    state.setModel(slash.model, { custom: slash.custom });
     appendMessage({ role: "assistant", text: slash.model ? `Model set to ${slash.model}.` : "Model reset to provider default." });
     return;
   }
+  if (slash.type === "model-list") {
+    const snapshot = state.getSnapshot();
+    const provider = snapshot.resolvedProvider || resolveInitialProvider(snapshot.provider, env) || snapshot.provider;
+    const discovery = await discoverAssistantModels({ provider, productDir, env });
+    appendMessage({ role: "assistant", text: formatModelChoices(discovery, { currentModel: snapshot.model }) });
+    return;
+  }
   if (slash.type === "effort") {
     state.setEffort(slash.effort);
     appendMessage({ role: "assistant", text: slash.effort ? `Effort set to ${slash.effort}.` : "Effort reset to provider default." });
@@ -496,9 +551,16 @@ async function executeSlashTool(slash, context) {
 }
 
 function buildRunSlashCommand(options = {}) {
-  const parts = ["testkit", "run", "--dir", "."];
-  for (const type of options.type || []) {
-    parts.push("--type", type);
+  const types = options.type || [];
+  const parts = ["testkit", "run"];
+  if (types.length === 1) {
+    parts.push(types[0]);
+  }
+  parts.push("--dir", ".");
+  if (types.length !== 1) {
+    for (const type of types) {
+      parts.push("--type", type);
+    }
   }
   for (const suite of options.suite || []) {
     parts.push("--suite", suite);
@@ -526,3 +588,34 @@ function parseSlashCommandSafe(input) {
     };
   }
 }
+
+function sanitizeSettingsForResolvedProvider({ productDir, settings, resolvedProvider }) {
+  const mismatch = getModelProviderMismatch(resolvedProvider, settings.model);
+  if (!mismatch) return { settings, notice: null };
+  const previousModel = settings.model;
+  const sanitized = mergeAssistantSettings(settings, { model: null });
+  saveAssistantSettings(productDir, sanitized);
+  return {
+    settings: sanitized,
+    notice: `Cleared incompatible saved model "${previousModel}" for ${resolvedProvider}.`,
+  };
+}
+
+function routeLocalIntent(input) {
+  const normalized = String(input || "").trim().toLowerCase();
+  const runMatch = normalized.match(/^run\s+(int|e2e|scenario|dal|load|pw|all)(?:\s+tests?)?$/);
+  if (runMatch) {
+    return {
+      type: "run",
+      options: {
+        type: [runMatch[1]],
+        suite: [],
+        file: [],
+        service: null,
+      },
+    };
+  }
+  if (/^(show\s+)?latest\s+summary$/.test(normalized)) return { type: "status" };
+  if (/^list\s+test\s+files$/.test(normalized)) return { type: "discover" };
+  return null;
+}

package/lib/cli/assistant/tool-registry.mjs

@@ -5,7 +5,6 @@ import { loadCurrentRunArtifact, loadLatestRunArtifact, resolveFileSubject } fro
 import {
   readContextContent,
 } from "../context-resources.mjs";
-import { extractShellCommand, planShellCommand } from "./command-plan.mjs";
 
 const COMMAND_OUTPUT_LIMIT = 14_000;
 const COMMAND_LINE_LIMIT = 220;
@@ -15,7 +14,7 @@ export function listAssistantTools() {
   return [
     {
       name: "shell_exec",
-      description: "Execute a shell command inside the repository. Use local testkit commands for testkit work.",
+      description: "Execute a shell command inside the repository. Prefer real repo commands such as npm, npx, and testkit.",
     },
     {
       name: "read_context",
@@ -49,10 +48,10 @@ export async function executeAssistantTool(name, argumentsObject, context) {
 }
 
 async function shellExecTool(args, context) {
-  const command = extractShellCommand(args).trim();
+  const command = String(args.command || "").trim();
   if (!command) throw new Error("shell_exec requires a command string");
 
-  const shellCommand = planShellCommand(command);
+  const shellCommand = classifyShellCommand(command);
   const commandId = `cmd-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
   context.commandLog?.appendCommandLog({
     type: "command_start",
@@ -60,22 +59,17 @@ async function shellExecTool(args, context) {
     commandId,
     cwd: context.productDir,
     raw: command,
-    executable: shellCommand.executableCommand,
-    normalized: shellCommand.normalized,
   });
   context.onEvent?.({
     type: "tool-start",
     tool: "shell_exec",
-    command: shellCommand.executableCommand,
-    rawCommand: command,
+    command,
     title: shellCommand.title,
     testkitRelated: shellCommand.testkitRelated,
-    message: shellCommand.normalized
-      ? `Running ${shellCommand.displayCommand} (${shellCommand.normalizationReason})`
-      : `Running ${shellCommand.displayCommand}`,
+    message: `Running ${shellCommand.display}`,
   });
 
-  const result = await execaCommand(shellCommand.executableCommand, {
+  const result = await execaCommand(command, {
     cwd: context.productDir,
     reject: false,
     shell: true,
@@ -93,21 +87,18 @@ async function shellExecTool(args, context) {
     commandId,
     cwd: context.productDir,
     raw: command,
-    executable: shellCommand.executableCommand,
-    normalized: shellCommand.normalized,
     code: result.exitCode ?? 0,
     signal: result.signal ?? null,
   });
   context.onEvent?.({
     type: "tool-exit",
     tool: "shell_exec",
-    command: shellCommand.executableCommand,
-    rawCommand: command,
+    command,
     title: shellCommand.title,
     testkitRelated: shellCommand.testkitRelated,
     code: result.exitCode ?? 0,
     signal: result.signal ?? null,
-    message: `${shellCommand.displayCommand} exited ${result.exitCode ?? 0}`,
+    message: `${shellCommand.display} exited ${result.exitCode ?? 0}`,
   });
 
   if (shellCommand.testkitRelated) {
@@ -115,15 +106,13 @@ async function shellExecTool(args, context) {
   }
   context.commandLog?.refresh?.();
 
-  const lines = formatCommandResult(result, shellCommand);
+  const lines = formatCommandResult(command, result, shellCommand);
   return {
     ok: (result.exitCode ?? 0) === 0,
     title: shellCommand.title,
     text: lines.join("\n"),
     data: {
       command,
-      executableCommand: shellCommand.executableCommand,
-      normalizedCommand: shellCommand.normalized,
       stdout: result.stdout || "",
       stderr: result.stderr || "",
       exitCode: result.exitCode ?? 0,
@@ -227,11 +216,42 @@ async function searchRepoTool(args, context) {
   };
 }
 
-function formatCommandResult(result, shellCommand) {
-  const lines = [`$ ${shellCommand.displayCommand}`];
-  if (shellCommand.normalized) {
-    lines.push(`normalized from: ${shellCommand.rawCommand}`);
+function classifyShellCommand(command) {
+  const normalized = command.trim();
+  if (/^(testkit)\b/.test(normalized)) {
+    return {
+      command: "testkit",
+      display: normalized,
+      title: "testkit command",
+      testkitRelated: true,
+    };
   }
+  if (/^(npx)\s+testkit\b/.test(normalized)) {
+    return {
+      command: "npx testkit",
+      display: normalized,
+      title: "npx testkit",
+      testkitRelated: true,
+    };
+  }
+  if (/^(npm)\s+run\s+testkit\b/.test(normalized) || /^(npm)\s+run\s+testkit:/.test(normalized)) {
+    return {
+      command: "npm run testkit",
+      display: normalized,
+      title: "npm testkit script",
+      testkitRelated: true,
+    };
+  }
+  return {
+    command: normalized.split(/\s+/)[0] || "command",
+    display: normalized,
+    title: "Shell command",
+    testkitRelated: false,
+  };
+}
+
+function formatCommandResult(command, result, shellCommand) {
+  const lines = [`$ ${command}`];
   const stdout = (result.stdout || "").trim();
   const stderr = (result.stderr || "").trim();
   const merged = [];

package/lib/database/template-steps.mjs

@@ -1,7 +1,7 @@
 import fs from "fs";
 import path from "path";
 import { execa } from "execa";
-import { buildExecutionEnv } from "../runner/template.mjs";
+import { buildTemplateExecutionEnv } from "../runner/template.mjs";
 import {
   collectConfiguredInputs,
   runConfiguredSteps,
@@ -13,7 +13,7 @@ export async function runTemplateStage(config, stageName, databaseUrl, options =
   if (steps.length === 0) return;
 
   const env = {
-    ...buildExecutionEnv(config, {}, process.env),
+    ...buildTemplateExecutionEnv(config, {}, process.env),
     DATABASE_URL: databaseUrl,
   };
 
@@ -54,7 +54,7 @@ export async function captureTemplateSnapshot(config, outputPath, databaseUrl, o
     {
       cwd: config.productDir,
       env: {
-        ...buildExecutionEnv(config, {}, process.env),
+        ...buildTemplateExecutionEnv(config, {}, process.env),
         DATABASE_URL: templateDbUrl,
       },
       stdout: "pipe",

package/lib/runner/runtime-preparation.mjs

@@ -127,6 +127,7 @@ export async function computeRuntimePrepareFingerprint(config) {
         : null,
     })
   );
+  hash.update(JSON.stringify(collectRuntimeDatabaseFingerprintInputs(config)));
 
   for (const envFile of config.testkit.envFiles || []) {
     appendFileToHash(hash, config.productDir, resolveServiceCwd(config.productDir, envFile));
@@ -138,6 +139,41 @@ export async function computeRuntimePrepareFingerprint(config) {
   return hash.digest("hex");
 }
 
+function collectRuntimeDatabaseFingerprintInputs(config) {
+  const inputs = [];
+  const ownDatabaseUrl = readDatabaseUrl(config.stateDir);
+  if (ownDatabaseUrl) {
+    inputs.push({ service: config.name, url: ownDatabaseUrl });
+  }
+
+  const referencedServices = new Set();
+  for (const value of Object.values(config.testkit.serviceEnv || {})) {
+    collectDatabasePlaceholderServices(value, referencedServices, config.name);
+  }
+  for (const value of Object.values(config.testkit.local?.env || {})) {
+    collectDatabasePlaceholderServices(value, referencedServices, config.name);
+  }
+
+  const stateDirByService = config.testkit.templateContext?.stateDirByService;
+  for (const serviceName of [...referencedServices].sort()) {
+    const stateDir = stateDirByService?.get?.(serviceName);
+    const databaseUrl = stateDir ? readDatabaseUrl(stateDir) : null;
+    inputs.push({ service: serviceName, url: databaseUrl || null });
+  }
+
+  return inputs;
+}
+
+function collectDatabasePlaceholderServices(value, out, defaultServiceName) {
+  if (typeof value !== "string") return;
+  const matcher = /\{db(?:Url|Host|Port|Name|User|Password)(?::([a-zA-Z0-9_-]+))?\}/g;
+  let match = matcher.exec(value);
+  while (match) {
+    out.add(match[1] || defaultServiceName);
+    match = matcher.exec(value);
+  }
+}
+
 function appendResolvedInputToHash(hash, productDir, absPath) {
   const relative = path.relative(productDir, absPath);
   appendInputToHash(hash, productDir, relative);

package/lib/runner/template.mjs

@@ -204,17 +204,29 @@ export function buildExecutionEnv(config, extraEnv = {}, processEnv = process.en
   return buildExecutionEnvWithContext(config, null, extraEnv, processEnv);
 }
 
+export function buildTemplateExecutionEnv(config, extraEnv = {}, processEnv = process.env) {
+  return buildExecutionEnvWithContext(config, null, extraEnv, processEnv, {
+    omitRuntimeDatabaseBindings: true,
+  });
+}
+
 export function buildTaskExecutionEnv(config, lease, extraEnv = {}, processEnv = process.env) {
   return buildExecutionEnvWithContext(config, lease, extraEnv, processEnv);
 }
 
-function buildExecutionEnvWithContext(config, lease, extraEnv, processEnv) {
+function buildExecutionEnvWithContext(config, lease, extraEnv, processEnv, options = {}) {
   const inheritedEnv = { ...processEnv };
   const templateContext = buildTemplateContext(config, lease);
+  const serviceEnv = options.omitRuntimeDatabaseBindings
+    ? omitRuntimeDatabaseBindings(config.testkit.serviceEnv || {})
+    : config.testkit.serviceEnv || {};
+  const localEnv = options.omitRuntimeDatabaseBindings
+    ? omitRuntimeDatabaseBindings(config.testkit.local?.env || {})
+    : config.testkit.local?.env || {};
   const env = {
     ...inheritedEnv,
-    ...resolveEnvTemplates(config.testkit.serviceEnv || {}, templateContext),
-    ...resolveEnvTemplates(config.testkit.local?.env || {}, templateContext),
+    ...resolveEnvTemplates(serviceEnv, templateContext),
+    ...resolveEnvTemplates(localEnv, templateContext),
     ...resolveEnvTemplates(extraEnv, templateContext),
     TESTKIT_ACTIVE: "1",
     ...(config.runtimeId ? { TESTKIT_RUNTIME_ID: String(config.runtimeId) } : {}),
@@ -340,6 +352,15 @@ function resolveEnvTemplates(values, templateContext) {
   );
 }
 
+function omitRuntimeDatabaseBindings(values = {}) {
+  return Object.fromEntries(
+    Object.entries(values).filter(([_key, value]) => {
+      if (typeof value !== "string") return true;
+      return !/\{db(?:Url|Host|Port|Name|User|Password)(?::[a-zA-Z0-9_-]+)?\}/.test(value);
+    })
+  );
+}
+
 function finalizeRuntimePrepare(prepare, context) {
   if (!prepare) {
     return {

package/node_modules/@elench/next-analysis/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/next-analysis",
-  "version": "0.1.96",
+  "version": "0.1.97",
   "description": "SWC-backed Next.js source analysis primitives for Erench tools",
   "type": "module",
   "exports": {

package/node_modules/@elench/testkit-bridge/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit-bridge",
-  "version": "0.1.96",
+  "version": "0.1.97",
   "description": "Browser bridge helpers for testkit",
   "type": "module",
   "main": "./dist/index.js",
@@ -22,7 +22,7 @@
     "typecheck": "tsc -p tsconfig.json --noEmit"
   },
   "dependencies": {
-    "@elench/testkit-protocol": "0.1.96"
+    "@elench/testkit-protocol": "0.1.97"
   },
   "private": false
 }

package/node_modules/@elench/testkit-protocol/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit-protocol",
-  "version": "0.1.96",
+  "version": "0.1.97",
   "description": "Shared browser protocol for testkit bridge and extension consumers",
   "type": "module",
   "main": "./dist/index.js",

package/node_modules/@elench/ts-analysis/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/ts-analysis",
-  "version": "0.1.96",
+  "version": "0.1.97",
   "description": "TypeScript compiler-backed source analysis primitives for Erench tools",
   "type": "module",
   "exports": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.96",
+  "version": "0.1.97",
   "description": "Assistant-first CLI for running, inspecting, and debugging local testkit suites",
   "type": "module",
   "workspaces": [
@@ -62,7 +62,8 @@
     "test:audit": "node scripts/test-boundary-audit.mjs",
     "test:unit": "npm run build:packages && npm run test:audit && vitest run --config vitest.unit.config.mjs",
     "test:integration": "npm run build:packages && vitest run test/integration",
-    "test:system": "npm run build:packages && vitest run test/system --passWithNoTests"
+    "test:system": "npm run build:packages && vitest run test/system --passWithNoTests",
+    "test:live-providers": "npm run build:packages && vitest run --config vitest.live.config.mjs --passWithNoTests"
   },
   "files": [
     "bin/",
@@ -83,10 +84,10 @@
   },
   "dependencies": {
     "@babel/code-frame": "^7.29.0",
-    "@elench/next-analysis": "0.1.96",
-    "@elench/testkit-bridge": "0.1.96",
-    "@elench/testkit-protocol": "0.1.96",
-    "@elench/ts-analysis": "0.1.96",
+    "@elench/next-analysis": "0.1.97",
+    "@elench/testkit-bridge": "0.1.97",
+    "@elench/testkit-protocol": "0.1.97",
+    "@elench/ts-analysis": "0.1.97",
     "@oclif/core": "^4.10.6",
     "esbuild": "^0.25.11",
     "execa": "^9.5.0",