@elench/testkit 0.1.100 → 0.1.102

package/README.md

@@ -43,9 +43,6 @@ npx @elench/testkit --workers 8
 # One file-level wall clock budget for every suite file
 npx @elench/testkit --file-timeout-seconds 60
 
-# Run a deterministic shard
-npx @elench/testkit --shard 1/3
-
 # Specific service / suite
 npx @elench/testkit --service frontend --type pw -s navigation
 npx @elench/testkit --service api --type int -s health
@@ -126,6 +123,12 @@ output, emitted artifacts, and assistant-visible run state are persisted under
 run counts, pass/fail/skip counts, average duration, and last observed status,
 and those summaries are exposed in compact, verbose, and JSON discovery output.
 
+Test execution also maintains a scheduler cache at `.testkit/timings.json`.
+Completed file-level task durations are used to rank future runs with a
+longest-estimated-duration-first policy, so slow files start earlier when
+workers are available. Run artifacts include compact scheduler metadata under
+`planning` so ordering decisions are inspectable.
+
 ## Automatic Regression Diagnosis
 
 If `regressions.file` is configured, every run automatically classifies observed

package/lib/cli/args.mjs

@@ -58,25 +58,6 @@ export {
   parseWorkersOption,
 };
 
-export function parseShardOption(value) {
-  if (!value) return null;
-
-  const match = String(value).match(/^(\d+)\/(\d+)$/);
-  if (!match) {
-    throw new Error(
-      `Invalid --shard value "${value}". Expected the form "i/n", e.g. 1/3.`
-    );
-  }
-
-  const index = Number.parseInt(match[1], 10);
-  const total = Number.parseInt(match[2], 10);
-  if (index <= 0 || total <= 0 || index > total) {
-    throw new Error(`Invalid --shard value "${value}". Expected 1 <= i <= n.`);
-  }
-
-  return { index, total };
-}
-
 export function resolveRequestedFiles(fileNames, productDir, invocationCwd = process.cwd()) {
   const resolved = [];
   const seen = new Set();

package/lib/cli/assistant/app.mjs

@@ -281,6 +281,9 @@ function colorWelcomeValue(label, value) {
 function colorMarker(block) {
   if (block.kind === "user") return cyan(block.marker);
   if (block.kind === "system") return red(block.marker);
+  if (block.kind === "provider-error") return red(block.marker);
+  if (block.kind === "provider-activity") return dim(block.marker);
+  if (block.kind === "provider-tool") return yellow(block.marker);
   if (block.kind === "tool-running") return yellow(block.marker);
   if (block.kind === "testkit-run") return green(block.marker);
   return block.marker;
@@ -289,6 +292,9 @@ function colorMarker(block) {
 function colorBlockText(block, text) {
   if (block.kind === "user") return text;
   if (block.kind === "system") return red(text);
+  if (block.kind === "provider-error") return red(text);
+  if (block.kind === "provider-activity") return dim(text);
+  if (block.kind === "provider-tool") return yellow(text);
   if (block.kind === "tool-running") return yellow(text);
   return text;
 }

package/lib/cli/assistant/command-observer.mjs

@@ -1,9 +1,9 @@
 import fs from "fs";
 import path from "path";
-import { loadCurrentRunArtifact, loadLatestRunArtifact } from "../../results/artifacts.mjs";
 
 const POLL_INTERVAL_MS = 150;
 const OBSERVED_KINDS = new Set(["run", "discover", "status", "doctor", "typecheck"]);
+const RUN_KINDS = new Set(["run", "int", "e2e", "scenario", "dal", "load", "pw", "all"]);
 
 export function createAssistantCommandObserver({
   productDir,
@@ -13,15 +13,16 @@ export function createAssistantCommandObserver({
   intervalMs = POLL_INTERVAL_MS,
 } = {}) {
   const seenResultFiles = new Set();
+  const seenCommandLogEvents = new Set();
+  const observedRunCommandIds = new Set();
   let timer = null;
   let running = false;
-  let lastArtifactSignature = null;
-  let preferLatestArtifact = false;
+  let lastArtifactSignatures = new Map();
 
   function start() {
     if (running) return;
     running = true;
-    lastArtifactSignature = readArtifactSignature();
+    lastArtifactSignatures = readArtifactSignatures();
     scan();
     timer = setInterval(scan, intervalMs);
   }
@@ -34,10 +35,32 @@ export function createAssistantCommandObserver({
   }
 
   function scan() {
+    observeCommandLog();
     observeCommandResults();
     observeRunArtifact();
   }
 
+  function observeCommandLog() {
+    const commandLogPath = commandLog?.commandLogPath;
+    if (!commandLogPath || !fs.existsSync(commandLogPath)) return;
+    const lines = fs.readFileSync(commandLogPath, "utf8").split(/\r?\n/).filter(Boolean);
+    for (const [index, line] of lines.entries()) {
+      const eventKey = `${index}:${line}`;
+      if (seenCommandLogEvents.has(eventKey)) continue;
+      seenCommandLogEvents.add(eventKey);
+      let event = null;
+      try {
+        event = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      if (event.sessionId && commandLog.sessionId && event.sessionId !== commandLog.sessionId) continue;
+      if (event.type === "command_start" && isRunCommand(event)) {
+        observedRunCommandIds.add(event.commandId);
+      }
+    }
+  }
+
   function observeCommandResults() {
     const resultDir = commandLog?.resultDir;
     if (!resultDir || !fs.existsSync(resultDir)) return;
@@ -54,10 +77,11 @@ export function createAssistantCommandObserver({
       } catch {
         continue;
       }
+      if (document.sessionId && commandLog.sessionId && document.sessionId !== commandLog.sessionId) continue;
       if (!OBSERVED_KINDS.has(document.kind)) continue;
       seenResultFiles.add(filePath);
       if (document.kind === "run") {
-        preferLatestArtifact = true;
+        observedRunCommandIds.add(document.commandId);
         hydrateRunArtifact("command-result", document);
       }
       onEvent?.({
@@ -68,29 +92,34 @@ export function createAssistantCommandObserver({
   }
 
   function observeRunArtifact() {
-    const signature = readArtifactSignature();
-    if (!signature) return;
-    if (signature === lastArtifactSignature) return;
-    lastArtifactSignature = signature;
-    hydrateRunArtifact("artifact", { artifactPath: signature.split(":")[0] });
-  }
+    const signatures = readArtifactSignatures();
+    const changed = [...signatures.entries()]
+      .filter(([artifactPath, signature]) => lastArtifactSignatures.get(artifactPath) !== signature)
+      .map(([artifactPath, signature]) => ({
+        artifactPath,
+        signature,
+        mtimeMs: Number(signature.split(":").at(-2) || 0),
+      }))
+      .sort((left, right) => left.mtimeMs - right.mtimeMs);
+    lastArtifactSignatures = signatures;
 
-  function readArtifactSignature() {
-    const artifactPath = resolveObservableArtifactPath();
-    if (!artifactPath) return null;
-    const stat = fs.statSync(artifactPath);
-    return `${artifactPath}:${stat.mtimeMs}:${stat.size}`;
+    for (const entry of changed) {
+      const artifact = readJsonFile(entry.artifactPath);
+      if (!artifact || !shouldHydrateObservedArtifact(artifact)) continue;
+      hydrateRunArtifact("artifact", { artifactPath: entry.artifactPath, artifact });
+    }
   }
 
-  function resolveObservableArtifactPath() {
+  function readArtifactSignatures() {
     const livePath = path.join(productDir, ".testkit", "results", "live.json");
     const latestPath = path.join(productDir, ".testkit", "results", "latest.json");
-    const liveStat = safeStat(livePath);
-    const latestStat = safeStat(latestPath);
-    if (preferLatestArtifact && latestStat) return latestPath;
-    if (latestStat && (!liveStat || latestStat.mtimeMs >= liveStat.mtimeMs)) return latestPath;
-    if (liveStat) return livePath;
-    return null;
+    const signatures = new Map();
+    for (const artifactPath of [livePath, latestPath]) {
+      const stat = safeStat(artifactPath);
+      if (!stat) continue;
+      signatures.set(artifactPath, `${artifactPath}:${stat.mtimeMs}:${stat.size}`);
+    }
+    return signatures;
   }
 
   function hydrateRunArtifact(source, command = null) {
@@ -107,29 +136,31 @@ export function createAssistantCommandObserver({
 
   function loadObservedRunArtifact(command = null) {
     if (command?.result?.runArtifact) return command.result.runArtifact;
-    if (command?.artifactPath && fs.existsSync(command.artifactPath)) {
-      try {
-        return JSON.parse(fs.readFileSync(command.artifactPath, "utf8"));
-      } catch {
-        return null;
-      }
-    }
-    const artifactPath = resolveObservableArtifactPath();
-    if (artifactPath) {
-      try {
-        return JSON.parse(fs.readFileSync(artifactPath, "utf8"));
-      } catch {
-        return null;
-      }
-    }
+    if (command?.artifact) return command.artifact;
+    if (command?.artifactPath && fs.existsSync(command.artifactPath)) return readJsonFile(command.artifactPath);
+    return null;
+  }
+
+  function shouldHydrateObservedArtifact(artifact) {
+    const assistant = artifact?.provenance?.assistant || {};
+    if (!assistant.sessionId || !assistant.commandId) return false;
+    if (commandLog?.sessionId && assistant.sessionId !== commandLog.sessionId) return false;
+    return observedRunCommandIds.has(assistant.commandId);
+  }
+
+  function isRunCommand(event) {
+    if (!event?.commandId) return false;
+    if (event.kind === "run") return true;
+    if (RUN_KINDS.has(event.kind)) return true;
+    const argv = Array.isArray(event.argv) ? event.argv : [];
+    return argv[0] === "run" || RUN_KINDS.has(argv[0]);
+  }
+
+  function readJsonFile(filePath) {
     try {
-      return loadCurrentRunArtifact(productDir);
+      return JSON.parse(fs.readFileSync(filePath, "utf8"));
     } catch {
-      try {
-        return loadLatestRunArtifact(productDir);
-      } catch {
-        return null;
-      }
+      return null;
     }
   }
 

package/lib/cli/assistant/command-results.mjs

@@ -7,6 +7,19 @@ export const ASSISTANT_COMMAND_LOG_ENV = "TESTKIT_ASSISTANT_COMMAND_LOG";
 export const ASSISTANT_COMMAND_ID_ENV = "TESTKIT_ASSISTANT_COMMAND_ID";
 export const ASSISTANT_WRAPPER_LOGGED_ENV = "TESTKIT_ASSISTANT_WRAPPER_LOGGED";
 
+const RUN_SHORTCUTS = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
+const FLAGS_WITH_VALUES = new Set([
+  "--dir",
+  "--service",
+  "--type",
+  "--suite",
+  "--file",
+  "--workers",
+  "--file-timeout-seconds",
+  "--seed",
+  "--output-mode",
+]);
+
 export function createAssistantCommandContext({
   kind,
   argv = process.argv.slice(2),
@@ -136,8 +149,22 @@ export function appendAssistantCommandLog(context, event) {
 }
 
 function inferCommandKind(argv) {
-  const positionals = (Array.isArray(argv) ? argv : []).filter((arg) => !String(arg).startsWith("-"));
-  return positionals[0] || "run";
+  const first = findFirstPositional(argv);
+  if (!first || RUN_SHORTCUTS.has(first)) return "run";
+  return first;
+}
+
+function findFirstPositional(argv) {
+  const args = Array.isArray(argv) ? argv : [];
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = String(args[index]);
+    if (FLAGS_WITH_VALUES.has(arg)) {
+      index += 1;
+      continue;
+    }
+    if (!arg.startsWith("-")) return arg;
+  }
+  return null;
 }
 
 function inferExitCode(result) {

package/lib/cli/assistant/context-pack.mjs

@@ -178,7 +178,27 @@ function appendCommandLog(event) {
 }
 
 function inferKind(args) {
-  return args.find((arg) => !String(arg).startsWith("-")) || "run";
+  const runShortcuts = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
+  const flagsWithValues = new Set([
+    "--dir",
+    "--service",
+    "--type",
+    "--suite",
+    "--file",
+    "--workers",
+    "--file-timeout-seconds",
+    "--seed",
+    "--output-mode",
+  ]);
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = String(args[index]);
+    if (flagsWithValues.has(arg)) {
+      index += 1;
+      continue;
+    }
+    if (!arg.startsWith("-")) return runShortcuts.has(arg) ? "run" : arg;
+  }
+  return "run";
 }
 `;
 }

package/lib/cli/assistant/providers/claude.mjs

@@ -4,14 +4,15 @@ import {
   buildStatusEvent,
   buildToolEvent,
   createHostedSessionRunner,
-  extractTextFragments,
 } from "./shared.mjs";
+import { providerAssistantDelta, providerAssistantFinal, providerToolEnd, providerToolStart } from "./events.mjs";
 
 export function startClaudeHostedSession({
   command = "claude",
   cwd,
   prompt,
   onEvent,
+  onRawLine,
   purpose = "assistant",
   model = null,
   effort = null,
@@ -49,6 +50,7 @@ export function startClaudeHostedSession({
     provider: "claude",
     child,
     onEvent,
+    onRawLine,
     parsePayload: parseClaudePayload,
     readFinalText(result) {
       return readClaudeFinalText(result?.stdout || "") || null;
@@ -77,7 +79,20 @@ export function parseClaudePayload(payload) {
     const streamEvent = payload.event || {};
     if (streamEvent.type === "content_block_delta" && streamEvent.delta?.type === "text_delta") {
       const text = String(streamEvent.delta.text || "");
-      if (text) events.push({ type: "delta", text });
+      const event = providerAssistantDelta(text);
+      if (event) events.push(event);
+      return events;
+    }
+    if (streamEvent.type === "content_block_start" && streamEvent.content_block?.type === "tool_use") {
+      const tool = streamEvent.content_block;
+      const event = providerToolStart(
+        tool.name || tool.tool_name || "tool_use",
+        {
+          id: tool.id || streamEvent.index,
+          input: tool.input || null,
+        }
+      );
+      if (event) events.push(event);
       return events;
     }
     if (streamEvent.type === "tool_use" || streamEvent.content_block?.type === "tool_use") {
@@ -102,10 +117,9 @@ export function parseClaudePayload(payload) {
   }
 
   if (type === "assistant") {
-    const fragments = [...new Set(extractTextFragments(payload.message?.content || payload.content || [], []))];
-    for (const fragment of fragments) {
-      events.push({ type: "delta", text: fragment });
-    }
+    const fragments = extractClaudeTextFragments(payload.message?.content || payload.content || []);
+    const event = providerAssistantFinal(fragments.join(""));
+    if (event) events.push(event);
     return events;
   }
 
@@ -113,6 +127,10 @@ export function parseClaudePayload(payload) {
     if (payload.is_error || payload.subtype === "error") {
       const event = buildErrorEvent(payload.result || payload.error || "Claude command failed");
       if (event) events.push(event);
+    } else if (typeof payload.result === "string") {
+      const event = providerAssistantFinal(payload.result);
+      if (event) events.push(event);
+      events.push(providerToolEnd("claude", { status: "ok", durationMs: payload.duration_ms || null }));
     }
     return events;
   }
@@ -147,10 +165,27 @@ export function readClaudeFinalText(stdout) {
     }
 
     if (payload.type === "assistant") {
-      const fragments = [...new Set(extractTextFragments(payload.message?.content || payload.content || [], []))];
+      const fragments = extractClaudeTextFragments(payload.message?.content || payload.content || []);
       if (fragments.length > 0) fallback = fragments.join("");
     }
   }
 
   return fallback;
 }
+
+function extractClaudeTextFragments(content) {
+  const entries = Array.isArray(content) ? content : [content];
+  const fragments = [];
+  for (const entry of entries) {
+    if (typeof entry === "string") {
+      const text = entry.trim();
+      if (text) fragments.push(text);
+      continue;
+    }
+    if (!entry || typeof entry !== "object") continue;
+    if (entry.type !== "text") continue;
+    const text = String(entry.text || "").trim();
+    if (text) fragments.push(text);
+  }
+  return [...new Set(fragments)];
+}

package/lib/cli/assistant/providers/codex.mjs

@@ -7,15 +7,16 @@ import {
   buildStatusEvent,
   buildToolEvent,
   createHostedSessionRunner,
-  extractTextFragments,
   readTextFileIfPresent,
 } from "./shared.mjs";
+import { providerAssistantDelta, providerAssistantFinal, providerToolEnd, providerToolStart, providerToolUpdate } from "./events.mjs";
 
 export function startCodexHostedSession({
   command = "codex",
   cwd,
   prompt,
   onEvent,
+  onRawLine,
   purpose = "assistant",
   model = null,
   providerArgs = [],
@@ -44,6 +45,7 @@ export function startCodexHostedSession({
     provider: "codex",
     child,
     onEvent,
+    onRawLine,
     parsePayload: parseCodexPayload,
     shouldIgnoreStatus(message) {
       return String(message || "").trim() === "Reading additional input from stdin...";
@@ -96,6 +98,45 @@ export function parseCodexPayload(payload) {
     return events;
   }
 
+  if (type === "thread.started") {
+    const event = buildStatusEvent(payload.thread_id ? `Codex thread ${payload.thread_id} started` : "Codex thread started");
+    if (event) events.push(event);
+    return events;
+  }
+
+  if (type === "turn.started") {
+    const event = buildStatusEvent("Codex turn started");
+    if (event) events.push(event);
+    return events;
+  }
+
+  if (type === "turn.completed") {
+    const event = providerToolEnd("codex", { status: "ok", usage: payload.usage || null });
+    if (event) events.push(event);
+    return events;
+  }
+
+  if (type === "item.started") {
+    const item = payload.item || {};
+    const event = codexItemStartedEvent(item);
+    if (event) events.push(event);
+    return events;
+  }
+
+  if (type === "item.updated") {
+    const item = payload.item || {};
+    const event = codexItemUpdatedEvent(item);
+    if (event) events.push(event);
+    return events;
+  }
+
+  if (type === "item.completed") {
+    const item = payload.item || {};
+    const event = codexItemCompletedEvent(item);
+    if (event) events.push(event);
+    return events;
+  }
+
   if (type && /(tool|command|patch|exec)/i.test(type)) {
     const event = buildToolEvent(
       payload.name || payload.tool_name || payload.command || type,
@@ -105,15 +146,52 @@ export function parseCodexPayload(payload) {
     return events;
   }
 
-  const fragments = [...new Set(extractTextFragments(payload, []))];
-  if (fragments.length > 0) {
-    for (const fragment of fragments) {
-      events.push({ type: "delta", text: fragment });
-    }
-    return events;
-  }
-
   const statusEvent = buildStatusEvent(type ? `Codex event: ${type}` : JSON.stringify(payload));
   if (statusEvent) events.push(statusEvent);
   return events;
 }
+
+function codexItemStartedEvent(item) {
+  if (!item || typeof item !== "object") return null;
+  if (item.type === "command_execution" || item.type === "tool_call") {
+    return providerToolStart(item.command || item.name || item.type, {
+      id: item.id || null,
+      input: item.arguments || item.input || null,
+    });
+  }
+  if (item.type === "agent_message") return buildStatusEvent("Codex started response");
+  if (item.type) return buildStatusEvent(`Codex started ${item.type}`);
+  return null;
+}
+
+function codexItemUpdatedEvent(item) {
+  if (!item || typeof item !== "object") return null;
+  if (item.type === "agent_message" && typeof item.text === "string") {
+    return providerAssistantDelta(item.text);
+  }
+  if (item.type === "command_execution" || item.type === "tool_call") {
+    return providerToolUpdate(item.command || item.name || item.type, {
+      id: item.id || null,
+      text: item.output || item.status || null,
+      data: item,
+    });
+  }
+  return null;
+}
+
+function codexItemCompletedEvent(item) {
+  if (!item || typeof item !== "object") return null;
+  if (item.type === "agent_message" && typeof item.text === "string") {
+    return providerAssistantFinal(item.text);
+  }
+  if (item.type === "command_execution" || item.type === "tool_call") {
+    return providerToolEnd(item.command || item.name || item.type, {
+      id: item.id || null,
+      status: item.status === "failed" ? "error" : "ok",
+      output: item.output || null,
+      data: item,
+    });
+  }
+  if (item.type) return buildStatusEvent(`Codex completed ${item.type}`);
+  return null;
+}

package/lib/cli/assistant/providers/events.mjs

@@ -0,0 +1,71 @@
+export const PROVIDER_EVENT_TYPES = new Set([
+  "session-start",
+  "status",
+  "assistant-delta",
+  "assistant-final",
+  "tool-start",
+  "tool-update",
+  "tool-end",
+  "error",
+  "session-end",
+]);
+
+export function providerEvent(type, fields = {}) {
+  if (!PROVIDER_EVENT_TYPES.has(type)) {
+    throw new Error(`Unknown provider event type: ${type}`);
+  }
+  return {
+    type,
+    ...fields,
+  };
+}
+
+export function providerStatus(text, fields = {}) {
+  const normalized = normalizeText(text);
+  if (!normalized) return null;
+  return providerEvent("status", { text: normalized, ...fields });
+}
+
+export function providerAssistantDelta(text, fields = {}) {
+  if (!text) return null;
+  return providerEvent("assistant-delta", { text: String(text), ...fields });
+}
+
+export function providerAssistantFinal(text, fields = {}) {
+  const normalized = normalizeText(text);
+  if (!normalized) return null;
+  return providerEvent("assistant-final", { text: normalized, ...fields });
+}
+
+export function providerToolStart(name, fields = {}) {
+  const normalized = normalizeText(name);
+  if (!normalized) return null;
+  return providerEvent("tool-start", { name: normalized, ...fields });
+}
+
+export function providerToolUpdate(name, fields = {}) {
+  const normalized = normalizeText(name);
+  if (!normalized && !fields.text && !fields.data) return null;
+  return providerEvent("tool-update", { ...(normalized ? { name: normalized } : {}), ...fields });
+}
+
+export function providerToolEnd(name, fields = {}) {
+  const normalized = normalizeText(name);
+  if (!normalized) return null;
+  return providerEvent("tool-end", { name: normalized, status: fields.status || "ok", ...fields });
+}
+
+export function providerError(text, fields = {}) {
+  const normalized = normalizeText(text);
+  if (!normalized) return null;
+  return providerEvent("error", { text: normalized, ...fields });
+}
+
+export function providerSessionEnd(fields = {}) {
+  return providerEvent("session-end", fields);
+}
+
+export function normalizeText(value) {
+  const text = String(value || "").trim();
+  return text || null;
+}

package/lib/cli/assistant/providers/index.mjs

@@ -3,7 +3,7 @@ import path from "path";
 import { startClaudeHostedSession } from "./claude.mjs";
 import { startCodexHostedSession } from "./codex.mjs";
 
-const PROVIDERS = ["codex", "claude"];
+export const HOSTED_ASSISTANT_PROVIDERS = Object.freeze(["codex", "claude"]);
 
 export function resolvePreferredProvider(preferred = null, env = process.env) {
   if (preferred && preferred !== "auto") {
@@ -13,7 +13,7 @@ export function resolvePreferredProvider(preferred = null, env = process.env) {
     return preferred;
   }
 
-  for (const provider of PROVIDERS) {
+  for (const provider of HOSTED_ASSISTANT_PROVIDERS) {
     if (isProviderInstalled(provider, env)) return provider;
   }
   throw new Error("Neither codex nor claude was found on PATH");
@@ -63,13 +63,14 @@ export function startProviderSession({
   cwd,
   prompt,
   onEvent,
+  onRawLine,
   purpose = "assistant",
   env = process.env,
 } = {}) {
   const resolvedProvider = resolvePreferredProvider(provider, env);
   const command = resolveProviderBinary(resolvedProvider, env);
   if (resolvedProvider === "claude") {
-    return startClaudeHostedSession({ command, cwd, prompt, onEvent, purpose, model, effort, providerArgs, env });
+    return startClaudeHostedSession({ command, cwd, prompt, onEvent, onRawLine, purpose, model, effort, providerArgs, env });
   }
-  return startCodexHostedSession({ command, cwd, prompt, onEvent, purpose, model, providerArgs, env });
+  return startCodexHostedSession({ command, cwd, prompt, onEvent, onRawLine, purpose, model, providerArgs, env });
 }