@elench/testkit 0.1.100 → 0.1.101

package/README.md

@@ -43,9 +43,6 @@ npx @elench/testkit --workers 8
 # One file-level wall clock budget for every suite file
 npx @elench/testkit --file-timeout-seconds 60
 
-# Run a deterministic shard
-npx @elench/testkit --shard 1/3
-
 # Specific service / suite
 npx @elench/testkit --service frontend --type pw -s navigation
 npx @elench/testkit --service api --type int -s health
@@ -126,6 +123,12 @@ output, emitted artifacts, and assistant-visible run state are persisted under
 run counts, pass/fail/skip counts, average duration, and last observed status,
 and those summaries are exposed in compact, verbose, and JSON discovery output.
 
+Test execution also maintains a scheduler cache at `.testkit/timings.json`.
+Completed file-level task durations are used to rank future runs with a
+longest-estimated-duration-first policy, so slow files start earlier when
+workers are available. Run artifacts include compact scheduler metadata under
+`planning` so ordering decisions are inspectable.
+
 ## Automatic Regression Diagnosis
 
 If `regressions.file` is configured, every run automatically classifies observed

package/lib/cli/args.mjs

@@ -58,25 +58,6 @@ export {
   parseWorkersOption,
 };
 
-export function parseShardOption(value) {
-  if (!value) return null;
-
-  const match = String(value).match(/^(\d+)\/(\d+)$/);
-  if (!match) {
-    throw new Error(
-      `Invalid --shard value "${value}". Expected the form "i/n", e.g. 1/3.`
-    );
-  }
-
-  const index = Number.parseInt(match[1], 10);
-  const total = Number.parseInt(match[2], 10);
-  if (index <= 0 || total <= 0 || index > total) {
-    throw new Error(`Invalid --shard value "${value}". Expected 1 <= i <= n.`);
-  }
-
-  return { index, total };
-}
-
 export function resolveRequestedFiles(fileNames, productDir, invocationCwd = process.cwd()) {
   const resolved = [];
   const seen = new Set();

package/lib/cli/assistant/command-observer.mjs

@@ -1,9 +1,9 @@
 import fs from "fs";
 import path from "path";
-import { loadCurrentRunArtifact, loadLatestRunArtifact } from "../../results/artifacts.mjs";
 
 const POLL_INTERVAL_MS = 150;
 const OBSERVED_KINDS = new Set(["run", "discover", "status", "doctor", "typecheck"]);
+const RUN_KINDS = new Set(["run", "int", "e2e", "scenario", "dal", "load", "pw", "all"]);
 
 export function createAssistantCommandObserver({
   productDir,
@@ -13,15 +13,16 @@ export function createAssistantCommandObserver({
   intervalMs = POLL_INTERVAL_MS,
 } = {}) {
   const seenResultFiles = new Set();
+  const seenCommandLogEvents = new Set();
+  const observedRunCommandIds = new Set();
   let timer = null;
   let running = false;
-  let lastArtifactSignature = null;
-  let preferLatestArtifact = false;
+  let lastArtifactSignatures = new Map();
 
   function start() {
     if (running) return;
     running = true;
-    lastArtifactSignature = readArtifactSignature();
+    lastArtifactSignatures = readArtifactSignatures();
     scan();
     timer = setInterval(scan, intervalMs);
   }
@@ -34,10 +35,32 @@ export function createAssistantCommandObserver({
   }
 
   function scan() {
+    observeCommandLog();
     observeCommandResults();
     observeRunArtifact();
   }
 
+  function observeCommandLog() {
+    const commandLogPath = commandLog?.commandLogPath;
+    if (!commandLogPath || !fs.existsSync(commandLogPath)) return;
+    const lines = fs.readFileSync(commandLogPath, "utf8").split(/\r?\n/).filter(Boolean);
+    for (const [index, line] of lines.entries()) {
+      const eventKey = `${index}:${line}`;
+      if (seenCommandLogEvents.has(eventKey)) continue;
+      seenCommandLogEvents.add(eventKey);
+      let event = null;
+      try {
+        event = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      if (event.sessionId && commandLog.sessionId && event.sessionId !== commandLog.sessionId) continue;
+      if (event.type === "command_start" && isRunCommand(event)) {
+        observedRunCommandIds.add(event.commandId);
+      }
+    }
+  }
+
   function observeCommandResults() {
     const resultDir = commandLog?.resultDir;
     if (!resultDir || !fs.existsSync(resultDir)) return;
@@ -54,10 +77,11 @@ export function createAssistantCommandObserver({
       } catch {
         continue;
       }
+      if (document.sessionId && commandLog.sessionId && document.sessionId !== commandLog.sessionId) continue;
       if (!OBSERVED_KINDS.has(document.kind)) continue;
       seenResultFiles.add(filePath);
       if (document.kind === "run") {
-        preferLatestArtifact = true;
+        observedRunCommandIds.add(document.commandId);
         hydrateRunArtifact("command-result", document);
       }
       onEvent?.({
@@ -68,29 +92,34 @@ export function createAssistantCommandObserver({
   }
 
   function observeRunArtifact() {
-    const signature = readArtifactSignature();
-    if (!signature) return;
-    if (signature === lastArtifactSignature) return;
-    lastArtifactSignature = signature;
-    hydrateRunArtifact("artifact", { artifactPath: signature.split(":")[0] });
-  }
+    const signatures = readArtifactSignatures();
+    const changed = [...signatures.entries()]
+      .filter(([artifactPath, signature]) => lastArtifactSignatures.get(artifactPath) !== signature)
+      .map(([artifactPath, signature]) => ({
+        artifactPath,
+        signature,
+        mtimeMs: Number(signature.split(":").at(-2) || 0),
+      }))
+      .sort((left, right) => left.mtimeMs - right.mtimeMs);
+    lastArtifactSignatures = signatures;
 
-  function readArtifactSignature() {
-    const artifactPath = resolveObservableArtifactPath();
-    if (!artifactPath) return null;
-    const stat = fs.statSync(artifactPath);
-    return `${artifactPath}:${stat.mtimeMs}:${stat.size}`;
+    for (const entry of changed) {
+      const artifact = readJsonFile(entry.artifactPath);
+      if (!artifact || !shouldHydrateObservedArtifact(artifact)) continue;
+      hydrateRunArtifact("artifact", { artifactPath: entry.artifactPath, artifact });
+    }
   }
 
-  function resolveObservableArtifactPath() {
+  function readArtifactSignatures() {
     const livePath = path.join(productDir, ".testkit", "results", "live.json");
     const latestPath = path.join(productDir, ".testkit", "results", "latest.json");
-    const liveStat = safeStat(livePath);
-    const latestStat = safeStat(latestPath);
-    if (preferLatestArtifact && latestStat) return latestPath;
-    if (latestStat && (!liveStat || latestStat.mtimeMs >= liveStat.mtimeMs)) return latestPath;
-    if (liveStat) return livePath;
-    return null;
+    const signatures = new Map();
+    for (const artifactPath of [livePath, latestPath]) {
+      const stat = safeStat(artifactPath);
+      if (!stat) continue;
+      signatures.set(artifactPath, `${artifactPath}:${stat.mtimeMs}:${stat.size}`);
+    }
+    return signatures;
   }
 
   function hydrateRunArtifact(source, command = null) {
@@ -107,29 +136,31 @@ export function createAssistantCommandObserver({
 
   function loadObservedRunArtifact(command = null) {
     if (command?.result?.runArtifact) return command.result.runArtifact;
-    if (command?.artifactPath && fs.existsSync(command.artifactPath)) {
-      try {
-        return JSON.parse(fs.readFileSync(command.artifactPath, "utf8"));
-      } catch {
-        return null;
-      }
-    }
-    const artifactPath = resolveObservableArtifactPath();
-    if (artifactPath) {
-      try {
-        return JSON.parse(fs.readFileSync(artifactPath, "utf8"));
-      } catch {
-        return null;
-      }
-    }
+    if (command?.artifact) return command.artifact;
+    if (command?.artifactPath && fs.existsSync(command.artifactPath)) return readJsonFile(command.artifactPath);
+    return null;
+  }
+
+  function shouldHydrateObservedArtifact(artifact) {
+    const assistant = artifact?.provenance?.assistant || {};
+    if (!assistant.sessionId || !assistant.commandId) return false;
+    if (commandLog?.sessionId && assistant.sessionId !== commandLog.sessionId) return false;
+    return observedRunCommandIds.has(assistant.commandId);
+  }
+
+  function isRunCommand(event) {
+    if (!event?.commandId) return false;
+    if (event.kind === "run") return true;
+    if (RUN_KINDS.has(event.kind)) return true;
+    const argv = Array.isArray(event.argv) ? event.argv : [];
+    return argv[0] === "run" || RUN_KINDS.has(argv[0]);
+  }
+
+  function readJsonFile(filePath) {
     try {
-      return loadCurrentRunArtifact(productDir);
+      return JSON.parse(fs.readFileSync(filePath, "utf8"));
     } catch {
-      try {
-        return loadLatestRunArtifact(productDir);
-      } catch {
-        return null;
-      }
+      return null;
     }
   }
 

package/lib/cli/assistant/command-results.mjs

@@ -7,6 +7,19 @@ export const ASSISTANT_COMMAND_LOG_ENV = "TESTKIT_ASSISTANT_COMMAND_LOG";
 export const ASSISTANT_COMMAND_ID_ENV = "TESTKIT_ASSISTANT_COMMAND_ID";
 export const ASSISTANT_WRAPPER_LOGGED_ENV = "TESTKIT_ASSISTANT_WRAPPER_LOGGED";
 
+const RUN_SHORTCUTS = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
+const FLAGS_WITH_VALUES = new Set([
+  "--dir",
+  "--service",
+  "--type",
+  "--suite",
+  "--file",
+  "--workers",
+  "--file-timeout-seconds",
+  "--seed",
+  "--output-mode",
+]);
+
 export function createAssistantCommandContext({
   kind,
   argv = process.argv.slice(2),
@@ -136,8 +149,22 @@ export function appendAssistantCommandLog(context, event) {
 }
 
 function inferCommandKind(argv) {
-  const positionals = (Array.isArray(argv) ? argv : []).filter((arg) => !String(arg).startsWith("-"));
-  return positionals[0] || "run";
+  const first = findFirstPositional(argv);
+  if (!first || RUN_SHORTCUTS.has(first)) return "run";
+  return first;
+}
+
+function findFirstPositional(argv) {
+  const args = Array.isArray(argv) ? argv : [];
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = String(args[index]);
+    if (FLAGS_WITH_VALUES.has(arg)) {
+      index += 1;
+      continue;
+    }
+    if (!arg.startsWith("-")) return arg;
+  }
+  return null;
 }
 
 function inferExitCode(result) {

package/lib/cli/assistant/context-pack.mjs

@@ -178,7 +178,27 @@ function appendCommandLog(event) {
 }
 
 function inferKind(args) {
-  return args.find((arg) => !String(arg).startsWith("-")) || "run";
+  const runShortcuts = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
+  const flagsWithValues = new Set([
+    "--dir",
+    "--service",
+    "--type",
+    "--suite",
+    "--file",
+    "--workers",
+    "--file-timeout-seconds",
+    "--seed",
+    "--output-mode",
+  ]);
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = String(args[index]);
+    if (flagsWithValues.has(arg)) {
+      index += 1;
+      continue;
+    }
+    if (!arg.startsWith("-")) return runShortcuts.has(arg) ? "run" : arg;
+  }
+  return "run";
 }
 `;
 }

package/lib/cli/assistant/settings.mjs

@@ -19,7 +19,11 @@ export function loadAssistantSettings(productDir) {
   const filePath = assistantSettingsPath(productDir);
   try {
     const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
-    return normalizeAssistantSettings(parsed);
+    const normalized = normalizeAssistantSettings(parsed);
+    if (JSON.stringify(parsed) !== JSON.stringify(normalized)) {
+      writeAssistantSettingsFile(filePath, normalized);
+    }
+    return normalized;
   } catch {
     return { ...DEFAULT_ASSISTANT_SETTINGS };
   }
@@ -27,8 +31,7 @@ export function loadAssistantSettings(productDir) {
 
 export function saveAssistantSettings(productDir, settings) {
   const filePath = assistantSettingsPath(productDir);
-  fs.mkdirSync(path.dirname(filePath), { recursive: true });
-  fs.writeFileSync(filePath, `${JSON.stringify(normalizeAssistantSettings(settings), null, 2)}\n`);
+  writeAssistantSettingsFile(filePath, normalizeAssistantSettings(settings));
 }
 
 export function resetAssistantSettings(productDir) {
@@ -52,8 +55,9 @@ export function mergeAssistantSettings(...settingsObjects) {
 
 export function normalizeAssistantSettings(value = {}) {
   const provider = normalizeProvider(value.provider);
-  const effort = normalizeEffort(value.effort);
-  const model = normalizeOptionalString(value.model);
+  const extracted = extractEffortFromModel(value.model);
+  const effort = normalizeEffort(value.effort || extracted.effort);
+  const model = extracted.model;
   const providerArgs = Array.isArray(value.providerArgs)
     ? value.providerArgs.map((entry) => normalizeOptionalString(entry)).filter(Boolean)
     : [];
@@ -89,6 +93,19 @@ export function normalizeOptionalString(value) {
   return stringValue || null;
 }
 
+export function extractEffortFromModel(value) {
+  const model = normalizeOptionalString(value);
+  if (!model) return { model: null, effort: null };
+  const tokens = model.split(/\s+/);
+  if (tokens.length < 2) return { model, effort: null };
+  const trailingEffort = tokens.at(-1);
+  if (!ASSISTANT_EFFORTS.includes(trailingEffort)) return { model, effort: null };
+  return {
+    model: tokens.slice(0, -1).join(" ").trim() || null,
+    effort: trailingEffort,
+  };
+}
+
 function dropNullishSettings(settings) {
   const result = {};
   for (const [key, value] of Object.entries(settings)) {
@@ -96,3 +113,8 @@ function dropNullishSettings(settings) {
   }
   return result;
 }
+
+function writeAssistantSettingsFile(filePath, settings) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, `${JSON.stringify(settings, null, 2)}\n`);
+}

package/lib/cli/assistant/state.mjs

@@ -470,7 +470,13 @@ async function executeSlashCommand({
   }
   if (slash.type === "model") {
     state.setModel(slash.model, { custom: slash.custom });
-    appendMessage({ role: "assistant", text: slash.model ? `Model set to ${slash.model}.` : "Model reset to provider default." });
+    const snapshot = state.getSnapshot();
+    appendMessage({
+      role: "assistant",
+      text: slash.model
+        ? `Model set to ${snapshot.model}${snapshot.effort ? ` with ${snapshot.effort} effort` : ""}.`
+        : "Model reset to provider default.",
+    });
     return;
   }
   if (slash.type === "model-list") {

package/lib/cli/command-flags.mjs

@@ -32,9 +32,6 @@ export const runFlags = {
   "file-timeout-seconds": Flags.string({
     description: "Per-file wall-clock timeout in seconds",
   }),
-  shard: Flags.string({
-    description: "Run only shard i of n at suite granularity",
-  }),
   seed: Flags.string({
     description: "Deterministic seed for scenario suites",
   }),

package/lib/cli/entrypoint.mjs

@@ -23,7 +23,6 @@ export function normalizeCliArgs(argv) {
     "--file",
     "--workers",
     "--file-timeout-seconds",
-    "--shard",
     "--seed",
     "--input",
     "--output",
@@ -54,7 +53,6 @@ export function normalizeCliArgs(argv) {
       "-f",
       "--workers",
       "--file-timeout-seconds",
-      "--shard",
       "--seed",
       "--write-status",
       "--allow-partial-status",

package/lib/cli/operations/run/operation.mjs

@@ -2,7 +2,6 @@ import * as runner from "../../../runner/index.mjs";
 import { loadManagedConfigs } from "../../../app/configs.mjs";
 import {
   parseFileTimeoutOption,
-  parseShardOption,
   parseSuiteOption,
   parseTypeOption,
   parseWorkersOption,
@@ -25,7 +24,6 @@ export async function buildRunRequest(flags, positionalType = null, cwd = proces
     flags["file-timeout-seconds"] == null
       ? null
       : parseFileTimeoutOption(flags["file-timeout-seconds"]);
-  const shard = parseShardOption(flags.shard);
   const typeValues = parseTypeOption(flags.type, positionalType);
   const suiteSelectors = parseSuiteOption(flags.suite);
   const rawFileNames = Array.isArray(flags.file) ? flags.file : [flags.file].filter(Boolean);
@@ -44,7 +42,6 @@ export async function buildRunRequest(flags, positionalType = null, cwd = proces
       fileNames,
       workers,
       fileTimeoutSeconds,
-      shard,
       scenarioSeed: flags.seed || null,
       serviceFilter: flags.service || null,
       writeStatus: flags["write-status"],

package/lib/runner/live-run.mjs

@@ -4,12 +4,14 @@ import { writeLiveRunArtifact } from "./artifacts.mjs";
 
 export function createLiveSnapshotWriter({
   productDir,
+  runId,
   configs,
   trackers,
   startedAt,
   execution,
   workerState,
   selection,
+  provenance = null,
   metadata,
   logRegistry,
   setupRegistry,
@@ -21,6 +23,7 @@ export function createLiveSnapshotWriter({
       productDir,
       buildLiveRunArtifact({
         productDir,
+        runId,
         results: partialResults,
         startedAt,
         updatedAt: now,
@@ -31,9 +34,10 @@ export function createLiveSnapshotWriter({
         typeValues: selection.typeValues,
         suiteSelectors: selection.suiteSelectors,
         fileNames: selection.fileNames,
-        shard: selection.shard,
         serviceFilter: selection.serviceFilter,
         scenarioSeed: selection.scenarioSeed,
+        planning: selection.planning || null,
+        provenance,
         metadata,
         summarizeDbBackend,
         serviceLogs: logRegistry.listServiceLogs(),

package/lib/runner/orchestrator.mjs

@@ -1,11 +1,10 @@
 import {
-  applyShard,
   buildRuntimeGraphs,
-  buildTaskQueue,
   claimNextTask,
   collectSuites,
   resolveRuntimeConfigs,
 } from "./planning.mjs";
+import { buildRunPlanningMetadata, buildScheduledQueue } from "./scheduler/index.mjs";
 import {
   addTrackerError,
   buildServiceTrackers,
@@ -20,6 +19,7 @@ import {
   resetResultArtifacts,
   saveTimings,
 } from "./artifacts.mjs";
+import { loadHistory } from "../history/index.mjs";
 import { createRunLogRegistry } from "./logs.mjs";
 import { createSetupOperationRegistry } from "./setup-operations.mjs";
 import {
@@ -40,12 +40,14 @@ import { createWorker, runWorker } from "./worker-loop.mjs";
 import { ensureRequestedFilesMatch, ensureStatusWriteAllowed } from "./run-guards.mjs";
 import { createLiveSnapshotWriter } from "./live-run.mjs";
 import { finalizeRunArtifacts } from "./run-finalization.mjs";
+import { buildRunProvenance } from "./provenance.mjs";
 
 export async function runAll(configs, typeValues, suiteSelectors, opts, allConfigs = configs) {
   const configMap = new Map(allConfigs.map((config) => [config.name, config]));
   const startedAt = Date.now();
   const telemetry = configs[0]?.telemetry || null;
   const productDir = configs[0]?.productDir || process.cwd();
+  const provenance = buildRunProvenance(opts.env || process.env);
   await cleanupStaleRuns(productDir);
   resetResultArtifacts(productDir);
   const metadata = {
@@ -87,30 +89,33 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   );
   reporter?.setServicePlans?.(servicePlans);
   const trackers = buildServiceTrackers(servicePlans, startedAt);
+  const runSelection = {
+    typeValues,
+    suiteSelectors,
+    fileNames: requestedFiles,
+    serviceFilter: opts.serviceFilter || null,
+    scenarioSeed: opts.scenarioSeed || null,
+    planning: null,
+  };
   let writeLiveSnapshot = () => {};
   const setupRegistry = createSetupOperationRegistry({ logRegistry, onChange: () => writeLiveSnapshot() });
+  const executedPlans = servicePlans.filter((plan) => !plan.skipped);
+  let exitCode = 0;
+  const lifecycle = createRunLifecycle(productDir);
   writeLiveSnapshot = createLiveSnapshotWriter({
     productDir,
+    runId: lifecycle.runId,
     configs,
     trackers,
     startedAt,
     execution,
     workerState,
-    selection: {
-      typeValues,
-      suiteSelectors,
-      fileNames: requestedFiles,
-      shard: opts.shard || null,
-      serviceFilter: opts.serviceFilter || null,
-      scenarioSeed: opts.scenarioSeed || null,
-    },
+    selection: runSelection,
+    provenance,
     metadata,
     logRegistry,
     setupRegistry,
   });
-  const executedPlans = servicePlans.filter((plan) => !plan.skipped);
-  let exitCode = 0;
-  const lifecycle = createRunLifecycle(productDir);
   lifecycle.markRunning();
   lifecycle.installSignalHandlers();
   let results = [];
@@ -120,8 +125,11 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   try {
     if (executedPlans.length > 0) {
       const timings = loadTimings(productDir);
+      const history = loadHistory(productDir);
       const graphs = buildRuntimeGraphs(executedPlans);
-      const queue = buildTaskQueue(executedPlans, graphs, timings);
+      const queue = buildScheduledQueue(executedPlans, graphs, { timings, history });
+      runSelection.planning = buildRunPlanningMetadata(queue);
+      writeLiveSnapshot();
       reporter?.setTotalFileCount?.(queue.length);
       for (const task of queue) {
         task.scenarioSeed = opts.scenarioSeed || null;
@@ -191,6 +199,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     );
     const finalized = await finalizeRunArtifacts({
       productDir,
+      runId: lifecycle.runId,
       results,
       startedAt,
       finishedAt,
@@ -198,14 +207,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       workerCount: workerState.workerCount,
       runtimeInstanceCount: workerState.runtimeInstanceCount,
       runtimeStats: workerState.runtimeStats,
-      selection: {
-        typeValues,
-        suiteSelectors,
-        fileNames: requestedFiles,
-        shard: opts.shard || null,
-        serviceFilter: opts.serviceFilter || null,
-        scenarioSeed: opts.scenarioSeed || null,
-      },
+      selection: runSelection,
+      provenance,
       metadata,
       logRegistry,
       setupRegistry,
@@ -245,10 +248,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
 
 function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opts, execution, reporter) {
   return configs.map((config) => {
-    const suites = applyShard(
-      collectSuites(config, typeValues, suiteSelectors, opts.fileNames || [], opts),
-      opts.shard
-    );
+    const suites = collectSuites(config, typeValues, suiteSelectors, opts.fileNames || [], opts);
 
     if (suites.length === 0) {
       reporter?.serviceSkipped?.(