@elench/testkit 0.1.99 → 0.1.101

package/lib/runner/orchestrator.mjs

@@ -1,11 +1,10 @@
 import {
-  applyShard,
   buildRuntimeGraphs,
-  buildTaskQueue,
   claimNextTask,
   collectSuites,
   resolveRuntimeConfigs,
 } from "./planning.mjs";
+import { buildRunPlanningMetadata, buildScheduledQueue } from "./scheduler/index.mjs";
 import {
   addTrackerError,
   buildServiceTrackers,
@@ -20,6 +19,7 @@ import {
   resetResultArtifacts,
   saveTimings,
 } from "./artifacts.mjs";
+import { loadHistory } from "../history/index.mjs";
 import { createRunLogRegistry } from "./logs.mjs";
 import { createSetupOperationRegistry } from "./setup-operations.mjs";
 import {
@@ -40,12 +40,14 @@ import { createWorker, runWorker } from "./worker-loop.mjs";
 import { ensureRequestedFilesMatch, ensureStatusWriteAllowed } from "./run-guards.mjs";
 import { createLiveSnapshotWriter } from "./live-run.mjs";
 import { finalizeRunArtifacts } from "./run-finalization.mjs";
+import { buildRunProvenance } from "./provenance.mjs";
 
 export async function runAll(configs, typeValues, suiteSelectors, opts, allConfigs = configs) {
   const configMap = new Map(allConfigs.map((config) => [config.name, config]));
   const startedAt = Date.now();
   const telemetry = configs[0]?.telemetry || null;
   const productDir = configs[0]?.productDir || process.cwd();
+  const provenance = buildRunProvenance(opts.env || process.env);
   await cleanupStaleRuns(productDir);
   resetResultArtifacts(productDir);
   const metadata = {
@@ -87,30 +89,33 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   );
   reporter?.setServicePlans?.(servicePlans);
   const trackers = buildServiceTrackers(servicePlans, startedAt);
+  const runSelection = {
+    typeValues,
+    suiteSelectors,
+    fileNames: requestedFiles,
+    serviceFilter: opts.serviceFilter || null,
+    scenarioSeed: opts.scenarioSeed || null,
+    planning: null,
+  };
   let writeLiveSnapshot = () => {};
   const setupRegistry = createSetupOperationRegistry({ logRegistry, onChange: () => writeLiveSnapshot() });
+  const executedPlans = servicePlans.filter((plan) => !plan.skipped);
+  let exitCode = 0;
+  const lifecycle = createRunLifecycle(productDir);
   writeLiveSnapshot = createLiveSnapshotWriter({
     productDir,
+    runId: lifecycle.runId,
     configs,
     trackers,
     startedAt,
     execution,
     workerState,
-    selection: {
-      typeValues,
-      suiteSelectors,
-      fileNames: requestedFiles,
-      shard: opts.shard || null,
-      serviceFilter: opts.serviceFilter || null,
-      scenarioSeed: opts.scenarioSeed || null,
-    },
+    selection: runSelection,
+    provenance,
     metadata,
     logRegistry,
     setupRegistry,
   });
-  const executedPlans = servicePlans.filter((plan) => !plan.skipped);
-  let exitCode = 0;
-  const lifecycle = createRunLifecycle(productDir);
   lifecycle.markRunning();
   lifecycle.installSignalHandlers();
   let results = [];
@@ -120,8 +125,11 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   try {
     if (executedPlans.length > 0) {
       const timings = loadTimings(productDir);
+      const history = loadHistory(productDir);
       const graphs = buildRuntimeGraphs(executedPlans);
-      const queue = buildTaskQueue(executedPlans, graphs, timings);
+      const queue = buildScheduledQueue(executedPlans, graphs, { timings, history });
+      runSelection.planning = buildRunPlanningMetadata(queue);
+      writeLiveSnapshot();
       reporter?.setTotalFileCount?.(queue.length);
       for (const task of queue) {
         task.scenarioSeed = opts.scenarioSeed || null;
@@ -191,6 +199,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     );
     const finalized = await finalizeRunArtifacts({
       productDir,
+      runId: lifecycle.runId,
       results,
       startedAt,
       finishedAt,
@@ -198,14 +207,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       workerCount: workerState.workerCount,
       runtimeInstanceCount: workerState.runtimeInstanceCount,
       runtimeStats: workerState.runtimeStats,
-      selection: {
-        typeValues,
-        suiteSelectors,
-        fileNames: requestedFiles,
-        shard: opts.shard || null,
-        serviceFilter: opts.serviceFilter || null,
-        scenarioSeed: opts.scenarioSeed || null,
-      },
+      selection: runSelection,
+      provenance,
       metadata,
       logRegistry,
       setupRegistry,
@@ -245,10 +248,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
 
 function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opts, execution, reporter) {
   return configs.map((config) => {
-    const suites = applyShard(
-      collectSuites(config, typeValues, suiteSelectors, opts.fileNames || [], opts),
-      opts.shard
-    );
+    const suites = collectSuites(config, typeValues, suiteSelectors, opts.fileNames || [], opts);
 
     if (suites.length === 0) {
       reporter?.serviceSkipped?.(

package/lib/runner/planning.mjs

@@ -1,4 +1,3 @@
-import { buildTimingKey, estimateTaskDuration } from "../timing/index.mjs";
 import {
   matchesSelectedTypes,
   matchesSuiteSelectors,
@@ -86,11 +85,6 @@ export function collectSuites(config, typeValues, suiteSelectors, fileNames = []
   return suites;
 }
 
-export function applyShard(suites, shard) {
-  if (!shard) return suites;
-  return suites.filter((_unused, index) => index % shard.total === shard.index - 1);
-}
-
 export function orderedTypes(types) {
   const ordered = [];
   for (const known of TYPE_ORDER) {
@@ -143,52 +137,6 @@ export function buildRuntimeGraphs(servicePlans) {
   }));
 }
 
-export function buildTaskQueue(servicePlans, graphs, timings) {
-  const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
-  const tasks = [];
-  let nextId = 1;
-
-  for (const plan of servicePlans) {
-    if (plan.skipped) continue;
-
-    const graph = graphByKey.get(plan.assignedGraphKey);
-    if (!graph) {
-      throw new Error(`Assigned graph "${plan.assignedGraphKey}" not found for ${plan.config.name}`);
-    }
-
-    for (const suite of plan.suites) {
-      for (const file of suite.files) {
-        const timingKey = buildTimingKey(plan.config.name, suite, file);
-        tasks.push({
-          id: nextId,
-          graphKey: graph.key,
-          targetName: plan.config.name,
-          serviceName: plan.config.name,
-          suiteKey: `${suite.type}:${suite.name}`,
-          suiteName: suite.name,
-          type: suite.type,
-          framework: suite.framework,
-          orderIndex: suite.orderIndex,
-          file,
-          locks: resolveTaskLocks(plan.config, suite, file),
-          resourceCost: 1,
-          timingKey,
-          estimatedDurationMs: estimateTaskDuration(timings, timingKey, suite),
-        });
-        nextId += 1;
-      }
-    }
-  }
-
-  return tasks.sort(
-    (a, b) =>
-      b.estimatedDurationMs - a.estimatedDurationMs ||
-      a.serviceName.localeCompare(b.serviceName) ||
-      a.suiteKey.localeCompare(b.suiteKey) ||
-      a.file.localeCompare(b.file)
-  );
-}
-
 export function claimNextTask(queue, preferredGraphKey, isRunnable = () => true) {
   if (queue.length === 0) return null;
 
@@ -259,29 +207,6 @@ function applySkipRules(config, displayType, suiteName, files, opts = {}) {
   };
 }
 
-function resolveTaskLocks(config, suite, file) {
-  const locks = new Set();
-  const matchedSuiteRules = config.testkit.requirements?.suites || [];
-  for (const rule of matchedSuiteRules) {
-    if (matchesSuiteSelectors(suite.displayType, suite.name, [rule.selector])) {
-      for (const lockName of rule.locks || []) {
-        locks.add(lockName);
-      }
-    }
-  }
-
-  const normalizedFile = normalizePathSeparators(file);
-  const fileMetadata = config.testkit.fileMetadataByPath?.get(normalizedFile) || null;
-  for (const lockName of config.testkit.requirements?.fileLocksByPath?.get(normalizedFile) || []) {
-    locks.add(lockName);
-  }
-  for (const lockName of fileMetadata?.locks || []) {
-    locks.add(lockName);
-  }
-
-  return [...locks].sort();
-}
-
 function normalizePathSeparators(filePath) {
   return String(filePath).split("\\").join("/");
 }

package/lib/runner/provenance.mjs

@@ -0,0 +1,20 @@
+const ASSISTANT_SESSION_ENV = "TESTKIT_ASSISTANT_SESSION_ID";
+const ASSISTANT_COMMAND_ID_ENV = "TESTKIT_ASSISTANT_COMMAND_ID";
+
+export function buildRunProvenance(env = process.env) {
+  const sessionId = normalizeOptionalString(env?.[ASSISTANT_SESSION_ENV]);
+  const commandId = normalizeOptionalString(env?.[ASSISTANT_COMMAND_ID_ENV]);
+  if (!sessionId && !commandId) return null;
+  return {
+    assistant: {
+      sessionId,
+      commandId,
+    },
+  };
+}
+
+function normalizeOptionalString(value) {
+  if (value == null) return null;
+  const stringValue = String(value).trim();
+  return stringValue || null;
+}

package/lib/runner/reporting.mjs

@@ -6,7 +6,6 @@ export function buildStatusArtifact({
   typeValues,
   suiteSelectors,
   fileNames,
-  shard,
   serviceFilter,
   scenarioSeed,
   metadata,
@@ -67,7 +66,6 @@ export function buildStatusArtifact({
     types: [...(typeValues || ["all"])].sort(),
     suiteSelectors: [...(suiteSelectors || [])].map((selector) => selector.raw).sort(),
     fileNames: [...(fileNames || [])].sort(),
-    shard: shard || null,
     serviceFilter: serviceFilter || null,
     scenarioSeed: scenarioSeed || null,
   };
@@ -76,11 +74,10 @@ export function buildStatusArtifact({
     scope.types[0] === "all" &&
     scope.suiteSelectors.length === 0 &&
     scope.fileNames.length === 0 &&
-    scope.shard === null &&
     scope.serviceFilter === null;
 
   return {
-    schemaVersion: 7,
+    schemaVersion: 8,
     source: "testkit",
     notice: "Generated file. Do not edit manually.",
     product: {
@@ -99,6 +96,7 @@ export function buildStatusArtifact({
 
 export function buildRunArtifact({
   productDir,
+  runId = null,
   results,
   startedAt,
   finishedAt,
@@ -109,9 +107,10 @@ export function buildRunArtifact({
   typeValues,
   suiteSelectors,
   fileNames,
-  shard,
   serviceFilter,
   scenarioSeed,
+  planning = null,
+  provenance = null,
   metadata,
   summarizeDbBackend,
   serviceLogs = [],
@@ -134,7 +133,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 9,
+    schemaVersion: 10,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {
@@ -144,6 +143,7 @@ export function buildRunArtifact({
     git: metadata.git,
     host: metadata.host,
     run: {
+      id: runId,
       status: runStatus || (failedServices.length > 0 ? "failed" : "passed"),
       startedAt: new Date(startedAt).toISOString(),
       finishedAt: new Date(finishedAt).toISOString(),
@@ -157,11 +157,12 @@ export function buildRunArtifact({
       types: typeValues,
       suiteSelectors: suiteSelectors.map((selector) => selector.raw),
       fileNames,
-      shard,
       serviceFilter,
       scenarioSeed: scenarioSeed || null,
       testkitVersion: metadata.testkitVersion,
     },
+    ...(provenance ? { provenance } : {}),
+    ...(planning ? { planning } : {}),
     summary: {
       services: {
         total: executed.length,
@@ -216,6 +217,7 @@ export function buildRunArtifact({
 
 export function buildLiveRunArtifact({
   productDir,
+  runId = null,
   results,
   startedAt,
   updatedAt,
@@ -226,9 +228,10 @@ export function buildLiveRunArtifact({
   typeValues,
   suiteSelectors,
   fileNames,
-  shard,
   serviceFilter,
   scenarioSeed,
+  planning = null,
+  provenance = null,
   metadata,
   summarizeDbBackend,
   serviceLogs = [],
@@ -237,6 +240,7 @@ export function buildLiveRunArtifact({
 }) {
   return buildRunArtifact({
     productDir,
+    runId,
     results,
     startedAt,
     finishedAt: updatedAt,
@@ -247,9 +251,10 @@ export function buildLiveRunArtifact({
     typeValues,
     suiteSelectors,
     fileNames,
-    shard,
     serviceFilter,
     scenarioSeed,
+    planning,
+    provenance,
     metadata,
     summarizeDbBackend,
     serviceLogs,

package/lib/runner/run-finalization.mjs

@@ -9,6 +9,7 @@ import { shouldFailRegressionSync, validateRegressionIssues } from "../regressio
 
 export async function finalizeRunArtifacts({
   productDir,
+  runId,
   results,
   startedAt,
   finishedAt,
@@ -17,6 +18,7 @@ export async function finalizeRunArtifacts({
   runtimeInstanceCount,
   runtimeStats,
   selection,
+  provenance = null,
   metadata,
   logRegistry,
   setupRegistry,
@@ -28,6 +30,7 @@ export async function finalizeRunArtifacts({
 }) {
   const runArtifact = buildRunArtifact({
     productDir,
+    runId,
     results,
     startedAt,
     finishedAt,
@@ -38,9 +41,10 @@ export async function finalizeRunArtifacts({
     typeValues: selection.typeValues,
     suiteSelectors: selection.suiteSelectors,
     fileNames: selection.fileNames,
-    shard: selection.shard,
     serviceFilter: selection.serviceFilter,
     scenarioSeed: selection.scenarioSeed,
+    planning: selection.planning || null,
+    provenance,
     metadata,
     summarizeDbBackend,
     serviceLogs: logRegistry.listServiceLogs(),
@@ -54,7 +58,6 @@ export async function finalizeRunArtifacts({
         typeValues: selection.typeValues,
         suiteSelectors: selection.suiteSelectors,
         fileNames: selection.fileNames,
-        shard: selection.shard,
         serviceFilter: selection.serviceFilter,
         scenarioSeed: selection.scenarioSeed,
         metadata,

package/lib/runner/run-guards.mjs

@@ -27,7 +27,6 @@ export function ensureStatusWriteAllowed(opts, typeValues, suiteSelectors, reque
       typeValues,
       suiteSelectors,
       requestedFiles,
-      opts.shard || null,
       opts.serviceFilter || null
     )
   ) {

package/lib/runner/scheduler/estimates.mjs

@@ -0,0 +1,61 @@
+export const DEFAULT_ESTIMATE_SOURCE = "default";
+
+export function estimateTask({ task, suite, timings, history } = {}) {
+  const timingEstimate = estimateFromTimings(timings, task);
+  if (timingEstimate) return timingEstimate;
+
+  const historyEstimate = estimateFromHistory(history, task);
+  if (historyEstimate) return historyEstimate;
+
+  return {
+    durationMs: estimateDefaultDuration(task, suite),
+    source: DEFAULT_ESTIMATE_SOURCE,
+    confidence: "low",
+    runs: 0,
+    key: null,
+  };
+}
+
+export function estimateDefaultDuration(task = {}, suite = {}) {
+  const base =
+    task.framework === "playwright" || suite.framework === "playwright"
+      ? 20_000
+      : task.type === "dal" || suite.type === "dal"
+        ? 4_000
+        : 8_000;
+  const suiteWeight = Number(suite.weight || 1);
+  const suiteFileCount = Math.max(1, Number(suite.files?.length || 1));
+  return Math.max(1_000, Math.round((base * suiteWeight) / suiteFileCount));
+}
+
+function estimateFromTimings(timings, task = {}) {
+  const candidates = [task.timingKey, ...(task.legacyTimingKeys || [])].filter(Boolean);
+  for (const key of candidates) {
+    const entry = timings?.files?.[key];
+    const durationMs = Number(entry?.durationMs || entry?.avgDurationMs || 0);
+    if (durationMs <= 0) continue;
+    const runs = Number(entry.runs || 0);
+    return {
+      durationMs: Math.max(1, Math.round(durationMs)),
+      source: "timings",
+      confidence: runs >= 3 ? "high" : "medium",
+      runs,
+      key,
+    };
+  }
+  return null;
+}
+
+function estimateFromHistory(history, task = {}) {
+  const entry = history?.tests?.[task.historyKey];
+  const durationMs = Number(entry?.avgDurationMs || 0);
+  if (durationMs <= 0) return null;
+  const runs = Number(entry.durationCount || entry.runCount || 0);
+  return {
+    durationMs: Math.max(1, Math.round(durationMs)),
+    source: "history",
+    confidence: runs >= 3 ? "medium" : "low",
+    runs,
+    key: task.historyKey,
+  };
+}

package/lib/runner/scheduler/identity.mjs

@@ -0,0 +1,31 @@
+import path from "path";
+
+export function normalizeTaskPath(filePath) {
+  return String(filePath || "").split(path.sep).join("/").replace(/^\.\/+/, "");
+}
+
+export function buildSchedulerTaskKey({ serviceName, displayType, type, framework, file } = {}) {
+  return [
+    serviceName || "service",
+    displayType || type || "unknown",
+    framework || "k6",
+    normalizeTaskPath(file),
+  ].join("|");
+}
+
+export function buildLegacyTimingKey({ serviceName, type, framework, file } = {}) {
+  return [
+    serviceName || "service",
+    framework || "k6",
+    type || "unknown",
+    normalizeTaskPath(file),
+  ].join("|");
+}
+
+export function buildSchedulerHistoryKey({ serviceName, displayType, type, file } = {}) {
+  return [
+    serviceName || "service",
+    displayType || type || "unknown",
+    normalizeTaskPath(file),
+  ].join("|");
+}

package/lib/runner/scheduler/index.mjs

@@ -0,0 +1,126 @@
+import {
+  buildLegacyTimingKey,
+  buildSchedulerHistoryKey,
+  buildSchedulerTaskKey,
+} from "./identity.mjs";
+import { estimateTask } from "./estimates.mjs";
+import { matchesSuiteSelectors } from "../suite-selection.mjs";
+
+export const SCHEDULER_POLICY = "longest-estimated-duration-first";
+export const SCHEDULER_VERSION = 1;
+
+export function buildScheduledQueue(servicePlans, graphs, { timings, history } = {}) {
+  const graphByKey = new Map(graphs.map((graph) => [graph.key, graph]));
+  const tasks = [];
+  let nextId = 1;
+
+  for (const plan of servicePlans) {
+    if (plan.skipped) continue;
+
+    const graph = graphByKey.get(plan.assignedGraphKey);
+    if (!graph) {
+      throw new Error(`Assigned graph "${plan.assignedGraphKey}" not found for ${plan.config.name}`);
+    }
+
+    for (const suite of plan.suites) {
+      for (const file of suite.files) {
+        const baseTask = {
+          id: nextId,
+          graphKey: graph.key,
+          targetName: plan.config.name,
+          serviceName: plan.config.name,
+          suiteKey: `${suite.type}:${suite.name}`,
+          suiteName: suite.name,
+          type: suite.type,
+          displayType: suite.displayType || suite.type,
+          framework: suite.framework,
+          orderIndex: suite.orderIndex,
+          file,
+          locks: resolveTaskLocks(plan.config, suite, file),
+          resourceCost: 1,
+        };
+        const timingKey = buildSchedulerTaskKey(baseTask);
+        const task = {
+          ...baseTask,
+          timingKey,
+          legacyTimingKeys: [buildLegacyTimingKey(baseTask)].filter((key) => key !== timingKey),
+          historyKey: buildSchedulerHistoryKey(baseTask),
+        };
+        const estimate = estimateTask({ task, suite, timings, history });
+        tasks.push({
+          ...task,
+          estimatedDurationMs: estimate.durationMs,
+          estimateSource: estimate.source,
+          estimateConfidence: estimate.confidence,
+          estimateRuns: estimate.runs,
+          estimateKey: estimate.key,
+        });
+        nextId += 1;
+      }
+    }
+  }
+
+  return tasks
+    .sort(compareScheduledTasks)
+    .map((task, index) => ({
+      ...task,
+      schedulerRank: index + 1,
+      schedulerPolicy: SCHEDULER_POLICY,
+    }));
+}
+
+export function buildRunPlanningMetadata(queue) {
+  return {
+    scheduler: {
+      policy: SCHEDULER_POLICY,
+      version: SCHEDULER_VERSION,
+    },
+    tasks: (queue || []).map((task) => ({
+      rank: task.schedulerRank || null,
+      id: task.id,
+      service: task.serviceName,
+      suite: task.suiteName,
+      type: task.displayType || task.type,
+      framework: task.framework,
+      file: task.file,
+      graphKey: task.graphKey,
+      estimatedDurationMs: task.estimatedDurationMs,
+      estimateSource: task.estimateSource,
+      estimateConfidence: task.estimateConfidence,
+      estimateRuns: task.estimateRuns || 0,
+      timingKey: task.timingKey,
+      historyKey: task.historyKey,
+    })),
+  };
+}
+
+export function compareScheduledTasks(a, b) {
+  return (
+    b.estimatedDurationMs - a.estimatedDurationMs ||
+    a.serviceName.localeCompare(b.serviceName) ||
+    a.suiteKey.localeCompare(b.suiteKey) ||
+    a.file.localeCompare(b.file)
+  );
+}
+
+function resolveTaskLocks(config, suite, file) {
+  const locks = new Set();
+  const matchedSuiteRules = config.testkit.requirements?.suites || [];
+  for (const rule of matchedSuiteRules) {
+    if (matchesSuiteSelectors(suite.displayType, suite.name, [rule.selector])) {
+      for (const lockName of rule.locks || []) {
+        locks.add(lockName);
+      }
+    }
+  }
+
+  const normalizedFile = String(file).split("\\").join("/");
+  const fileMetadata = config.testkit.fileMetadataByPath?.get(normalizedFile) || null;
+  for (const lockName of config.testkit.requirements?.fileLocksByPath?.get(normalizedFile) || []) {
+    locks.add(lockName);
+  }
+  for (const lockName of fileMetadata?.locks || []) {
+    locks.add(lockName);
+  }
+  return [...locks].sort();
+}

package/lib/runner/scheduler/observations.mjs

@@ -0,0 +1,27 @@
+export function buildTimingObservation(task, outcome = {}) {
+  const status = normalizeOutcomeStatus(outcome);
+  if (status === "skipped" || status === "not_run") return null;
+
+  const durationMs = Number(outcome.durationMs || 0);
+  if (durationMs <= 0) return null;
+
+  return {
+    key: task.timingKey,
+    legacyKeys: task.legacyTimingKeys || [],
+    durationMs,
+    status,
+    startedAt: outcome.startedAt || null,
+    finishedAt: outcome.finishedAt || null,
+    estimate: {
+      durationMs: task.estimatedDurationMs || null,
+      source: task.estimateSource || null,
+      confidence: task.estimateConfidence || null,
+    },
+  };
+}
+
+function normalizeOutcomeStatus(outcome = {}) {
+  if (outcome.status === "skipped") return "skipped";
+  if (outcome.status === "not_run") return "not_run";
+  return outcome.failed ? "failed" : "passed";
+}

package/lib/runner/selection.mjs

@@ -23,13 +23,12 @@ export function findUnmatchedRequestedFiles(
   );
 }
 
-export function isFullRunSelection(typeValues, suiteSelectors, fileNames, shard, serviceFilter) {
+export function isFullRunSelection(typeValues, suiteSelectors, fileNames, serviceFilter) {
   return (
     (typeValues || []).length === 1 &&
     typeValues[0] === "all" &&
     (suiteSelectors || []).length === 0 &&
     (fileNames || []).length === 0 &&
-    (shard || null) === null &&
     (serviceFilter || null) === null
   );
 }