@elench/testkit 0.1.41 → 0.1.43

package/lib/runner/runtime-manager.mjs

@@ -21,53 +21,56 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
   };
 
   return {
+    canAcquire(task) {
+      const pool = getPool(pools, graphByKey, task, productDir, lifecycle);
+      if (!locksAvailable(locks, task.locks || [])) {
+        return false;
+      }
+      return claimableRuntimeSlot(pool, task) !== null;
+    },
     async acquire(task) {
       const pool = getPool(pools, graphByKey, task, productDir, lifecycle);
+      if (lifecycle.isStopRequested()) {
+        throw lifecycle.signal.reason || new Error("testkit run interrupted");
+      }
 
-      while (true) {
-        if (lifecycle.isStopRequested()) {
-          throw lifecycle.signal.reason || new Error("testkit run interrupted");
-        }
-
-        const leaseId = `lease-${task.id}-${nextLeaseCounter}`;
-        nextLeaseCounter += 1;
+      const leaseId = `lease-${task.id}-${nextLeaseCounter}`;
+      nextLeaseCounter += 1;
 
-        if (!tryAcquireLocks(locks, task.locks || [], leaseId)) {
-          await runtimeHooks.sleep(10);
-          continue;
-        }
+      if (!tryAcquireLocks(locks, task.locks || [], leaseId)) {
+        throw new Error(`Task ${task.id} was claimed before its locks were available`);
+      }
 
-        const slot = claimRuntimeSlot(pool);
-        if (!slot) {
-          releaseLocks(locks, task.locks || [], leaseId);
-          await runtimeHooks.sleep(10);
-          continue;
-        }
+      const slot = claimRuntimeSlot(pool, task);
+      if (!slot) {
+        releaseLocks(locks, task.locks || [], leaseId);
+        throw new Error(`Task ${task.id} was claimed before runtime capacity was available`);
+      }
 
-        try {
-          const context = await getReadyContext(slot, productDir, task, lifecycle, runtimeHooks);
-          const leaseDir = path.join(context.runtimeDir, "leases", leaseId);
-          fs.mkdirSync(leaseDir, { recursive: true });
-          return {
-            leaseId,
-            leaseDir,
-            lockNames: task.locks || [],
-            slot,
-            context,
-          };
-        } catch (error) {
-          releaseRuntimeSlot(slot);
-          releaseLocks(locks, task.locks || [], leaseId);
-          cleanupLeaseDir({ leaseDir: path.join(slot.context?.runtimeDir || productDir, "leases", leaseId) });
-          await drainRuntimeSlot(slot, lifecycle, runtimeHooks);
-          throw error;
-        }
+      try {
+        const context = await getReadyContext(slot, productDir, task, lifecycle, runtimeHooks);
+        const leaseDir = path.join(context.runtimeDir, "leases", leaseId);
+        fs.mkdirSync(leaseDir, { recursive: true });
+        return {
+          leaseId,
+          leaseDir,
+          lockNames: task.locks || [],
+          resourceCost: task.resourceCost || 1,
+          slot,
+          context,
+        };
+      } catch (error) {
+        releaseRuntimeSlot(slot, task);
+        releaseLocks(locks, task.locks || [], leaseId);
+        cleanupLeaseDir({ leaseDir: path.join(slot.context?.runtimeDir || productDir, "leases", leaseId) });
+        await drainRuntimeSlot(slot, lifecycle, runtimeHooks);
+        throw error;
       }
     },
     async release(lease, options = {}) {
       if (!lease?.slot) return;
       releaseLocks(locks, lease.lockNames || [], lease.leaseId);
-      releaseRuntimeSlot(lease.slot);
+      releaseRuntimeSlot(lease.slot, { resourceCost: lease.resourceCost || 1 });
       cleanupLeaseDir(lease);
       if (options.invalidate) {
         lease.slot.draining = true;
@@ -84,6 +87,23 @@ export function createRuntimeManager({ productDir, graphs, lifecycle, hooks = {}
         }
       }
     },
+    getStats() {
+      return [...pools.values()]
+        .map((pool) => ({
+          graphKey: pool.slots[0]?.graph.key || null,
+          targetNames: [...(pool.slots[0]?.graph.targetNames || [])],
+          maxConcurrentTasks: Number.isFinite(pool.slots[0]?.graph.maxConcurrentTasks)
+            ? pool.slots[0]?.graph.maxConcurrentTasks
+            : null,
+          runtimeCount: pool.slots.length,
+          runtimes: pool.slots.map((slot) => ({
+            runtimeId: slot.runtimeId,
+            peakLeaseCount: slot.peakLeaseCount,
+            peakResourceUnits: slot.peakResourceUnits,
+          })),
+        }))
+        .sort((left, right) => String(left.graphKey).localeCompare(String(right.graphKey)));
+    },
   };
 }
 
@@ -105,6 +125,9 @@ function getPool(pools, graphByKey, task, productDir, lifecycle) {
       context: null,
       contextPromise: null,
       activeLeaseCount: 0,
+      activeResourceUnits: 0,
+      peakLeaseCount: 0,
+      peakResourceUnits: 0,
       draining: false,
     })),
   };
@@ -112,21 +135,30 @@ function getPool(pools, graphByKey, task, productDir, lifecycle) {
   return pool;
 }
 
-function claimRuntimeSlot(pool) {
+function claimRuntimeSlot(pool, task) {
+  const resourceCost = task.resourceCost || 1;
   const available = pool.slots.filter((slot) => !slot.draining);
   if (available.length === 0) return null;
 
-  const slot = [...available].sort(
-    (left, right) =>
-      left.activeLeaseCount - right.activeLeaseCount ||
-      left.runtimeId.localeCompare(right.runtimeId)
-  )[0];
+  const slot = [...available]
+    .filter((candidate) => slotHasCapacity(candidate, resourceCost))
+    .sort(
+      (left, right) =>
+        left.activeResourceUnits - right.activeResourceUnits ||
+        left.activeLeaseCount - right.activeLeaseCount ||
+        left.runtimeId.localeCompare(right.runtimeId)
+    )[0];
+  if (!slot) return null;
   slot.activeLeaseCount += 1;
+  slot.activeResourceUnits += resourceCost;
+  slot.peakLeaseCount = Math.max(slot.peakLeaseCount, slot.activeLeaseCount);
+  slot.peakResourceUnits = Math.max(slot.peakResourceUnits, slot.activeResourceUnits);
   return slot;
 }
 
-function releaseRuntimeSlot(slot) {
+function releaseRuntimeSlot(slot, task = {}) {
   slot.activeLeaseCount = Math.max(0, slot.activeLeaseCount - 1);
+  slot.activeResourceUnits = Math.max(0, slot.activeResourceUnits - (task.resourceCost || 1));
 }
 
 async function getReadyContext(slot, productDir, task, lifecycle, runtimeHooks) {
@@ -163,6 +195,21 @@ function tryAcquireLocks(lockMap, lockNames, leaseId) {
   return true;
 }
 
+function locksAvailable(lockMap, lockNames) {
+  return [...new Set(lockNames)].every((lockName) => !lockMap.has(lockName));
+}
+
+function claimableRuntimeSlot(pool, task) {
+  const resourceCost = task.resourceCost || 1;
+  return (
+    pool.slots.find((slot) => !slot.draining && slotHasCapacity(slot, resourceCost)) || null
+  );
+}
+
+function slotHasCapacity(slot, resourceCost) {
+  return slot.activeResourceUnits + resourceCost <= slot.graph.maxConcurrentTasks;
+}
+
 function releaseLocks(lockMap, lockNames, leaseId) {
   for (const lockName of [...new Set(lockNames)].sort()) {
     if (lockMap.get(lockName) === leaseId) {

package/lib/runner/runtime-manager.test.mjs

@@ -76,6 +76,7 @@ describe("runtime-manager", () => {
           dirName: "api",
           targetNames: ["api"],
           instanceCount: 2,
+          maxConcurrentTasks: 1,
         },
       ],
       hooks: makeHooks(events),
@@ -93,7 +94,7 @@ describe("runtime-manager", () => {
     await manager.cleanupAll();
   });
 
-  it("blocks conflicting locks until the first lease releases", async () => {
+  it("marks conflicting locks unavailable until the first lease releases", async () => {
     const productDir = makeTempDir("testkit-runtime-manager-");
     const manager = createRuntimeManager({
       productDir,
@@ -104,24 +105,17 @@ describe("runtime-manager", () => {
           dirName: "api",
           targetNames: ["api"],
           instanceCount: 1,
+          maxConcurrentTasks: 1,
         },
       ],
       hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
     });
 
     const firstLease = await manager.acquire(makeTask(1, { locks: ["shared-lock"] }));
-    let secondSettled = false;
-    const secondPromise = manager.acquire(makeTask(2, { locks: ["shared-lock"] })).then((lease) => {
-      secondSettled = true;
-      return lease;
-    });
-
-    await new Promise((resolve) => setTimeout(resolve, 20));
-    expect(secondSettled).toBe(false);
+    expect(manager.canAcquire(makeTask(2, { locks: ["shared-lock"] }))).toBe(false);
 
     await manager.release(firstLease);
-    const secondLease = await secondPromise;
-    expect(secondSettled).toBe(true);
+    const secondLease = await manager.acquire(makeTask(2, { locks: ["shared-lock"] }));
 
     await manager.release(secondLease);
     await manager.cleanupAll();
@@ -139,6 +133,7 @@ describe("runtime-manager", () => {
           dirName: "api",
           targetNames: ["api"],
           instanceCount: 1,
+          maxConcurrentTasks: 2,
         },
       ],
       hooks: makeHooks(events),
@@ -165,6 +160,7 @@ describe("runtime-manager", () => {
           dirName: "api",
           targetNames: ["api"],
           instanceCount: 1,
+          maxConcurrentTasks: 1,
         },
       ],
       hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
@@ -178,4 +174,33 @@ describe("runtime-manager", () => {
 
     await manager.cleanupAll();
   });
+
+  it("exposes runtime capacity through canAcquire", async () => {
+    const productDir = makeTempDir("testkit-runtime-manager-");
+    const manager = createRuntimeManager({
+      productDir,
+      lifecycle: makeLifecycle(),
+      graphs: [
+        {
+          key: "api",
+          dirName: "api",
+          targetNames: ["api"],
+          instanceCount: 1,
+          maxConcurrentTasks: 2,
+        },
+      ],
+      hooks: makeHooks({ created: [], ready: [], cleaned: [] }),
+    });
+
+    expect(manager.canAcquire(makeTask(1))).toBe(true);
+    const leaseOne = await manager.acquire(makeTask(1));
+    expect(manager.canAcquire(makeTask(2))).toBe(true);
+    const leaseTwo = await manager.acquire(makeTask(2));
+    expect(manager.canAcquire(makeTask(3))).toBe(false);
+
+    await manager.release(leaseOne);
+    expect(manager.canAcquire(makeTask(3))).toBe(true);
+    await manager.release(leaseTwo);
+    await manager.cleanupAll();
+  });
 });

package/lib/runner/services.mjs

@@ -1,7 +1,7 @@
 import { resolveServiceCwd } from "../config/index.mjs";
 import { buildExecutionEnv, numericPortFromUrl } from "./template.mjs";
 import { DEFAULT_READY_TIMEOUT_MS, assertLocalServicePortsAvailable, isPortInUse, waitForReady } from "./readiness.mjs";
-import { pipeOutput, startDetachedCommand, stopChildProcess, sleep } from "./processes.mjs";
+import { killChildProcess, pipeOutput, startDetachedCommand, stopChildProcess, sleep } from "./processes.mjs";
 import { readDatabaseUrl } from "./state-io.mjs";
 
 export async function startLocalServices(runtimeConfigs, lifecycle) {
@@ -43,7 +43,9 @@ export async function startLocalService(config, lifecycle) {
     pipeOutput(child.stdout, `[${config.runtimeLabel}:${config.name}]`),
     pipeOutput(child.stderr, `[${config.runtimeLabel}:${config.name}]`),
   ];
-  lifecycle.registerService(config, child, cwd);
+  lifecycle.registerService(config, child, cwd, () => {
+    killChildProcess(child, "SIGTERM");
+  });
 
   const readyTimeoutMs = config.testkit.local.readyTimeoutMs || DEFAULT_READY_TIMEOUT_MS;
 

package/lib/runner/triage.mjs

@@ -0,0 +1,330 @@
+import fs from "fs";
+import path from "path";
+
+const CLASSIFICATIONS = new Set([
+  "expected_failure",
+  "infra",
+  "product_bug",
+  "stale_test",
+  "test_bug",
+]);
+const STATES = new Set(["closed", "open"]);
+
+export function loadKnownFailuresConfig(productDir, config) {
+  const relativePath = config?.knownFailuresFile;
+  if (!relativePath) return null;
+
+  const absolutePath = path.resolve(productDir, relativePath);
+  if (!fs.existsSync(absolutePath)) {
+    throw new Error(`Known failures file not found: ${relativePath}`);
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(fs.readFileSync(absolutePath, "utf8"));
+  } catch (error) {
+    throw new Error(
+      `Could not parse known failures file ${relativePath}: ${formatErrorMessage(error)}`
+    );
+  }
+
+  return normalizeKnownFailuresDocument(parsed, relativePath);
+}
+
+export function applyKnownFailuresToArtifacts(runArtifact, statusArtifact, knownFailures) {
+  if (!knownFailures) return { runArtifact, statusArtifact };
+
+  const runEntries = extractRunFileEntries(runArtifact);
+  const statusEntries = extractStatusFileEntries(statusArtifact);
+  const fileSummaries = new Map();
+
+  for (const entry of [...runEntries, ...statusEntries]) {
+    const key = buildFileIdentity(entry.service, entry.type, entry.path);
+    if (!fileSummaries.has(key)) {
+      fileSummaries.set(key, {
+        service: entry.service,
+        type: entry.type,
+        path: entry.path,
+        status: entry.status,
+        error: entry.error || null,
+        failureDetails: Array.isArray(entry.failureDetails) ? entry.failureDetails : [],
+      });
+    }
+  }
+
+  const matchesByFileKey = new Map();
+  const matchedByFailedEntryIds = new Set();
+
+  for (const entry of knownFailures.entries) {
+    for (const fileSummary of fileSummaries.values()) {
+      if (!matchesKnownFailureEntry(entry, fileSummary)) continue;
+      const fileKey = buildFileIdentity(fileSummary.service, fileSummary.type, fileSummary.path);
+      if (!matchesByFileKey.has(fileKey)) {
+        matchesByFileKey.set(fileKey, []);
+      }
+      matchesByFileKey.get(fileKey).push(toArtifactTriageEntry(entry));
+      if (fileSummary.status === "failed") {
+        matchedByFailedEntryIds.add(entry.id);
+      }
+    }
+  }
+
+  for (const entry of [...runEntries, ...statusEntries]) {
+    const fileKey = buildFileIdentity(entry.service, entry.type, entry.path);
+    const matches = matchesByFileKey.get(fileKey) || [];
+    if (matches.length === 0) {
+      if (entry.status === "failed") {
+        setEntryTriage(entry, {
+          status: "untriaged",
+          entries: [],
+        });
+      }
+      continue;
+    }
+
+    setEntryTriage(entry, {
+      status: entry.status === "failed" ? "known_failure" : "known_issue_not_reproduced",
+      classifications: [...new Set(matches.map((match) => match.classification))].sort(),
+      entries: matches,
+    });
+  }
+
+  const summaryTests = statusArtifact?.tests || runEntries;
+  const triageSummary = buildTriageSummary(
+    summaryTests,
+    knownFailures.entries,
+    matchedByFailedEntryIds
+  );
+  runArtifact.triageSummary = triageSummary;
+  if (statusArtifact) {
+    statusArtifact.triageSummary = triageSummary;
+  }
+  return { runArtifact, statusArtifact };
+}
+
+export function normalizeKnownFailuresDocument(document, relativePath = "known failures file") {
+  if (!document || typeof document !== "object") {
+    throw new Error(`${relativePath} must contain a JSON object`);
+  }
+  if (document.schemaVersion !== 1) {
+    throw new Error(`${relativePath} schemaVersion must be 1`);
+  }
+  if (!Array.isArray(document.entries)) {
+    throw new Error(`${relativePath} entries must be an array`);
+  }
+
+  const ids = new Set();
+  const entries = document.entries.map((entry, index) => {
+    const normalized = normalizeKnownFailureEntry(entry, `${relativePath} entries[${index}]`);
+    if (ids.has(normalized.id)) {
+      throw new Error(`${relativePath} has duplicate entry id "${normalized.id}"`);
+    }
+    ids.add(normalized.id);
+    return normalized;
+  });
+
+  return {
+    schemaVersion: 1,
+    issueRepo: normalizeOptionalString(document.issueRepo),
+    entries,
+  };
+}
+
+function normalizeKnownFailureEntry(entry, label) {
+  if (!entry || typeof entry !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const id = requireNonEmptyString(entry.id, `${label}.id`);
+  const title = requireNonEmptyString(entry.title, `${label}.title`);
+  const classification = requireEnumValue(
+    entry.classification,
+    CLASSIFICATIONS,
+    `${label}.classification`
+  );
+  const state = requireEnumValue(entry.state, STATES, `${label}.state`);
+  const description = requireNonEmptyString(entry.description, `${label}.description`);
+  const whyFailing = requireNonEmptyString(entry.whyFailing, `${label}.whyFailing`);
+  const lastReviewedAt = requireNonEmptyString(entry.lastReviewedAt, `${label}.lastReviewedAt`);
+  if (!Array.isArray(entry.matches) || entry.matches.length === 0) {
+    throw new Error(`${label}.matches must be a non-empty array`);
+  }
+
+  return {
+    id,
+    title,
+    classification,
+    state,
+    issue: normalizeKnownFailureIssue(entry.issue, `${label}.issue`),
+    description,
+    whyFailing,
+    lastReviewedAt,
+    matches: entry.matches.map((match, index) =>
+      normalizeKnownFailureMatch(match, `${label}.matches[${index}]`)
+    ),
+  };
+}
+
+function normalizeKnownFailureIssue(issue, label) {
+  if (!issue || typeof issue !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const repo = requireNonEmptyString(issue.repo, `${label}.repo`);
+  const number = issue.number;
+  if (!Number.isInteger(number) || number <= 0) {
+    throw new Error(`${label}.number must be a positive integer`);
+  }
+  const url = requireNonEmptyString(issue.url, `${label}.url`);
+
+  return { repo, number, url };
+}
+
+function normalizeKnownFailureMatch(match, label) {
+  if (!match || typeof match !== "object") {
+    throw new Error(`${label} must be an object`);
+  }
+
+  const pathValue = requireNonEmptyString(match.path, `${label}.path`);
+  const normalized = {
+    path: pathValue,
+  };
+
+  const service = normalizeOptionalString(match.service);
+  if (service) normalized.service = service;
+
+  const type = normalizeOptionalString(match.type);
+  if (type) normalized.type = type;
+
+  const failureKey = normalizeOptionalString(match.failureKey);
+  if (failureKey) normalized.failureKey = failureKey;
+
+  const errorIncludes = normalizeOptionalString(match.errorIncludes);
+  if (errorIncludes) normalized.errorIncludes = errorIncludes;
+
+  return normalized;
+}
+
+function matchesKnownFailureEntry(entry, fileSummary) {
+  return entry.matches.some((match) => matchesKnownFailureMatch(match, fileSummary));
+}
+
+function matchesKnownFailureMatch(match, fileSummary) {
+  if (match.service && match.service !== fileSummary.service) return false;
+  if (match.type && match.type !== fileSummary.type) return false;
+  if (match.path !== fileSummary.path) return false;
+  if (match.failureKey) {
+    const failureKeys = Array.isArray(fileSummary.failureDetails)
+      ? fileSummary.failureDetails.map((detail) => detail.key)
+      : [];
+    if (!failureKeys.includes(match.failureKey)) return false;
+  }
+  if (match.errorIncludes && !String(fileSummary.error || "").includes(match.errorIncludes)) {
+    return false;
+  }
+  return true;
+}
+
+function toArtifactTriageEntry(entry) {
+  return {
+    id: entry.id,
+    title: entry.title,
+    classification: entry.classification,
+    state: entry.state,
+    issue: entry.issue,
+    description: entry.description,
+    whyFailing: entry.whyFailing,
+    lastReviewedAt: entry.lastReviewedAt,
+  };
+}
+
+function buildTriageSummary(tests, entries, matchedEntryIds) {
+  const failedTests = tests.filter((test) => test.status === "failed");
+  const knownFailedTests = failedTests.filter((test) => test.triage?.status === "known_failure");
+  const byClassification = {};
+
+  for (const test of knownFailedTests) {
+    for (const classification of test.triage?.classifications || []) {
+      byClassification[classification] = (byClassification[classification] || 0) + 1;
+    }
+  }
+
+  return {
+    failed: {
+      total: failedTests.length,
+      known: knownFailedTests.length,
+      untriaged: failedTests.length - knownFailedTests.length,
+      byClassification,
+    },
+    entries: {
+      total: entries.length,
+      matchedByFailedTests: matchedEntryIds.size,
+      unmatched: entries.length - matchedEntryIds.size,
+    },
+  };
+}
+
+function extractRunFileEntries(runArtifact) {
+  const entries = [];
+
+  for (const service of runArtifact.services || []) {
+    for (const suite of service.suites || []) {
+      for (const file of suite.files || []) {
+        entries.push({
+          target: file,
+          service: service.name,
+          type: suite.type,
+          path: file.path,
+          status: file.status,
+          error: file.error || null,
+          failureDetails: Array.isArray(file.failureDetails) ? file.failureDetails : [],
+        });
+      }
+    }
+  }
+
+  return entries;
+}
+
+function extractStatusFileEntries(statusArtifact) {
+  return statusArtifact?.tests || [];
+}
+
+function setEntryTriage(entry, triage) {
+  if (entry?.target) {
+    entry.target.triage = triage;
+    return;
+  }
+  entry.triage = triage;
+}
+
+function buildFileIdentity(service, type, filePath) {
+  return `${service}::${type}::${filePath}`;
+}
+
+function requireNonEmptyString(value, label) {
+  const normalized = normalizeOptionalString(value);
+  if (!normalized) {
+    throw new Error(`${label} must be a non-empty string`);
+  }
+  return normalized;
+}
+
+function requireEnumValue(value, allowed, label) {
+  const normalized = requireNonEmptyString(value, label);
+  if (!allowed.has(normalized)) {
+    throw new Error(`${label} must be one of: ${[...allowed].sort().join(", ")}`);
+  }
+  return normalized;
+}
+
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function formatErrorMessage(error) {
+  if (error instanceof Error) return error.message;
+  return String(error);
+}