@elench/testkit 0.1.65 → 0.1.67

package/lib/database/fingerprint.mjs

@@ -1,7 +1,7 @@
 import crypto from "crypto";
 import fs from "fs";
 import path from "path";
-import { resolveServiceCwd } from "../config/index.mjs";
+import { resolveServiceCwd } from "../config/paths.mjs";
 import { collectTemplateInputs } from "./template-steps.mjs";
 
 const LOCAL_IMAGE = "pgvector/pgvector:pg16";

package/lib/discovery/file-metadata.mjs

@@ -0,0 +1,122 @@
+import fs from "fs";
+import path from "path";
+import ts from "typescript";
+
+export function loadTestFileMetadataMap(productDir, suitesByService = {}) {
+  const metadataByPath = new Map();
+
+  for (const suites of Object.values(suitesByService || {})) {
+    for (const suiteList of Object.values(suites || {})) {
+      for (const suite of suiteList || []) {
+        for (const filePath of suite.files || []) {
+          if (metadataByPath.has(filePath)) continue;
+          metadataByPath.set(filePath, readTestFileMetadata(productDir, filePath));
+        }
+      }
+    }
+  }
+
+  return metadataByPath;
+}
+
+export function readTestFileMetadata(productDir, filePath) {
+  const absolutePath = path.join(productDir, filePath);
+  if (!fs.existsSync(absolutePath)) {
+    return emptyMetadata();
+  }
+
+  const sourceText = fs.readFileSync(absolutePath, "utf8");
+  const sourceFile = ts.createSourceFile(absolutePath, sourceText, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
+
+  for (const statement of sourceFile.statements) {
+    if (!ts.isVariableStatement(statement) || !hasExportModifier(statement)) continue;
+    for (const declaration of statement.declarationList.declarations) {
+      if (!ts.isIdentifier(declaration.name) || declaration.name.text !== "testkit") continue;
+      const config = parseMetadataInitializer(declaration.initializer);
+      if (config) return config;
+    }
+  }
+
+  return emptyMetadata();
+}
+
+function parseMetadataInitializer(initializer) {
+  if (!initializer) return null;
+  if (ts.isCallExpression(initializer)) {
+    const callee = getCallIdentifier(initializer.expression);
+    if (callee === "defineTestkitFile" && initializer.arguments[0] && ts.isObjectLiteralExpression(initializer.arguments[0])) {
+      return parseMetadataObject(initializer.arguments[0]);
+    }
+    return null;
+  }
+  if (ts.isObjectLiteralExpression(initializer)) {
+    return parseMetadataObject(initializer);
+  }
+  return null;
+}
+
+function parseMetadataObject(node) {
+  const metadata = emptyMetadata();
+
+  for (const property of node.properties) {
+    if (!ts.isPropertyAssignment(property)) continue;
+    const key = getPropertyName(property.name);
+    if (key === "locks") {
+      metadata.locks = parseLocks(property.initializer);
+      continue;
+    }
+    if (key === "skip") {
+      metadata.skipReason = parseSkipReason(property.initializer);
+    }
+  }
+
+  return metadata;
+}
+
+function parseLocks(node) {
+  if (!ts.isArrayLiteralExpression(node)) return [];
+  const values = [];
+  for (const entry of node.elements) {
+    const value = readStringLiteral(entry);
+    if (value) values.push(value);
+  }
+  return [...new Set(values)].sort();
+}
+
+function parseSkipReason(node) {
+  const literal = readStringLiteral(node);
+  if (literal) return literal;
+  if (!ts.isObjectLiteralExpression(node)) return null;
+  for (const property of node.properties) {
+    if (!ts.isPropertyAssignment(property)) continue;
+    if (getPropertyName(property.name) !== "reason") continue;
+    return readStringLiteral(property.initializer) || null;
+  }
+  return null;
+}
+
+function getCallIdentifier(node) {
+  if (ts.isIdentifier(node)) return node.text;
+  return null;
+}
+
+function getPropertyName(node) {
+  if (ts.isIdentifier(node) || ts.isStringLiteral(node)) return node.text;
+  return null;
+}
+
+function readStringLiteral(node) {
+  if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) return node.text;
+  return null;
+}
+
+function hasExportModifier(node) {
+  return (node.modifiers || []).some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword);
+}
+
+function emptyMetadata() {
+  return {
+    locks: [],
+    skipReason: null,
+  };
+}

package/lib/discovery/file-metadata.test.mjs

@@ -0,0 +1,51 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { readTestFileMetadata } from "./file-metadata.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  for (const dir of tempDirs.splice(0)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+describe("test file metadata", () => {
+  it("reads exported testkit metadata from object literals and helper calls", () => {
+    const productDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-file-meta-"));
+    tempDirs.push(productDir);
+    fs.mkdirSync(path.join(productDir, "__testkit__"), { recursive: true });
+    fs.writeFileSync(
+      path.join(productDir, "__testkit__", "billing.int.testkit.ts"),
+      [
+        'import { defineTestkitFile } from "@elench/testkit/setup";',
+        'export const testkit = defineTestkitFile({',
+        '  skip: "Billing is stubbed locally",',
+        '  locks: ["background-workers", "background-workers"],',
+        "});",
+      ].join("\n")
+    );
+
+    expect(readTestFileMetadata(productDir, "__testkit__/billing.int.testkit.ts")).toEqual({
+      skipReason: "Billing is stubbed locally",
+      locks: ["background-workers"],
+    });
+  });
+
+  it("returns empty metadata when no export is present", () => {
+    const productDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-file-meta-"));
+    tempDirs.push(productDir);
+    fs.mkdirSync(path.join(productDir, "__testkit__"), { recursive: true });
+    fs.writeFileSync(
+      path.join(productDir, "__testkit__", "health.int.testkit.ts"),
+      "export default {};\n"
+    );
+
+    expect(readTestFileMetadata(productDir, "__testkit__/health.int.testkit.ts")).toEqual({
+      skipReason: null,
+      locks: [],
+    });
+  });
+});

package/lib/discovery/index.mjs

@@ -207,8 +207,16 @@ function buildResolvedSuiteEntries(config, suite, internalType, filters) {
 
   for (const filePath of suite.files || []) {
     if (filters.fileNameSet.size > 0 && !filters.fileNameSet.has(filePath)) continue;
-    const fileLocks = config.testkit.requirements?.fileLocksByPath?.get(filePath) || [];
-    const skipReason = config.testkit.skip?.fileReasonByPath?.get(filePath) || suiteSkipReason || null;
+    const fileMetadata = config.testkit.fileMetadataByPath?.get(filePath) || { locks: [], skipReason: null };
+    const fileLocks = [
+      ...(config.testkit.requirements?.fileLocksByPath?.get(filePath) || []),
+      ...(fileMetadata.locks || []),
+    ];
+    const skipReason =
+      fileMetadata.skipReason ||
+      config.testkit.skip?.fileReasonByPath?.get(filePath) ||
+      suiteSkipReason ||
+      null;
     const skipped = Boolean(skipReason);
     if (filters.runnableOnly && skipped) continue;
     visibleFiles.push({

package/lib/discovery/index.test.mjs

@@ -30,14 +30,6 @@ describe("public discovery", () => {
                 start: "node server.js",
                 baseUrl: "http://127.0.0.1:3000",
                 readyUrl: "http://127.0.0.1:3000"
-              },
-              requirements: {
-                files: [
-                  {
-                    path: "src/api/routes/__testkit__/agent-configs-auth-gate.int.testkit.ts",
-                    locks: ["route-lock"]
-                  }
-                ]
               }
             },
             frontend: {
@@ -47,22 +39,30 @@ describe("public discovery", () => {
                 baseUrl: "http://127.0.0.1:3001",
                 readyUrl: "http://127.0.0.1:3001"
               },
-              dependsOn: ["api"],
-              skip: {
-                files: [
-                  {
-                    path: "frontend/src/app/login/__testkit__/auth.pw.testkit.ts",
-                    reason: "Auth is stubbed locally"
-                  }
-                ]
-              }
+              dependsOn: ["api"]
             }
           }
         });
       `
     );
-    writeFile(productDir, "src/api/routes/__testkit__/agent-configs-auth-gate.int.testkit.ts");
-    writeFile(productDir, "frontend/src/app/login/__testkit__/auth.pw.testkit.ts");
+    writeFile(
+      productDir,
+      "src/api/routes/__testkit__/agent-configs-auth-gate.int.testkit.ts",
+      [
+        'import { defineTestkitFile } from "@elench/testkit/setup";',
+        'export const testkit = defineTestkitFile({ locks: ["route-lock"] });',
+        "export {};",
+      ].join("\n")
+    );
+    writeFile(
+      productDir,
+      "frontend/src/app/login/__testkit__/auth.pw.testkit.ts",
+      [
+        'import { defineTestkitFile } from "@elench/testkit/setup";',
+        'export const testkit = defineTestkitFile({ skip: "Auth is stubbed locally" });',
+        "export {};",
+      ].join("\n")
+    );
 
     saveHistory(productDir, {
       version: 1,

package/lib/known-failures/github-cache.mjs

@@ -0,0 +1,159 @@
+import fs from "fs";
+import path from "path";
+
+export function loadIssueCache(productDir, cachePath, schemaVersion) {
+  const filePath = path.join(productDir, ...cachePath);
+  if (!fs.existsSync(filePath)) {
+    return {
+      schemaVersion,
+      entries: {},
+    };
+  }
+
+  try {
+    const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
+    if (parsed?.schemaVersion !== schemaVersion || typeof parsed.entries !== "object") {
+      return {
+        schemaVersion,
+        entries: {},
+      };
+    }
+    return parsed;
+  } catch {
+    return {
+      schemaVersion,
+      entries: {},
+    };
+  }
+}
+
+export function resolveIssueCache(cache, issueNumbersByRepo, config, now) {
+  const issuesByRepo = new Map();
+  const missingByRepo = new Map();
+  const staleByRepo = new Map();
+  const ttlMs = config.cacheTtlSeconds * 1000;
+
+  for (const [repo, numbers] of issueNumbersByRepo.entries()) {
+    const cachedIssues = new Map();
+    const missing = [];
+    const stale = [];
+
+    for (const number of numbers) {
+      const key = buildIssueCacheKey(repo, number);
+      const cached = cache.entries[key];
+      if (!cached) {
+        missing.push(number);
+        continue;
+      }
+
+      const checkedAt = Date.parse(cached.checkedAt);
+      const normalized = {
+        repo,
+        number,
+        exists: Boolean(cached.exists),
+        title: normalizeOptionalString(cached.title),
+        state: normalizeOptionalString(cached.state),
+        url: normalizeOptionalString(cached.url),
+        checkedAt: cached.checkedAt || null,
+        source: "cache",
+      };
+
+      if (!Number.isFinite(checkedAt) || now - checkedAt > ttlMs) {
+        stale.push(number);
+        cachedIssues.set(number, normalized);
+        continue;
+      }
+
+      cachedIssues.set(number, normalized);
+    }
+
+    issuesByRepo.set(repo, cachedIssues);
+    if (missing.length > 0 || stale.length > 0) {
+      missingByRepo.set(repo, [...new Set([...missing, ...stale])].sort((a, b) => a - b));
+    }
+    if (stale.length > 0) {
+      staleByRepo.set(repo, stale.sort((a, b) => a - b));
+    }
+  }
+
+  return {
+    issuesByRepo,
+    missingByRepo,
+    staleByRepo,
+  };
+}
+
+export function updateIssueCache(cache, issuesByRepo, now, schemaVersion) {
+  cache.schemaVersion = schemaVersion;
+  cache.entries = cache.entries || {};
+  const checkedAt = new Date(now).toISOString();
+
+  for (const [repo, issues] of issuesByRepo.entries()) {
+    for (const [number, issue] of issues.entries()) {
+      issue.checkedAt = checkedAt;
+      cache.entries[buildIssueCacheKey(repo, number)] = {
+        exists: issue.exists,
+        title: issue.title,
+        state: issue.state,
+        url: issue.url,
+        checkedAt,
+      };
+    }
+  }
+}
+
+export function writeIssueCache(productDir, cachePath, cache) {
+  const filePath = path.join(productDir, ...cachePath);
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, `${JSON.stringify(cache, null, 2)}\n`);
+}
+
+export function applyStaleCacheFallback(issuesByRepo, staleByRepo, findings) {
+  let usedStale = false;
+  for (const [repo, numbers] of staleByRepo.entries()) {
+    const issues = issuesByRepo.get(repo);
+    if (!issues) continue;
+    for (const number of numbers) {
+      if (issues.has(number)) {
+        usedStale = true;
+      }
+    }
+  }
+  if (usedStale) {
+    findings.push({
+      code: "used_stale_cache",
+      severity: "warning",
+      message: "Used stale cached GitHub issue metadata because fresh validation failed",
+    });
+  }
+  return issuesByRepo;
+}
+
+export function mergeIssuesByRepo(left, right) {
+  const merged = new Map();
+  for (const [repo, issues] of left.entries()) {
+    merged.set(repo, new Map(issues.entries()));
+  }
+
+  for (const [repo, issues] of right.entries()) {
+    if (!merged.has(repo)) {
+      merged.set(repo, new Map());
+    }
+    const mergedIssues = merged.get(repo);
+    for (const [number, issue] of issues.entries()) {
+      mergedIssues.set(number, issue);
+    }
+  }
+
+  return merged;
+}
+
+function buildIssueCacheKey(repo, number) {
+  return `${repo}#${number}`;
+}
+
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}

package/lib/known-failures/github-transport.mjs

@@ -0,0 +1,174 @@
+import { execFile } from "child_process";
+import { promisify } from "util";
+
+const execFileAsync = promisify(execFile);
+
+export function parseGitHubRepoSlug(remoteUrl) {
+  const normalized = normalizeOptionalString(remoteUrl);
+  if (!normalized) return null;
+
+  let match = normalized.match(/^https:\/\/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?$/i);
+  if (match) return `${match[1]}/${match[2]}`;
+
+  match = normalized.match(/^git@github\.com:([^/]+)\/([^/]+?)(?:\.git)?$/i);
+  if (match) return `${match[1]}/${match[2]}`;
+
+  match = normalized.match(/^ssh:\/\/git@github\.com\/([^/]+)\/([^/]+?)(?:\.git)?$/i);
+  if (match) return `${match[1]}/${match[2]}`;
+
+  return null;
+}
+
+export async function createDefaultGitHubIssueTransport(env = process.env) {
+  const token = env.GH_TOKEN || env.GITHUB_TOKEN || null;
+  if (token) {
+    return {
+      type: "token",
+      async fetchRepoIssues(repo, numbers) {
+        return fetchRepoIssuesViaToken(repo, numbers, token);
+      },
+    };
+  }
+
+  try {
+    await execFileAsync("gh", ["auth", "status"], {
+      encoding: "utf8",
+      env,
+      maxBuffer: 1024 * 1024,
+    });
+    return {
+      type: "gh",
+      async fetchRepoIssues(repo, numbers) {
+        return fetchRepoIssuesViaGh(repo, numbers, env);
+      },
+    };
+  } catch {
+    return null;
+  }
+}
+
+export async function fetchIssuesByRepo(client, issueNumbersByRepo) {
+  const issuesByRepo = new Map();
+
+  for (const [repo, numbers] of issueNumbersByRepo.entries()) {
+    const issueMap = new Map();
+    const chunks = chunkValues(numbers, 40);
+    for (const chunk of chunks) {
+      const fetched = await client.fetchRepoIssues(repo, chunk);
+      for (const [number, issue] of fetched.entries()) {
+        issueMap.set(number, issue);
+      }
+    }
+    issuesByRepo.set(repo, issueMap);
+  }
+
+  return issuesByRepo;
+}
+
+async function fetchRepoIssuesViaToken(repo, numbers, token) {
+  const query = buildIssueQuery(repo, numbers);
+  const response = await fetch("https://api.github.com/graphql", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/vnd.github+json",
+      Authorization: `Bearer ${token}`,
+      "X-GitHub-Api-Version": "2022-11-28",
+    },
+    body: JSON.stringify({ query }),
+  });
+
+  const payload = await response.json().catch(() => null);
+  if (!response.ok) {
+    throw new Error(
+      `GitHub GraphQL request failed with ${response.status}${payload?.message ? `: ${payload.message}` : ""}`
+    );
+  }
+  if (Array.isArray(payload?.errors) && payload.errors.length > 0) {
+    throw new Error(payload.errors.map((error) => error.message).join("; "));
+  }
+  return normalizeGraphqlIssuesResponse(repo, numbers, payload?.data);
+}
+
+async function fetchRepoIssuesViaGh(repo, numbers, env) {
+  const query = buildIssueQuery(repo, numbers);
+  const { stdout } = await execFileAsync("gh", ["api", "graphql", "-f", `query=${query}`], {
+    encoding: "utf8",
+    env,
+    maxBuffer: 1024 * 1024,
+  });
+  const payload = JSON.parse(stdout);
+  if (Array.isArray(payload?.errors) && payload.errors.length > 0) {
+    throw new Error(payload.errors.map((error) => error.message).join("; "));
+  }
+  return normalizeGraphqlIssuesResponse(repo, numbers, payload?.data);
+}
+
+function buildIssueQuery(repo, numbers) {
+  const [owner, name] = repo.split("/");
+  return `
+    query TestkitKnownFailureIssues {
+      repository(owner: ${JSON.stringify(owner)}, name: ${JSON.stringify(name)}) {
+        ${numbers
+          .map(
+            (number) => `
+          issue_${number}: issue(number: ${number}) {
+            number
+            title
+            state
+            url
+          }
+        `
+          )
+          .join("\n")}
+      }
+    }
+  `;
+}
+
+function normalizeGraphqlIssuesResponse(repo, numbers, data) {
+  const map = new Map();
+  const repository = data?.repository || null;
+  for (const number of numbers) {
+    const issue = repository?.[`issue_${number}`] || null;
+    if (!issue) {
+      map.set(number, {
+        repo,
+        number,
+        exists: false,
+        title: null,
+        state: null,
+        url: null,
+        checkedAt: null,
+        source: "github",
+      });
+      continue;
+    }
+
+    map.set(number, {
+      repo,
+      number: issue.number,
+      exists: true,
+      title: normalizeOptionalString(issue.title),
+      state: normalizeOptionalString(issue.state),
+      url: normalizeOptionalString(issue.url),
+      checkedAt: null,
+      source: "github",
+    });
+  }
+  return map;
+}
+
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function chunkValues(values, size) {
+  const chunks = [];
+  for (let index = 0; index < values.length; index += size) {
+    chunks.push(values.slice(index, index + size));
+  }
+  return chunks;
+}