@elench/testkit 0.1.43 → 0.1.45

package/lib/known-failures/github.mjs

@@ -0,0 +1,739 @@
+import fs from "fs";
+import path from "path";
+import { execFile } from "child_process";
+import { promisify } from "util";
+import { findMatchingKnownFailureEntries } from "./index.mjs";
+
+const execFileAsync = promisify(execFile);
+const DEFAULT_CACHE_TTL_SECONDS = 15 * 60;
+const CACHE_SCHEMA_VERSION = 1;
+const CACHE_PATH = [".testkit", "known-failures", "github-issues-cache.json"];
+const MODES = new Set(["off", "warn", "error"]);
+
+export function normalizeKnownFailureIssueValidationConfig(value) {
+  if (value == null) return null;
+  if (!value || typeof value !== "object") {
+    throw new Error("testkit.setup.ts reporting.issueValidation must be an object");
+  }
+
+  const provider = normalizeOptionalString(value.provider) || "github";
+  if (provider !== "github") {
+    throw new Error('testkit.setup.ts reporting.issueValidation.provider must be "github"');
+  }
+
+  const mode = normalizeOptionalString(value.mode) || "warn";
+  if (!MODES.has(mode)) {
+    throw new Error(
+      'testkit.setup.ts reporting.issueValidation.mode must be one of: off, warn, error'
+    );
+  }
+
+  const cacheTtlSeconds =
+    value.cacheTtlSeconds == null
+      ? DEFAULT_CACHE_TTL_SECONDS
+      : normalizePositiveInteger(
+          value.cacheTtlSeconds,
+          "testkit.setup.ts reporting.issueValidation.cacheTtlSeconds"
+        );
+
+  return {
+    provider,
+    mode,
+    cacheTtlSeconds,
+  };
+}
+
+export async function validateKnownFailureIssues({
+  productDir,
+  document,
+  runArtifact,
+  statusArtifact,
+  config,
+  gitMetadata = null,
+  transport = null,
+  now = Date.now(),
+}) {
+  const normalizedConfig = normalizeKnownFailureIssueValidationConfig(config);
+  if (!document || !normalizedConfig || normalizedConfig.mode === "off") return null;
+
+  const checks = collectObservedKnownFailureEntries(document, runArtifact, statusArtifact);
+  const issueNumbersByRepo = groupIssueNumbersByRepo(document.entries);
+  const repoSlug = gitMetadata?.repoSlug || null;
+  const remoteUrl = gitMetadata?.remoteUrl || null;
+  const cache = loadIssueCache(productDir);
+  const cacheResolution = resolveIssueCache(cache, issueNumbersByRepo, normalizedConfig, now);
+  const availabilityFindings = [];
+  let issuesByRepo = cacheResolution.issuesByRepo;
+
+  if (cacheResolution.missingByRepo.size > 0) {
+    const client = transport || (await createDefaultGitHubIssueTransport());
+    if (client) {
+      try {
+        const fetchedIssues = await fetchIssuesByRepo(client, cacheResolution.missingByRepo);
+        issuesByRepo = mergeIssuesByRepo(issuesByRepo, fetchedIssues);
+        updateIssueCache(cache, fetchedIssues, now);
+        writeIssueCache(productDir, cache);
+      } catch (error) {
+        issuesByRepo = applyStaleCacheFallback(
+          issuesByRepo,
+          cacheResolution.staleByRepo,
+          availabilityFindings
+        );
+        const severity = normalizedConfig.mode === "error" ? "error" : "warning";
+        availabilityFindings.push({
+          code: "validation_unavailable",
+          severity,
+          message: `GitHub issue validation unavailable: ${formatErrorMessage(error)}`,
+        });
+      }
+    } else {
+      issuesByRepo = applyStaleCacheFallback(
+        issuesByRepo,
+        cacheResolution.staleByRepo,
+        availabilityFindings
+      );
+      const severity = normalizedConfig.mode === "error" ? "error" : "warning";
+      availabilityFindings.push({
+        code: "validation_unavailable",
+        severity,
+        message:
+          "GitHub issue validation unavailable: set GH_TOKEN/GITHUB_TOKEN or install/authenticate gh",
+      });
+    }
+  }
+
+  const entries = document.entries.map((entry) => {
+    const observed = checks.get(entry.id) || createObservedCheck(entry);
+    const issueData = issuesByRepo.get(entry.issue.repo)?.get(entry.issue.number) || null;
+    return buildIssueValidationEntry({
+      entry,
+      observed,
+      issueData,
+    });
+  });
+
+  const globalFindings = [...availabilityFindings];
+  if (document.issueRepo && repoSlug && document.issueRepo !== repoSlug) {
+    globalFindings.push({
+      code: "detected_repo_mismatch",
+      severity: normalizedConfig.mode === "error" ? "error" : "warning",
+      message:
+        `Known failures issueRepo ${document.issueRepo} does not match detected repo ${repoSlug}` +
+        (remoteUrl ? ` (${remoteUrl})` : ""),
+    });
+  }
+
+  const summary = buildIssueValidationSummary(entries, globalFindings);
+  return {
+    schemaVersion: 1,
+    provider: "github",
+    mode: normalizedConfig.mode,
+    checkedAt: new Date(now).toISOString(),
+    repo: {
+      detected: repoSlug,
+      remoteUrl,
+      configured: document.issueRepo || null,
+    },
+    availability: {
+      hasFreshData: cacheResolution.missingByRepo.size === 0 || availabilityFindings.length === 0,
+      usedCachedFallback: availabilityFindings.some((finding) => finding.code === "used_stale_cache"),
+    },
+    findings: globalFindings,
+    summary,
+    entries,
+  };
+}
+
+export function shouldFailKnownFailureIssueValidation(result) {
+  if (!result || result.mode !== "error") return false;
+  return (result.summary?.errors || 0) > 0;
+}
+
+export function buildKnownFailureIssueValidationSummaryLines(result) {
+  if (!result) return [];
+
+  const parts = [];
+  const byCode = result.summary?.byCode || {};
+  if (byCode.closed_but_failing) {
+    parts.push(`${byCode.closed_but_failing} closed issue${pluralSuffix(byCode.closed_but_failing)} still failing`);
+  }
+  if (byCode.issue_missing) {
+    parts.push(`${byCode.issue_missing} missing issue reference${pluralSuffix(byCode.issue_missing)}`);
+  }
+  if (byCode.open_not_reproduced) {
+    parts.push(`${byCode.open_not_reproduced} open issue${pluralSuffix(byCode.open_not_reproduced)} not reproduced`);
+  }
+  if (byCode.closed_not_reproduced) {
+    parts.push(`${byCode.closed_not_reproduced} closed issue${pluralSuffix(byCode.closed_not_reproduced)} not reproduced`);
+  }
+  if (byCode.title_mismatch) {
+    parts.push(`${byCode.title_mismatch} title mismatch${pluralSuffix(byCode.title_mismatch)}`);
+  }
+  if (byCode.state_mismatch) {
+    parts.push(`${byCode.state_mismatch} state mismatch${pluralSuffix(byCode.state_mismatch)}`);
+  }
+  if (byCode.detected_repo_mismatch) {
+    parts.push(`${byCode.detected_repo_mismatch} repo mismatch${pluralSuffix(byCode.detected_repo_mismatch)}`);
+  }
+  if (byCode.validation_unavailable) {
+    parts.push("GitHub validation unavailable");
+  }
+  if (byCode.used_stale_cache) {
+    parts.push("used stale GitHub cache");
+  }
+
+  if (parts.length === 0) return [];
+
+  return [
+    "",
+    "Known-failure issues:",
+    ...parts.map((part) => `  - ${part}`),
+  ];
+}
+
+export function parseGitHubRepoSlug(remoteUrl) {
+  const normalized = normalizeOptionalString(remoteUrl);
+  if (!normalized) return null;
+
+  let match = normalized.match(/^https:\/\/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?$/i);
+  if (match) return `${match[1]}/${match[2]}`;
+
+  match = normalized.match(/^git@github\.com:([^/]+)\/([^/]+?)(?:\.git)?$/i);
+  if (match) return `${match[1]}/${match[2]}`;
+
+  match = normalized.match(/^ssh:\/\/git@github\.com\/([^/]+)\/([^/]+?)(?:\.git)?$/i);
+  if (match) return `${match[1]}/${match[2]}`;
+
+  return null;
+}
+
+async function createDefaultGitHubIssueTransport() {
+  const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN || null;
+  if (token) {
+    return {
+      type: "token",
+      async fetchRepoIssues(repo, numbers) {
+        return fetchRepoIssuesViaToken(repo, numbers, token);
+      },
+    };
+  }
+
+  try {
+    await execFileAsync("gh", ["auth", "status"], {
+      encoding: "utf8",
+      env: process.env,
+      maxBuffer: 1024 * 1024,
+    });
+    return {
+      type: "gh",
+      async fetchRepoIssues(repo, numbers) {
+        return fetchRepoIssuesViaGh(repo, numbers);
+      },
+    };
+  } catch {
+    return null;
+  }
+}
+
+async function fetchIssuesByRepo(client, issueNumbersByRepo) {
+  const issuesByRepo = new Map();
+
+  for (const [repo, numbers] of issueNumbersByRepo.entries()) {
+    const issueMap = new Map();
+    const chunks = chunkValues(numbers, 40);
+    for (const chunk of chunks) {
+      const fetched = await client.fetchRepoIssues(repo, chunk);
+      for (const [number, issue] of fetched.entries()) {
+        issueMap.set(number, issue);
+      }
+    }
+    issuesByRepo.set(repo, issueMap);
+  }
+
+  return issuesByRepo;
+}
+
+async function fetchRepoIssuesViaToken(repo, numbers, token) {
+  const query = buildIssueQuery(repo, numbers);
+  const response = await fetch("https://api.github.com/graphql", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/vnd.github+json",
+      Authorization: `Bearer ${token}`,
+      "X-GitHub-Api-Version": "2022-11-28",
+    },
+    body: JSON.stringify({ query }),
+  });
+
+  const payload = await response.json().catch(() => null);
+  if (!response.ok) {
+    throw new Error(
+      `GitHub GraphQL request failed with ${response.status}${
+        payload?.message ? `: ${payload.message}` : ""
+      }`
+    );
+  }
+  if (Array.isArray(payload?.errors) && payload.errors.length > 0) {
+    throw new Error(payload.errors.map((error) => error.message).join("; "));
+  }
+  return normalizeGraphqlIssuesResponse(repo, numbers, payload?.data);
+}
+
+async function fetchRepoIssuesViaGh(repo, numbers) {
+  const query = buildIssueQuery(repo, numbers);
+  const { stdout } = await execFileAsync(
+    "gh",
+    ["api", "graphql", "-f", `query=${query}`],
+    {
+      encoding: "utf8",
+      env: process.env,
+      maxBuffer: 1024 * 1024,
+    }
+  );
+  const payload = JSON.parse(stdout);
+  if (Array.isArray(payload?.errors) && payload.errors.length > 0) {
+    throw new Error(payload.errors.map((error) => error.message).join("; "));
+  }
+  return normalizeGraphqlIssuesResponse(repo, numbers, payload?.data);
+}
+
+function buildIssueQuery(repo, numbers) {
+  const [owner, name] = repo.split("/");
+  return `
+    query TestkitKnownFailureIssues {
+      repository(owner: ${JSON.stringify(owner)}, name: ${JSON.stringify(name)}) {
+        ${numbers
+          .map(
+            (number) => `
+          issue_${number}: issue(number: ${number}) {
+            number
+            title
+            state
+            url
+          }
+        `
+          )
+          .join("\n")}
+      }
+    }
+  `;
+}
+
+function normalizeGraphqlIssuesResponse(repo, numbers, data) {
+  const map = new Map();
+  const repository = data?.repository || null;
+  for (const number of numbers) {
+    const issue = repository?.[`issue_${number}`] || null;
+    if (!issue) {
+      map.set(number, {
+        repo,
+        number,
+        exists: false,
+        title: null,
+        state: null,
+        url: null,
+        checkedAt: null,
+        source: "github",
+      });
+      continue;
+    }
+
+    map.set(number, {
+      repo,
+      number: issue.number,
+      exists: true,
+      title: normalizeOptionalString(issue.title),
+      state: normalizeOptionalString(issue.state),
+      url: normalizeOptionalString(issue.url),
+      checkedAt: null,
+      source: "github",
+    });
+  }
+  return map;
+}
+
+function collectObservedKnownFailureEntries(document, runArtifact, statusArtifact) {
+  const observedTests = collectObservedTests(runArtifact, statusArtifact);
+  const checks = new Map(document.entries.map((entry) => [entry.id, createObservedCheck(entry)]));
+
+  for (const test of observedTests) {
+    const matchedEntries = findMatchingKnownFailureEntries(document, test);
+    for (const entry of matchedEntries) {
+      const observed = checks.get(entry.id) || createObservedCheck(entry);
+      observed.matchedTests += 1;
+      if (test.status === "failed") {
+        observed.failedTests += 1;
+      }
+      checks.set(entry.id, observed);
+    }
+  }
+
+  return checks;
+}
+
+function collectObservedTests(runArtifact, statusArtifact) {
+  if (Array.isArray(statusArtifact?.tests)) {
+    return statusArtifact.tests;
+  }
+
+  const tests = [];
+  for (const service of runArtifact?.services || []) {
+    for (const suite of service.suites || []) {
+      for (const file of suite.files || []) {
+        tests.push({
+          service: service.name,
+          type: suite.type,
+          path: file.path,
+          status: file.status,
+          error: file.error || null,
+          failureDetails: Array.isArray(file.failureDetails) ? file.failureDetails : [],
+        });
+      }
+    }
+  }
+  return tests;
+}
+
+function createObservedCheck(entry) {
+  return {
+    id: entry.id,
+    matchedTests: 0,
+    failedTests: 0,
+  };
+}
+
+function groupIssueNumbersByRepo(entries) {
+  const map = new Map();
+  for (const entry of entries) {
+    if (!map.has(entry.issue.repo)) {
+      map.set(entry.issue.repo, []);
+    }
+    const values = map.get(entry.issue.repo);
+    if (!values.includes(entry.issue.number)) {
+      values.push(entry.issue.number);
+    }
+  }
+
+  for (const numbers of map.values()) {
+    numbers.sort((left, right) => left - right);
+  }
+  return map;
+}
+
+function buildIssueValidationEntry({ entry, observed, issueData }) {
+  const findings = [];
+  const normalizedIssueState = normalizeIssueState(issueData?.state);
+
+  if (issueData && issueData.exists === false) {
+    findings.push({
+      code: "issue_missing",
+      severity: "error",
+      message: `Known failure ${entry.id} references missing issue #${entry.issue.number} in ${entry.issue.repo}`,
+    });
+  }
+
+  if (issueData?.exists && issueData.title && issueData.title !== entry.title) {
+    findings.push({
+      code: "title_mismatch",
+      severity: "error",
+      message: `Known failure ${entry.id} title does not match issue #${entry.issue.number}`,
+    });
+  }
+
+  if (issueData?.exists && normalizedIssueState && normalizedIssueState !== entry.state) {
+    findings.push({
+      code: "state_mismatch",
+      severity: "error",
+      message: `Known failure ${entry.id} state ${entry.state} does not match issue #${entry.issue.number} state ${normalizedIssueState}`,
+    });
+  }
+
+  if (issueData?.exists && observed.failedTests > 0 && normalizedIssueState === "closed") {
+    findings.push({
+      code: "closed_but_failing",
+      severity: "error",
+      message: `Known failure ${entry.id} still fails but issue #${entry.issue.number} is closed`,
+    });
+  }
+
+  if (issueData?.exists && observed.failedTests === 0 && observed.matchedTests > 0) {
+    if (normalizedIssueState === "open") {
+      findings.push({
+        code: "open_not_reproduced",
+        severity: "warning",
+        message: `Known failure ${entry.id} did not reproduce but issue #${entry.issue.number} is open`,
+      });
+    } else if (normalizedIssueState === "closed") {
+      findings.push({
+        code: "closed_not_reproduced",
+        severity: "warning",
+        message: `Known failure ${entry.id} did not reproduce and issue #${entry.issue.number} is closed`,
+      });
+    }
+  }
+
+  return {
+    id: entry.id,
+    title: entry.title,
+    issue: entry.issue,
+    observed: {
+      matchedTests: observed.matchedTests,
+      failedTests: observed.failedTests,
+      reproduced: observed.failedTests > 0,
+    },
+    github: issueData
+      ? {
+          exists: issueData.exists,
+          title: issueData.title,
+          state: issueData.state,
+          url: issueData.url,
+          checkedAt: issueData.checkedAt,
+          cached: issueData.source === "cache",
+        }
+      : {
+          exists: null,
+          title: null,
+          state: null,
+          url: null,
+          checkedAt: null,
+          cached: false,
+        },
+    findings,
+    status: resolveIssueValidationStatus(issueData, observed, findings),
+  };
+}
+
+function resolveIssueValidationStatus(issueData, observed, findings) {
+  if (!issueData) return observed.matchedTests > 0 ? "validation_unavailable" : "not_observed";
+  if (issueData.exists === false) return "issue_missing";
+  const normalizedState = normalizeIssueState(issueData.state);
+  if (observed.failedTests > 0 && normalizedState === "closed") return "closed_but_failing";
+  if (observed.failedTests > 0 && normalizedState === "open") return "open_and_failing";
+  if (observed.matchedTests > 0 && normalizedState === "open") return "open_not_reproduced";
+  if (observed.matchedTests > 0 && normalizedState === "closed") return "closed_not_reproduced";
+  if (findings.length > 0) return "metadata_mismatch";
+  return "not_observed";
+}
+
+function buildIssueValidationSummary(entries, globalFindings) {
+  const byCode = {};
+  const byStatus = {};
+  let errors = 0;
+  let warnings = 0;
+
+  for (const finding of globalFindings) {
+    byCode[finding.code] = (byCode[finding.code] || 0) + 1;
+    if (finding.severity === "error") errors += 1;
+    if (finding.severity === "warning") warnings += 1;
+  }
+
+  for (const entry of entries) {
+    byStatus[entry.status] = (byStatus[entry.status] || 0) + 1;
+    for (const finding of entry.findings) {
+      byCode[finding.code] = (byCode[finding.code] || 0) + 1;
+      if (finding.severity === "error") errors += 1;
+      if (finding.severity === "warning") warnings += 1;
+    }
+  }
+
+  return {
+    entries: entries.length,
+    observedEntries: entries.filter((entry) => entry.observed.matchedTests > 0).length,
+    errors,
+    warnings,
+    byCode,
+    byStatus,
+  };
+}
+
+function loadIssueCache(productDir) {
+  const filePath = getIssueCachePath(productDir);
+  if (!fs.existsSync(filePath)) {
+    return {
+      schemaVersion: CACHE_SCHEMA_VERSION,
+      entries: {},
+    };
+  }
+
+  try {
+    const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
+    if (parsed?.schemaVersion !== CACHE_SCHEMA_VERSION || typeof parsed.entries !== "object") {
+      return {
+        schemaVersion: CACHE_SCHEMA_VERSION,
+        entries: {},
+      };
+    }
+    return parsed;
+  } catch {
+    return {
+      schemaVersion: CACHE_SCHEMA_VERSION,
+      entries: {},
+    };
+  }
+}
+
+function resolveIssueCache(cache, issueNumbersByRepo, config, now) {
+  const issuesByRepo = new Map();
+  const missingByRepo = new Map();
+  const staleByRepo = new Map();
+  const ttlMs = config.cacheTtlSeconds * 1000;
+
+  for (const [repo, numbers] of issueNumbersByRepo.entries()) {
+    const cachedIssues = new Map();
+    const missing = [];
+    const stale = [];
+
+    for (const number of numbers) {
+      const key = buildIssueCacheKey(repo, number);
+      const cached = cache.entries[key];
+      if (!cached) {
+        missing.push(number);
+        continue;
+      }
+
+      const checkedAt = Date.parse(cached.checkedAt);
+      const normalized = {
+        repo,
+        number,
+        exists: Boolean(cached.exists),
+        title: normalizeOptionalString(cached.title),
+        state: normalizeOptionalString(cached.state),
+        url: normalizeOptionalString(cached.url),
+        checkedAt: cached.checkedAt || null,
+        source: "cache",
+      };
+
+      if (!Number.isFinite(checkedAt) || now - checkedAt > ttlMs) {
+        stale.push(number);
+        cachedIssues.set(number, normalized);
+        continue;
+      }
+
+      cachedIssues.set(number, normalized);
+    }
+
+    issuesByRepo.set(repo, cachedIssues);
+    if (missing.length > 0 || stale.length > 0) {
+      missingByRepo.set(repo, [...new Set([...missing, ...stale])].sort((a, b) => a - b));
+    }
+    if (stale.length > 0) {
+      staleByRepo.set(repo, stale.sort((a, b) => a - b));
+    }
+  }
+
+  return {
+    issuesByRepo,
+    missingByRepo,
+    staleByRepo,
+  };
+}
+
+function updateIssueCache(cache, issuesByRepo, now) {
+  cache.schemaVersion = CACHE_SCHEMA_VERSION;
+  cache.entries = cache.entries || {};
+  const checkedAt = new Date(now).toISOString();
+
+  for (const [repo, issues] of issuesByRepo.entries()) {
+    for (const [number, issue] of issues.entries()) {
+      issue.checkedAt = checkedAt;
+      cache.entries[buildIssueCacheKey(repo, number)] = {
+        exists: issue.exists,
+        title: issue.title,
+        state: issue.state,
+        url: issue.url,
+        checkedAt,
+      };
+    }
+  }
+}
+
+function writeIssueCache(productDir, cache) {
+  const filePath = getIssueCachePath(productDir);
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, `${JSON.stringify(cache, null, 2)}\n`);
+}
+
+function getIssueCachePath(productDir) {
+  return path.join(productDir, ...CACHE_PATH);
+}
+
+function applyStaleCacheFallback(issuesByRepo, staleByRepo, findings) {
+  let usedStale = false;
+  for (const [repo, numbers] of staleByRepo.entries()) {
+    const issues = issuesByRepo.get(repo);
+    if (!issues) continue;
+    for (const number of numbers) {
+      if (issues.has(number)) {
+        usedStale = true;
+      }
+    }
+  }
+  if (usedStale) {
+    findings.push({
+      code: "used_stale_cache",
+      severity: "warning",
+      message: "Used stale cached GitHub issue metadata because fresh validation failed",
+    });
+  }
+  return issuesByRepo;
+}
+
+function mergeIssuesByRepo(left, right) {
+  const merged = new Map();
+  for (const [repo, issues] of left.entries()) {
+    merged.set(repo, new Map(issues.entries()));
+  }
+
+  for (const [repo, issues] of right.entries()) {
+    if (!merged.has(repo)) {
+      merged.set(repo, new Map());
+    }
+    const mergedIssues = merged.get(repo);
+    for (const [number, issue] of issues.entries()) {
+      mergedIssues.set(number, issue);
+    }
+  }
+
+  return merged;
+}
+
+function buildIssueCacheKey(repo, number) {
+  return `${repo}#${number}`;
+}
+
+function normalizeIssueState(value) {
+  const normalized = normalizeOptionalString(value);
+  if (!normalized) return null;
+  return normalized.toLowerCase();
+}
+
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : null;
+}
+
+function normalizePositiveInteger(value, label) {
+  if (!Number.isInteger(value) || value <= 0) {
+    throw new Error(`${label} must be a positive integer`);
+  }
+  return value;
+}
+
+function chunkValues(values, size) {
+  const chunks = [];
+  for (let index = 0; index < values.length; index += size) {
+    chunks.push(values.slice(index, index + size));
+  }
+  return chunks;
+}
+
+function pluralSuffix(value) {
+  return value === 1 ? "" : "s";
+}
+
+function formatErrorMessage(error) {
+  if (error instanceof Error) return error.message;
+  return String(error);
+}