openings 0.1.0

package/src/source-enrichment.ts

@@ -0,0 +1,92 @@
+import { readFile } from "node:fs/promises";
+import { atomicJson } from "./atomic-file.ts";
+import { deriveLeadState, mergeEnrichmentLeads, readEnrichmentRegistry, retryDisposition, type CompanyMatch, type EnrichmentLead, type IdentityEvidence } from "./enrichment-registry.ts";
+import { stampReport, type ReportMeta } from "./report-meta.ts";
+import { mergeSourceCandidates } from "./source-discovery.ts";
+import { resolveSource } from "./source-verification.ts";
+import type { SourceCandidate } from "./types.ts";
+
+interface CompanySeed { companyName: string; companyDomain: string; reference: string }
+export interface SourceEnrichmentReport extends ReportMeta {
+  registryPath: string; companiesChecked: number; leadsChecked: number; matched: number; evidenceReady: number; rejected: number; promoted: number; candidatesPath: string;
+  companyInputs: string[];
+  states: Record<string, number>;
+  retry: Record<string, number>;
+  registryBytes: number; registryLockHeldMs: number; registryMergeDurationMs: number;
+}
+
+export async function enrichSourcesFromCompanies(registryPath: string, companiesPath: string | string[], candidatesPath: string, reportPath: string, options: { evidenceKind?: "authoritative_dataset" | "company_registry"; now?: Date } = {}): Promise<SourceEnrichmentReport> {
+  const now = options.now ?? new Date();
+  const companyInputs = Array.isArray(companiesPath) ? companiesPath : [companiesPath];
+  const registry = await readEnrichmentRegistry(registryPath);
+  const companies = await readCompanies(companyInputs);
+  const additions: EnrichmentLead[] = [];
+  for (const lead of registry.leads) {
+    if (!resolveSource(lead.sourceUrl)) continue;
+    const matches = companies.filter((company) => sourceMatchesCompany(lead.token, company));
+    const identities = new Set(matches.map((company) => company.companyDomain));
+    if (identities.size !== 1) continue;
+    const companyMatches: CompanyMatch[] = matches.map((company) => ({ companyName: company.companyName, companyDomain: company.companyDomain, method: "normalized_token", reference: company.reference }));
+    const identityEvidence: IdentityEvidence[] = options.evidenceKind ? matches.map((company) => ({ companyName: company.companyName, companyDomain: company.companyDomain, kind: options.evidenceKind!, reference: company.reference, observedAt: now.toISOString() })) : [];
+    additions.push({ ...lead, companyMatches, identityEvidence });
+  }
+  const merge = await mergeEnrichmentLeads(registryPath, additions, now);
+  const enriched = await readEnrichmentRegistry(registryPath);
+  const candidates: SourceCandidate[] = enriched.leads.flatMap((lead) => {
+    if (deriveLeadState(lead) !== "evidence_ready") return [];
+    if (["cooling_down", "repeatedly_failing"].includes(retryDisposition(lead, now))) return [];
+    const identity = winningIdentity(lead);
+    if (!identity) return [];
+    return [{ companyName: identity.companyName, companyDomain: identity.companyDomain, sourceUrl: lead.sourceUrl,
+      discoveredFrom: lead.discoveredFrom[0] ?? { channel: "dataset", reference: identity.reference },
+      domainEvidence: { kind: identity.kind === "provider_structured_domain" ? "authoritative_dataset" : identity.kind, reference: identity.reference } }];
+  });
+  const promoted = await mergeSourceCandidates(candidatesPath, candidates);
+  const states = countStates(enriched.leads);
+  const retry = countRetry(enriched.leads, now);
+  const report = stampReport("source-enrichment:1", 1, {
+    registryPath, companyInputs, companiesChecked: companies.length, leadsChecked: enriched.leads.length,
+    matched: states.matched ?? 0, evidenceReady: states.evidence_ready ?? 0, rejected: states.rejected ?? 0,
+    promoted, candidatesPath, states, retry, registryBytes: merge.bytes, registryLockHeldMs: merge.lockHeldMs, registryMergeDurationMs: merge.durationMs,
+  });
+  await atomicJson(reportPath, report);
+  return report;
+}
+
+function winningIdentity(lead: EnrichmentLead): IdentityEvidence | undefined {
+  const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect"].includes(evidence.kind));
+  const rank = { provider_structured_domain: 4, company_redirect: 3, company_registry: 2, authoritative_dataset: 1 } as const;
+  return [...qualifying].sort((left, right) => rank[right.kind] - rank[left.kind] || left.companyDomain.localeCompare(right.companyDomain))[0];
+}
+function countStates(leads: EnrichmentLead[]): Record<string, number> { const counts: Record<string, number> = {}; for (const lead of leads) { const state = deriveLeadState(lead); counts[state] = (counts[state] ?? 0) + 1; } return counts; }
+function countRetry(leads: EnrichmentLead[], now: Date): Record<string, number> { const counts: Record<string, number> = {}; for (const lead of leads) { const state = retryDisposition(lead, now); counts[state] = (counts[state] ?? 0) + 1; } return counts; }
+async function readCompanies(paths: string[]): Promise<CompanySeed[]> {
+  const companies: CompanySeed[] = [];
+  for (const path of paths) {
+    const value: unknown = JSON.parse(await readFile(path, "utf8"));
+    const catalog = !Array.isArray(value) && isRecord(value);
+    const rows = Array.isArray(value) ? value : catalog ? Object.values(value) : [];
+    if (!rows.every((company) => isRecord(company) && typeof (company.companyName ?? company.name) === "string" && String(company.companyName ?? company.name).trim() && typeof company.companyDomain === "string" && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(company.companyDomain))) {
+      throw new Error(`Company input ${path} must be a seed array or verified catalog with valid names and company domains`);
+    }
+    if (catalog && !rows.every(isVerifiedCatalogRecord)) {
+      throw new Error(`Company input ${path} is an object but does not contain verified catalog records`);
+    }
+    companies.push(...rows.map((company) => ({ companyName: String(company.companyName ?? company.name).trim(), companyDomain: String(company.companyDomain).toLowerCase().replace(/^www\./, ""), reference: path })));
+  }
+  return companies;
+}
+function isVerifiedCatalogRecord(company: unknown): boolean {
+  if (!isRecord(company) || typeof company.ats !== "string" || typeof company.token !== "string" || typeof company.sourceUrl !== "string" || !isRecord(company.verification)) return false;
+  const source = resolveSource(company.sourceUrl);
+  const verification = company.verification;
+  return source?.ats === company.ats && source.token.toLowerCase() === company.token.toLowerCase()
+    && verification.canonicalSourceUrl === source.canonicalSourceUrl
+    && typeof verification.checkedAt === "string" && Number.isFinite(Date.parse(verification.checkedAt))
+    && typeof verification.observedCompanyName === "string" && verification.observedCompanyName.length > 0
+    && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect"].includes(String(verification.identityEvidence))
+    && typeof verification.contentType === "string" && typeof verification.payloadVersion === "string" && verification.payloadVersion.length > 0
+    && Number.isInteger(verification.jobCount) && Number(verification.jobCount) > 0;
+}
+function sourceMatchesCompany(token: string, company: CompanySeed): boolean { const normalize = (value: string) => value.toLowerCase().replace(/\b(inc|llc|ltd|limited|corp|corporation|company)\b/g, "").replace(/[^a-z0-9]/g, ""); const tenant = token.includes("myworkdayjobs.com/") ? token.split("/")[1] : token; const source = normalize(tenant ?? token); const name = normalize(company.companyName); const domain = normalize(company.companyDomain.replace(/^www\./, "").split(".")[0] ?? ""); return source.length >= 3 && (source === name || source === domain); }
+function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }

package/src/source-pipeline.ts

@@ -0,0 +1,245 @@
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { atomicJson } from "./atomic-file.ts";
+import { withFileLock } from "./file-lock.ts";
+import { resolveSource, verifyCandidates } from "./source-verification.ts";
+import { deriveLeadState, mergeEnrichmentLeads, readEnrichmentRegistry, retryDisposition, strongestEvidenceRank, transientAttempt, type EnrichmentLead, type IdentityEvidence, type LeadAttempt } from "./enrichment-registry.ts";
+import type { RejectedSource, SourceCandidate, VerifiedCompany } from "./types.ts";
+import type { Ats } from "./types.ts";
+
+interface PipelineOptions {
+  fetch?: (input: string | URL, init?: RequestInit) => Promise<Response>;
+  now?: () => Date;
+  concurrency?: number;
+  timeoutMs?: number;
+  requireCountry?: string;
+  countryGateTimeoutMs?: number;
+  registryPath?: string;
+  retryDeferred?: boolean;
+  limit?: number;
+  providerConcurrency?: Partial<Record<Ats, number>>;
+  batchStatePath?: string;
+}
+
+export interface SourcePipelineReport {
+  candidates: number;
+  newCandidates: number;
+  selected: number;
+  selectedNew: number;
+  deferred: number;
+  verified: number;
+  rejected: number;
+  retryableFailures: number;
+  preserved: number;
+  carriedForward: number;
+  batchStatePath: string;
+  catalogPath: string;
+  rejections: RejectedSource[];
+}
+
+export async function runSourceVerification(candidatesPath: string, catalogPath: string, options: PipelineOptions = {}): Promise<SourcePipelineReport> {
+  await mkdir(dirname(catalogPath), { recursive: true });
+  const batchStatePath = options.batchStatePath ?? `${catalogPath}.verification-state.json`;
+  if (resolve(batchStatePath) === resolve(catalogPath)) throw new Error("Source verification batch state must not overwrite the catalog");
+  return withFileLock(batchStatePath,
+    () => withFileLock(catalogPath, () => runSourceVerificationUnlocked(candidatesPath, catalogPath, { ...options, batchStatePath }), { operation: "verify sources" }),
+    { operation: "schedule source verification batch" });
+}
+
+async function runSourceVerificationUnlocked(candidatesPath: string, catalogPath: string, options: PipelineOptions): Promise<SourcePipelineReport> {
+  const candidates = await readCandidates(candidatesPath);
+  const prior = await readPriorCatalog(catalogPath);
+  const batchStatePath = options.batchStatePath ?? `${catalogPath}.verification-state.json`;
+  const batchState = await readBatchState(batchStatePath);
+  const newCandidates = candidates.filter((candidate) => !candidateInCatalog(candidate, prior)).length;
+  let eligible = candidates;
+  const deferred: RejectedSource[] = [];
+  if (options.registryPath) {
+    const registry = await readEnrichmentRegistry(options.registryPath);
+    const byKey = new Map(registry.leads.map((lead) => [lead.sourceKey, lead]));
+    eligible = candidates.filter((candidate) => {
+      const source = resolveSource(candidate.sourceUrl);
+      const lead = source ? byKey.get(`${source.ats}:${source.token.toLowerCase()}`) : undefined;
+      if (!lead) return true;
+      const state = deriveLeadState(lead);
+      const retry = retryDisposition(lead, options.now?.() ?? new Date());
+      const providerCanAcquireIdentity = source?.ats === "recruitee" && state === "matched";
+      if ((["evidence_ready", "verified"].includes(state) || providerCanAcquireIdentity) && (options.retryDeferred || !["cooling_down", "repeatedly_failing"].includes(retry))) return true;
+      deferred.push({ ...candidate, reason: "unreachable", detail: `Verification deferred by enrichment policy (${state}, ${retry})` });
+      return false;
+    });
+  }
+  const limit = options.limit === undefined ? eligible.length : Math.max(1, Math.trunc(options.limit));
+  const selected = [...eligible].sort((left, right) => compareVerificationPriority(left, right, prior, batchState)).slice(0, limit);
+  const conflicts = selected.filter((candidate) => candidateConflictsCatalog(candidate, prior));
+  const verifiable = selected.filter((candidate) => !candidateConflictsCatalog(candidate, prior));
+  const result = await verifyCandidates(verifiable, options);
+  result.rejected.unshift(...conflicts.map((candidate) => ({ ...candidate, reason: "duplicate_slug" as const, detail: `Catalog slug ${candidateSlug(candidate)} belongs to a different verified company` })));
+  const retryableFailures = result.rejected.filter((candidate) => retryableRejection(candidate.reason)).length;
+  const freshlyVerified = new Set(result.verified.map((company) => company.slug));
+  const verifiedDomains = new Set(result.verified.map((company) => company.companyDomain));
+  const verifiedSources = new Set(result.verified.map((company) => `${company.ats}:${company.token.toLocaleLowerCase()}`));
+  const preserved = result.rejected.flatMap((candidate) => {
+    if (!preservableRejection(candidate.reason)) return [];
+    const slug = candidate.slug ?? slugFromDomain(candidate.companyDomain);
+    if (freshlyVerified.has(slug)) return [];
+    const previous = prior[slug];
+    if (!previous || previous.companyDomain !== candidate.companyDomain.toLocaleLowerCase()) return [];
+    if (verifiedDomains.has(previous.companyDomain) || verifiedSources.has(`${previous.ats}:${previous.token.toLocaleLowerCase()}`)) return [];
+    return [{ slug, ...previous } as VerifiedCompany];
+  });
+  const selectedSlugs = new Set(verifiable.flatMap((candidate) => { const slug = candidateSlug(candidate); return slug ? [slug] : []; }));
+  const untouched = Object.entries(prior).flatMap(([slug, company]) => selectedSlugs.has(slug) ? [] : [{ slug, ...company } as VerifiedCompany]);
+  const attemptedAt = (options.now?.() ?? new Date()).toISOString();
+  for (const candidate of selected) batchState.attempts[sourceKeyForCandidate(candidate)] = attemptedAt;
+  batchState.updatedAt = attemptedAt;
+  await atomicJson(batchStatePath, batchState);
+  await writeCatalog(catalogPath, [...result.verified, ...preserved, ...untouched]);
+  if (options.registryPath) await recordRegistryOutcomes(options.registryPath, result, options.now?.() ?? new Date());
+  return {
+    candidates: candidates.length,
+    newCandidates,
+    selected: selected.length,
+    selectedNew: selected.filter((candidate) => !candidateInCatalog(candidate, prior)).length,
+    deferred: deferred.length + Math.max(0, eligible.length - selected.length),
+    verified: result.verified.length,
+    rejected: result.rejected.length,
+    retryableFailures,
+    preserved: preserved.length,
+    carriedForward: untouched.length,
+    batchStatePath,
+    catalogPath,
+    rejections: result.rejected,
+  };
+}
+
+function candidateInCatalog(candidate: SourceCandidate, catalog: Record<string, Omit<VerifiedCompany, "slug">>): boolean {
+  if (typeof candidate.companyDomain !== "string" || typeof candidate.sourceUrl !== "string") return false;
+  const slug = candidateSlug(candidate);
+  if (!slug) return false;
+  const previous = catalog[slug];
+  return previous?.companyDomain === candidate.companyDomain.toLowerCase() && previous.sourceUrl === resolveSource(candidate.sourceUrl)?.canonicalSourceUrl;
+}
+
+interface VerificationBatchState { version: 1; updatedAt: string; attempts: Record<string, string> }
+
+function compareVerificationPriority(left: SourceCandidate, right: SourceCandidate, catalog: Record<string, Omit<VerifiedCompany, "slug">>, state: VerificationBatchState): number {
+  const leftAttempt = attemptTime(left, state);
+  const rightAttempt = attemptTime(right, state);
+  if (leftAttempt !== rightAttempt) return leftAttempt - rightAttempt;
+  const leftKnown = candidateInCatalog(left, catalog);
+  const rightKnown = candidateInCatalog(right, catalog);
+  if (leftKnown !== rightKnown) return Number(leftKnown) - Number(rightKnown);
+  if (!leftKnown) return 0;
+  return verificationTime(left, catalog) - verificationTime(right, catalog);
+}
+
+function attemptTime(candidate: SourceCandidate, state: VerificationBatchState): number {
+  const value = Date.parse(state.attempts[sourceKeyForCandidate(candidate)] ?? "");
+  return Number.isFinite(value) ? value : Number.NEGATIVE_INFINITY;
+}
+
+function verificationTime(candidate: SourceCandidate, catalog: Record<string, Omit<VerifiedCompany, "slug">>): number {
+  const slug = candidateSlug(candidate);
+  const value = slug ? Date.parse(catalog[slug]?.verification.checkedAt ?? "") : Number.NaN;
+  return Number.isFinite(value) ? value : Number.NEGATIVE_INFINITY;
+}
+
+function candidateConflictsCatalog(candidate: SourceCandidate, catalog: Record<string, Omit<VerifiedCompany, "slug">>): boolean {
+  if (typeof candidate.companyDomain !== "string" || typeof candidate.sourceUrl !== "string") return false;
+  const slug = candidateSlug(candidate);
+  if (!slug || !catalog[slug]) return false;
+  return !candidateInCatalog(candidate, catalog);
+}
+
+function candidateSlug(candidate: SourceCandidate): string | undefined {
+  if (typeof candidate.slug === "string" && candidate.slug) return candidate.slug;
+  return typeof candidate.companyDomain === "string" && candidate.companyDomain ? slugFromDomain(candidate.companyDomain) : undefined;
+}
+
+function sourceKeyForCandidate(candidate: SourceCandidate): string {
+  if (typeof candidate.sourceUrl === "string") {
+    const source = resolveSource(candidate.sourceUrl);
+    if (source) return `${source.ats}:${source.token.toLowerCase()}`;
+  }
+  return `candidate:${String(candidate.companyDomain ?? "").toLowerCase()}|${String(candidate.sourceUrl ?? "")}`;
+}
+
+async function readBatchState(path: string): Promise<VerificationBatchState> {
+  try {
+    const value: unknown = JSON.parse(await readFile(path, "utf8"));
+    if (typeof value === "object" && value !== null && !Array.isArray(value) && (value as { version?: unknown }).version === 1 && typeof (value as { attempts?: unknown }).attempts === "object" && (value as { attempts?: unknown }).attempts !== null && !Array.isArray((value as { attempts?: unknown }).attempts)) {
+      const updatedAt = (value as { updatedAt?: unknown }).updatedAt;
+      const entries = Object.entries((value as { attempts: Record<string, unknown> }).attempts);
+      if (typeof updatedAt === "string" && Number.isFinite(Date.parse(updatedAt)) && entries.every((entry): entry is [string, string] => entry[0].length > 0 && typeof entry[1] === "string" && Number.isFinite(Date.parse(entry[1])))) {
+        return { version: 1, updatedAt, attempts: Object.fromEntries(entries) };
+      }
+    }
+    throw new Error(`Invalid source verification batch state: ${path}`);
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") return { version: 1, updatedAt: new Date(0).toISOString(), attempts: {} };
+    if (error instanceof SyntaxError) throw new Error(`Invalid source verification batch state JSON: ${path}`);
+    throw error;
+  }
+}
+
+async function recordRegistryOutcomes(registryPath: string, result: Awaited<ReturnType<typeof verifyCandidates>>, now: Date): Promise<void> {
+  const registry = await readEnrichmentRegistry(registryPath);
+  const byKey = new Map(registry.leads.map((lead) => [lead.sourceKey, lead]));
+  const additions: EnrichmentLead[] = [];
+  for (const company of result.verified) {
+    const lead = byKey.get(`${company.ats}:${company.token.toLowerCase()}`);
+    if (!lead) continue;
+    const attempt: LeadAttempt = { attemptedAt: now.toISOString(), outcome: "success" };
+    const evidence: IdentityEvidence[] = company.verification.identityEvidence === "structured_domain_link"
+      ? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "provider_structured_domain", reference: company.sourceUrl, observedAt: now.toISOString() }]
+      : company.verification.identityEvidence === "company_redirect" && company.domainEvidence?.kind === "company_redirect"
+        ? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "company_redirect", reference: company.domainEvidence.reference, observedAt: now.toISOString() }]
+        : [];
+    additions.push({ ...lead, identityEvidence: evidence, attempts: [attempt], promotedAt: now.toISOString() });
+  }
+  for (const rejection of result.rejected) {
+    if (rejection.detail.startsWith("Verification deferred by enrichment policy")) continue;
+    const source = resolveSource(rejection.sourceUrl);
+    if (!source) continue;
+    const lead = byKey.get(`${source.ats}:${source.token.toLowerCase()}`);
+    if (!lead) continue;
+    const transient = retryableRejection(rejection.reason);
+    const consecutive = [...lead.attempts].reverse().findIndex((attempt) => attempt.outcome !== "transient_failure");
+    const attempt: LeadAttempt = transient
+      ? transientAttempt(now, consecutive < 0 ? lead.attempts.length + 1 : consecutive + 1, rejection.reason, rejection.detail)
+      : { attemptedAt: now.toISOString(), outcome: "permanent_failure", category: rejection.reason, detail: rejection.detail, evidenceRank: strongestEvidenceRank(lead) };
+    additions.push({ ...lead, attempts: [attempt] });
+  }
+  if (additions.length) await mergeEnrichmentLeads(registryPath, additions, now);
+}
+
+function preservableRejection(reason: RejectedSource["reason"]): boolean { return ["unreachable", "invalid_payload", "empty_board", "no_country_jobs"].includes(reason); }
+function retryableRejection(reason: RejectedSource["reason"]): boolean { return reason === "unreachable"; }
+
+async function readPriorCatalog(path: string): Promise<Record<string, Omit<VerifiedCompany, "slug">>> {
+  try {
+    const value: unknown = JSON.parse(await readFile(path, "utf8"));
+    return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, Omit<VerifiedCompany, "slug">> : {};
+  } catch { return {}; }
+}
+
+async function readCandidates(path: string): Promise<SourceCandidate[]> {
+  let value: unknown;
+  try { value = JSON.parse(await readFile(path, "utf8")); }
+  catch (error) { throw new Error(`Cannot read candidate file ${path}: ${error instanceof Error ? error.message : String(error)}`); }
+  if (!Array.isArray(value) || !value.every((candidate) => typeof candidate === "object" && candidate !== null && !Array.isArray(candidate))) {
+    throw new Error("Candidate file must contain a JSON array of objects");
+  }
+  return value as SourceCandidate[];
+}
+
+async function writeCatalog(path: string, companies: VerifiedCompany[]): Promise<void> {
+  const catalog = Object.fromEntries(companies.sort((a, b) => a.slug.localeCompare(b.slug)).map(({ slug, ...company }) => [slug, company]));
+  await mkdir(dirname(path), { recursive: true });
+  const temporary = `${path}.${process.pid}.tmp`;
+  await writeFile(temporary, `${JSON.stringify(catalog, null, 2)}\n`);
+  await rename(temporary, path);
+}
+
+function slugFromDomain(domain: string): string { return domain.toLocaleLowerCase().replace(/^www\./, "").split(".")[0]!.replace(/[^a-z0-9-]/g, "-"); }

package/src/source-verification.ts

@@ -0,0 +1,297 @@
+import type { Ats, RejectedSource, SourceCandidate, SourceRejectionReason, SourceVerificationResult, VerifiedCompany } from "./types.ts";
+import { fetchSafeHead, type HeadTransport, type ResolveHost } from "./safe-head.ts";
+import { abortableDelay, fetchSourceJobs, isTransientStatus, retryDelayMs } from "./catalog.ts";
+import { isEligibleForCountry } from "./locations.ts";
+
+type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
+
+interface VerificationOptions {
+  fetch?: Fetch;
+  now?: () => Date;
+  concurrency?: number;
+  timeoutMs?: number;
+  resolveHost?: ResolveHost;
+  headTransport?: HeadTransport;
+  requireCountry?: string;
+  countryGateTimeoutMs?: number;
+  providerConcurrency?: Partial<Record<Ats, number>>;
+}
+
+export interface ResolvedSource { ats: Ats; token: string; canonicalSourceUrl: string; structuredEndpoint: string }
+
+export async function verifyCandidates(candidates: SourceCandidate[], options: VerificationOptions = {}): Promise<SourceVerificationResult> {
+  const fetcher = options.fetch ?? globalThis.fetch;
+  const now = options.now ?? (() => new Date());
+  const concurrency = Math.max(1, Math.trunc(options.concurrency ?? 10));
+  const timeoutMs = Math.max(1, Math.trunc(options.timeoutMs ?? 30_000));
+  const countryGateTimeoutMs = Math.max(1, Math.trunc(options.countryGateTimeoutMs ?? 120_000));
+  const probed: Array<{ index: number; candidate: SourceCandidate; sourceKey: string; value: VerifiedCompany }> = [];
+  const rejected: Array<{ index: number; value: RejectedSource }> = [];
+  const providerLimits = new Map<Ats, Semaphore>();
+  const providerCooldowns = new Map<Ats, ProviderCooldown>();
+  for (const ats of ["greenhouse", "lever", "ashby", "workday", "recruitee"] as const) {
+    const fallback = ats === "workday" ? 2 : concurrency;
+    providerLimits.set(ats, new Semaphore(Math.max(1, Math.trunc(options.providerConcurrency?.[ats] ?? fallback))));
+    providerCooldowns.set(ats, new ProviderCooldown());
+  }
+  const scheduled = scheduleCandidates(candidates);
+  let cursor = 0;
+
+  async function worker() {
+    while (cursor < scheduled.length) {
+      const row = scheduled[cursor++];
+      const index = row?.index ?? -1;
+      const candidate = row?.candidate;
+      if (!candidate) continue;
+      const invalid = validateCandidate(candidate);
+      if (invalid) { rejected.push({ index, value: rejection(candidate, "invalid_candidate", invalid) }); continue; }
+      const source = resolveSource(candidate.sourceUrl);
+      if (!source) { rejected.push({ index, value: rejection(candidate, "unsupported_source", "URL is not a supported Greenhouse, Lever, Ashby, Workday, or Recruitee job source") }); continue; }
+      const sourceKey = `${source.ats}:${source.token.toLocaleLowerCase()}`;
+      const companyKey = candidate.companyDomain.toLocaleLowerCase();
+      const slug = candidate.slug ?? slugFromDomain(candidate.companyDomain);
+
+      const release = await providerLimits.get(source.ats)!.acquire();
+      const providerFetch = providerCooldowns.get(source.ats)!.wrap(fetcher);
+      try {
+        const evidence = await probe(candidate, source, providerFetch, timeoutMs, options.resolveHost, options.headTransport);
+        if (!identityMatches(candidate.companyName, candidate.companyDomain, evidence.observedCompanyName, source.token)) {
+          rejected.push({ index, value: rejection(candidate, "identity_mismatch", `Expected ${candidate.companyName}; observed ${evidence.observedCompanyName}`) });
+          continue;
+        }
+        const value: VerifiedCompany = {
+          slug, name: candidate.companyName.trim(), ats: source.ats, token: source.token,
+          cohorts: normalizeCohorts(candidate.cohorts), companyDomain: companyKey, sourceUrl: source.canonicalSourceUrl,
+          discoveredFrom: candidate.discoveredFrom,
+          domainEvidence: candidate.domainEvidence,
+          verification: { ...evidence, checkedAt: now().toISOString(), canonicalSourceUrl: source.canonicalSourceUrl },
+        };
+        if (options.requireCountry) {
+          const controller = new AbortController();
+          const timer = setTimeout(() => controller.abort(new Error(`Country gate timed out after ${countryGateTimeoutMs}ms`)), countryGateTimeoutMs);
+          try {
+            const jobs = await fetchSourceJobs(value, providerFetch, controller.signal);
+            if (!jobs.some((job) => isEligibleForCountry(job, options.requireCountry!))) {
+              rejected.push({ index, value: rejection(candidate, "no_country_jobs", `Complete source feed has no jobs eligible for ${options.requireCountry}`) });
+              continue;
+            }
+          } finally { clearTimeout(timer); }
+        }
+        probed.push({ index, candidate, sourceKey, value });
+      } catch (error) {
+        const reason = error instanceof VerificationError ? error.reason : "unreachable";
+        rejected.push({ index, value: rejection(candidate, reason, error instanceof Error ? error.message : String(error)) });
+      } finally { release(); }
+    }
+  }
+
+  await Promise.all(Array.from({ length: Math.min(concurrency, candidates.length) }, worker));
+  const verified: Array<{ index: number; value: VerifiedCompany }> = [];
+  const seenSources = new Set<string>();
+  const seenCompanies = new Set<string>();
+  const seenSlugs = new Set<string>();
+  for (const row of probed.sort((a, b) => a.index - b.index)) {
+    const companyKey = row.value.companyDomain;
+    if (seenSources.has(row.sourceKey)) rejected.push({ index: row.index, value: rejection(row.candidate, "duplicate_source", `Duplicate of ${row.sourceKey}`) });
+    else if (seenCompanies.has(companyKey)) rejected.push({ index: row.index, value: rejection(row.candidate, "duplicate_company", `Duplicate company domain: ${companyKey}`) });
+    else if (seenSlugs.has(row.value.slug)) rejected.push({ index: row.index, value: rejection(row.candidate, "duplicate_slug", `Generated catalog slug is already used: ${row.value.slug}`) });
+    else {
+      seenSources.add(row.sourceKey); seenCompanies.add(companyKey); seenSlugs.add(row.value.slug);
+      verified.push({ index: row.index, value: row.value });
+    }
+  }
+  return {
+    verified: verified.sort((a, b) => a.index - b.index).map((row) => row.value),
+    rejected: rejected.sort((a, b) => a.index - b.index).map((row) => row.value),
+  };
+}
+
+function scheduleCandidates(candidates: SourceCandidate[]): Array<{ index: number; candidate: SourceCandidate }> {
+  const groups = new Map<string, Array<{ index: number; candidate: SourceCandidate }>>();
+  candidates.forEach((candidate, index) => {
+    const provider = typeof candidate.sourceUrl === "string" ? resolveSource(candidate.sourceUrl)?.ats ?? "other" : "other";
+    const group = groups.get(provider) ?? [];
+    group.push({ index, candidate });
+    groups.set(provider, group);
+  });
+  const scheduled: Array<{ index: number; candidate: SourceCandidate }> = [];
+  while ([...groups.values()].some((group) => group.length)) {
+    for (const group of groups.values()) { const row = group.shift(); if (row) scheduled.push(row); }
+  }
+  return scheduled;
+}
+
+class Semaphore {
+  private active = 0;
+  private readonly waiters: Array<() => void> = [];
+  constructor(private readonly limit: number) {}
+  async acquire(): Promise<() => void> {
+    if (this.active >= this.limit) await new Promise<void>((resolve) => this.waiters.push(resolve));
+    this.active += 1;
+    return () => {
+      this.active -= 1;
+      this.waiters.shift()?.();
+    };
+  }
+}
+
+class ProviderCooldown {
+  private nextAllowedAt = 0;
+  wrap(fetcher: Fetch): Fetch {
+    return async (input, init) => {
+      await abortableDelay(Math.max(0, this.nextAllowedAt - Date.now()), init?.signal ?? undefined);
+      const response = await fetcher(input, init);
+      if (response.status === 429) this.nextAllowedAt = Math.max(this.nextAllowedAt, Date.now() + retryDelayMs(response, 0));
+      return response;
+    };
+  }
+}
+
+async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher: Fetch, timeoutMs: number, resolveHost?: ResolveHost, headTransport?: HeadTransport) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${timeoutMs}ms`)), timeoutMs);
+  const workday = source.ats === "workday" ? parseWorkdayToken(source.token) : undefined;
+  const endpoint = source.structuredEndpoint;
+  try {
+    const init = source.ats === "workday" ? { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ appliedFacets: {}, limit: 20, offset: 0, searchText: "" }), signal: controller.signal } : { signal: controller.signal };
+    const response = await fetchProbeWithRetry(fetcher, endpoint, init);
+    if (!response.ok) {
+      await response.body?.cancel().catch(() => undefined);
+      throw new VerificationError("unreachable", `HTTP ${response.status}`);
+    }
+    const contentType = response.headers.get("content-type") ?? "unknown";
+    let body: unknown;
+    try { body = await response.json(); } catch { throw new VerificationError("invalid_payload", "Endpoint did not return JSON"); }
+    const jobs = source.ats === "greenhouse" ? recordArray(body, "jobs") : source.ats === "lever" ? array(body) : source.ats === "workday" ? recordArray(body, "jobPostings") : source.ats === "recruitee" ? recordArray(body, "offers") : recordArray(body, "jobs");
+    if (!jobs) throw new VerificationError("invalid_payload", "Payload does not contain the expected jobs array");
+    if (jobs.length === 0) throw new VerificationError("empty_board", "Source has no jobs, so identity cannot be verified");
+    const providerName = source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
+    const hasDomainLink = !["greenhouse", "workday"].includes(source.ats) && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
+    const hasRedirectEvidence = !["greenhouse", "workday"].includes(source.ats) && await verifiedCompanyRedirect(candidate, source, resolveHost, headTransport, timeoutMs);
+    if (!["greenhouse", "workday"].includes(source.ats) && !hasDomainLink && !hasRedirectEvidence) throw new VerificationError("identity_mismatch", `Neither structured identity fields nor a verified company redirect link to ${candidate.companyDomain}`);
+    const observedCompanyName = providerName || candidate.companyName;
+    return {
+      observedCompanyName,
+      identityEvidence: source.ats === "workday" ? "provider_tenant" as const : (providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : "company_redirect" as const),
+      contentType,
+      payloadVersion: source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
+      jobCount: source.ats === "workday" && isRecord(body) && typeof body.total === "number" ? body.total : jobs.length,
+    };
+  } finally { clearTimeout(timer); }
+}
+
+async function fetchProbeWithRetry(fetcher: Fetch, endpoint: string, init: RequestInit): Promise<Response> {
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    const response = await fetcher(endpoint, init);
+    if (response.ok || !isTransientStatus(response.status) || attempt === 2) return response;
+    const delayMs = retryDelayMs(response, attempt);
+    await response.body?.cancel().catch(() => undefined);
+    await abortableDelay(delayMs, init.signal ?? undefined);
+  }
+  throw new Error("Verification probe exhausted retries");
+}
+
+export function resolveSource(value: string): ResolvedSource | null {
+  let url: URL;
+  try { url = new URL(value); } catch { return null; }
+  const parts = url.pathname.split("/").filter(Boolean);
+  const host = url.hostname.toLocaleLowerCase();
+  let ats: Ats | undefined;
+  let token: string | undefined;
+  if (host === "job-boards.greenhouse.io" || host === "boards.greenhouse.io") { ats = "greenhouse"; token = parts[0]; }
+  else if (host === "boards-api.greenhouse.io" && parts[0] === "v1" && parts[1] === "boards") { ats = "greenhouse"; token = parts[2]; }
+  else if (host === "jobs.lever.co") { ats = "lever"; token = parts[0]; }
+  else if (host === "jobs.ashbyhq.com") { ats = "ashby"; token = parts[0]; }
+  else if (/^[a-z0-9-]+\.recruitee\.com$/u.test(host)) {
+    const validPath = parts.length === 0 || parts.length === 2 && parts[0] === "o" || parts.join("/") === "api/offers";
+    if (!validPath) return null;
+    ats = "recruitee";
+    token = host.slice(0, -".recruitee.com".length);
+  }
+  else if (/\.myworkdayjobs\.com$/.test(host)) {
+    const cxs = parts.length === 5 && parts[0] === "wday" && parts[1] === "cxs" && parts[4] === "jobs";
+    const localeAndSite = /^[a-z]{2}-[a-z]{2}$/iu.test(parts[0] ?? "") && Boolean(parts[1]) && !parts[1]?.includes(".");
+    const board = localeAndSite && (parts.length === 2 || parts.length === 5 && parts[2] === "job");
+    if (!cxs && !board) return null;
+    const tenant = cxs ? parts[2] : host.split(".")[0];
+    const site = cxs ? parts[3] : parts[1];
+    if (tenant && site) { ats = "workday"; token = `${host}/${tenant}/${site}`; }
+  }
+  if (!ats || !token) return null;
+  const canonicalSourceUrl = ats === "greenhouse" ? `https://job-boards.greenhouse.io/${token}` : ats === "lever" ? `https://jobs.lever.co/${token}` : ats === "ashby" ? `https://jobs.ashbyhq.com/${token}` : ats === "recruitee" ? `https://${token}.recruitee.com` : (() => { const value = parseWorkdayToken(token); return `https://${value.host}/en-US/${value.site}`; })();
+  const structuredEndpoint = ats === "greenhouse"
+    ? `https://boards-api.greenhouse.io/v1/boards/${encodeURIComponent(token)}/jobs?content=true`
+    : ats === "lever"
+      ? `https://api.lever.co/v0/postings/${encodeURIComponent(token)}?mode=json`
+      : ats === "ashby"
+        ? `https://api.ashbyhq.com/posting-api/job-board/${encodeURIComponent(token)}`
+        : ats === "recruitee"
+          ? `https://${token}.recruitee.com/api/offers`
+          : (() => { const value = parseWorkdayToken(token); return `https://${value.host}/wday/cxs/${encodeURIComponent(value.tenant)}/${encodeURIComponent(value.site)}/jobs`; })();
+  return { ats, token, canonicalSourceUrl, structuredEndpoint };
+}
+
+function parseWorkdayToken(token: string): { host: string; tenant: string; site: string } {
+  const [host, tenant, site] = token.split("/");
+  if (!host || !tenant || !site) throw new Error("Invalid Workday token");
+  return { host, tenant, site };
+}
+
+function validateCandidate(candidate: SourceCandidate): string | null {
+  if (typeof candidate.companyName !== "string" || !candidate.companyName.trim()) return "companyName is required and must be a string";
+  if (candidate.slug !== undefined && (typeof candidate.slug !== "string" || !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(candidate.slug))) return "slug must contain lowercase letters, numbers, and single hyphens";
+  if (typeof candidate.companyDomain !== "string" || !/^[a-z0-9.-]+\.[a-z]{2,}$/i.test(candidate.companyDomain)) return "companyDomain must be a hostname";
+  if (typeof candidate.sourceUrl !== "string" || !candidate.sourceUrl) return "sourceUrl is required";
+  const channels = new Set(["search", "career_page", "provider_directory", "community", "dataset", "legacy"]);
+  if (!isRecord(candidate.discoveredFrom) || typeof candidate.discoveredFrom.channel !== "string" || !channels.has(candidate.discoveredFrom.channel) || typeof candidate.discoveredFrom.reference !== "string" || !candidate.discoveredFrom.reference.trim()) return "discoveredFrom must contain a supported channel and reference";
+  if (candidate.cohorts !== undefined && (!Array.isArray(candidate.cohorts) || !candidate.cohorts.every((code) => typeof code === "string"))) return "cohorts must be an array of country codes";
+  if (candidate.domainEvidence !== undefined) {
+    if (!isRecord(candidate.domainEvidence) || !["authoritative_dataset", "company_registry", "company_redirect"].includes(String(candidate.domainEvidence.kind)) || typeof candidate.domainEvidence.reference !== "string" || !candidate.domainEvidence.reference) return "domainEvidence must contain a supported kind and reference";
+  }
+  return null;
+}
+
+function identityMatches(expected: string, domain: string, observed: string, token: string): boolean {
+  const left = normalizeName(expected);
+  const right = normalizeName(observed);
+  const normalizedToken = normalizeName(token);
+  const domainLabel = normalizeName(domain.replace(/^www\./, "").split(".")[0] ?? "");
+  const sourceMatches = left === right || left.includes(right) || right.includes(left) || left.includes(normalizedToken) || normalizedToken.includes(left);
+  const domainMatches = left.includes(domainLabel) || domainLabel.includes(left);
+  return left.length >= 3 && domainLabel.length >= 3 && sourceMatches && domainMatches;
+}
+
+function normalizeName(value: string): string { return value.toLocaleLowerCase().replace(/\b(inc|llc|ltd|limited|corp|corporation|company)\b/g, "").replace(/[^a-z0-9]/g, ""); }
+function normalizeCohorts(value?: string[]): string[] | undefined { const result = [...new Set(value?.map((code) => code.toUpperCase()).filter((code) => /^[A-Z]{2}$/.test(code)) ?? [])]; return result.length ? result.sort() : undefined; }
+function slugFromDomain(domain: string): string { return domain.toLocaleLowerCase().replace(/^www\./, "").split(".")[0]!.replace(/[^a-z0-9-]/g, "-"); }
+function rejection(candidate: SourceCandidate, reason: SourceRejectionReason, detail: string): RejectedSource { return { ...candidate, reason, detail }; }
+function array(value: unknown): Record<string, unknown>[] | null { return Array.isArray(value) && value.every(isRecord) ? value : null; }
+function recordArray(value: unknown, key: string): Record<string, unknown>[] | null { return isRecord(value) ? array(value[key]) : null; }
+function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }
+function stringField(value: Record<string, unknown>, key: string): string { return typeof value[key] === "string" ? value[key] : ""; }
+function majority(values: string[]): string {
+  const counts = new Map<string, number>();
+  for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1);
+  return [...counts].sort((a, b) => b[1] - a[1])[0]?.[0] ?? "";
+}
+function structuredIdentityLinksDomain(jobs: Record<string, unknown>[], domain: string): boolean {
+  const escaped = domain.toLocaleLowerCase().replace(/^www\./, "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const pattern = new RegExp(`^https?://(?:www\\.)?${escaped}(?:/|$)`, "i");
+  const fields = ["companyUrl", "companyWebsite", "organizationUrl", "organizationWebsite", "website", "careers_url", "careers_apply_url"];
+  return jobs.some((job) => fields.some((field) => typeof job[field] === "string" && pattern.test(job[field])));
+}
+
+async function verifiedCompanyRedirect(candidate: SourceCandidate, expected: ResolvedSource, resolveHost?: ResolveHost, headTransport?: HeadTransport, timeoutMs?: number): Promise<boolean> {
+  if (candidate.domainEvidence?.kind !== "company_redirect") return false;
+  try {
+    const host = new URL(candidate.domainEvidence.reference).hostname.toLowerCase().replace(/^www\./, "");
+    const domain = candidate.companyDomain.toLowerCase().replace(/^www\./, "");
+    if (host !== domain && !host.endsWith(`.${domain}`)) return false;
+    const result = await fetchSafeHead(candidate.domainEvidence.reference, { resolveHost, transport: headTransport, timeoutMs });
+    const observed = resolveSource(result.finalUrl);
+    return result.response.ok && observed?.ats === expected.ats && observed.token.toLowerCase() === expected.token.toLowerCase();
+  } catch { return false; }
+}
+
+
+class VerificationError extends Error { constructor(readonly reason: SourceRejectionReason, message: string) { super(message); } }