openings 0.1.0

package/src/safe-head.ts

@@ -0,0 +1,79 @@
+import { lookup } from "node:dns/promises";
+import { request } from "node:https";
+import { isIP } from "node:net";
+import { connect as connectTls } from "node:tls";
+
+export type ResolveHost = (hostname: string) => Promise<string[]>;
+export type HeadTransport = (url: URL, address: string, signal: AbortSignal) => Promise<Response>;
+export interface SafeHeadResult { response: Response; finalUrl: string }
+
+export async function fetchSafeHead(url: string, options: { resolveHost?: ResolveHost; transport?: HeadTransport; timeoutMs?: number } = {}): Promise<SafeHeadResult> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${options.timeoutMs ?? 15_000}ms`)), options.timeoutMs ?? 15_000);
+  try {
+    let current = new URL(url);
+    for (let redirects = 0; redirects <= 5; redirects += 1) {
+      const address = await withAbort(publicAddress(current, options.resolveHost ?? resolveAddresses), controller.signal);
+      const response = await withAbort((options.transport ?? pinnedHead)(current, address, controller.signal), controller.signal);
+      if (![301, 302, 303, 307, 308].includes(response.status)) return { response, finalUrl: current.href };
+      const location = response.headers.get("location");
+      if (!location) return { response, finalUrl: current.href };
+      current = new URL(location, current);
+      if (current.protocol !== "https:") throw new Error("Career redirect must use HTTPS");
+    }
+    throw new Error("Career redirect limit exceeded");
+  } finally { clearTimeout(timer); }
+}
+
+function withAbort<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {
+  if (signal.aborted) return Promise.reject(signal.reason);
+  return new Promise((resolve, reject) => {
+    const abort = () => reject(signal.reason);
+    signal.addEventListener("abort", abort, { once: true });
+    operation.then(
+      (value) => { signal.removeEventListener("abort", abort); resolve(value); },
+      (error) => { signal.removeEventListener("abort", abort); reject(error); },
+    );
+  });
+}
+
+async function resolveAddresses(hostname: string): Promise<string[]> {
+  return (await lookup(hostname, { all: true, verbatim: true })).map(({ address }) => address);
+}
+
+async function publicAddress(url: URL, resolveHost: ResolveHost): Promise<string> {
+  if (url.protocol !== "https:") throw new Error("Career URL must use HTTPS");
+  const addresses = isIP(url.hostname) ? [url.hostname] : await resolveHost(url.hostname);
+  const publicAddresses = addresses.filter(isPublicAddress);
+  if (!addresses.length || publicAddresses.length !== addresses.length) throw new Error(`Career URL resolves to a non-public address: ${url.hostname}`);
+  return publicAddresses[0]!;
+}
+
+function pinnedHead(url: URL, address: string, signal: AbortSignal): Promise<Response> {
+  return new Promise((resolve, reject) => {
+    const req = request(url, {
+      method: "HEAD", signal, servername: url.hostname, agent: false,
+      createConnection: () => {
+        const socket = connectTls({ host: address, port: Number(url.port || 443), servername: url.hostname });
+        const destroy = () => socket.destroy(signal.reason instanceof Error ? signal.reason : undefined);
+        if (signal.aborted) destroy();
+        else signal.addEventListener("abort", destroy, { once: true });
+        return socket;
+      },
+    }, (response) => resolve(new Response(null, { status: response.statusCode ?? 500, headers: response.headers as HeadersInit })));
+    req.once("error", reject);
+    req.end();
+  });
+}
+
+function isPublicAddress(address: string): boolean {
+  if (isIP(address) === 4) {
+    const [a, b] = address.split(".").map(Number);
+    return !(a === 0 || a === 10 || a === 127 || (a === 100 && b! >= 64 && b! <= 127) || (a === 169 && b === 254) || (a === 172 && b! >= 16 && b! <= 31) || (a === 192 && (b === 0 || b === 168)) || (a === 198 && (b === 18 || b === 19 || b === 51)) || (a === 203 && b === 0) || a! >= 224);
+  }
+  if (isIP(address) === 6) {
+    const value = address.toLowerCase();
+    return !(value === "::" || value === "::1" || value.startsWith("::ffff:") || value.startsWith("fc") || value.startsWith("fd") || /^fe[89ab]/.test(value) || value.startsWith("ff") || value.startsWith("2001:db8"));
+  }
+  return false;
+}

package/src/screening-requirements.ts

@@ -0,0 +1,99 @@
+import type { CandidateProfile } from "./candidate-profile.ts";
+import type { Job } from "./types.ts";
+
+export interface ScreeningRequirement {
+  kind: "experience" | "education";
+  requirement: string;
+  status: "supported" | "partial" | "unsupported";
+  factIds: string[];
+}
+
+export function evaluateScreeningRequirements(profile: CandidateProfile, job: Job): ScreeningRequirement[] {
+  const description = normalizeStructuredText(job.description);
+  const requirements = [
+    ...experienceRequirements(profile, description),
+    ...educationRequirements(profile, description),
+  ];
+  return [...new Map(requirements.map((item) => [`${item.kind}:${item.requirement.toLocaleLowerCase()}`, item])).values()];
+}
+
+function normalizeStructuredText(value: string): string {
+  return value
+    .replace(/<br\s*\/?>/gi, "\n")
+    .replace(/<\/h[1-6]>/gi, "\n")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&nbsp;|&#160;|\u00a0/gi, " ")
+    .replace(/&amp;/gi, "&")
+    .replace(/[ \t]+/g, " ");
+}
+
+function experienceRequirements(profile: CandidateProfile, description: string): ScreeningRequirement[] {
+  const pattern = /\b(\d+)\s*\+?\s*years?\s+(?:of\s+)?(?:[A-Za-z][A-Za-z/-]*\s+){0,6}?experience\b/gi;
+  return [...description.matchAll(pattern)].flatMap<ScreeningRequirement>((match) => {
+    if (!isMandatory(description, match.index!, match[0])) return [];
+    const requirement = clean(match[0]);
+    const inference = profile.inferences.find((value) => value.kind === "approximate_experience_years");
+    if (!inference) return [{ kind: "experience", requirement, status: "unsupported", factIds: [] }];
+    return [{
+      kind: "experience",
+      requirement,
+      status: inference.value >= Number(match[1]) ? "supported" : "partial",
+      factIds: inference.derivedFromFactIds,
+    }];
+  });
+}
+
+function educationRequirements(profile: CandidateProfile, description: string): ScreeningRequirement[] {
+  const fieldPattern = /\b((?:bachelor|master)(?:['’]s)?(?:\s+or\s+(?:bachelor|master)(?:['’]s)?)*)\s+degree\s+in\s+([^.\n]+)/gi;
+  const fieldSpecific = [...description.matchAll(fieldPattern)].flatMap((match) => {
+    if (!isMandatory(description, match.index!, match[0])) return [];
+    const levels = [...match[1]!.matchAll(/bachelor|master/gi)].map((value) => value[0]!.toLocaleLowerCase());
+    const fact = profile.facts.find((value) => value.kind === "education" && levels.some((level) => educationLevelMatches(level, value.value)) && educationFieldMatches(match[2]!, value.value));
+    return [{ kind: "education", requirement: clean(match[0]), status: fact ? "supported" : "unsupported", factIds: fact ? [fact.id] : [] } satisfies ScreeningRequirement];
+  });
+  const genericPattern = /\b(bachelor(?:['’]s)?|master(?:['’]s)?|ph\.?d\.?|doctorate)\s+degree\b[^.\n]{0,30}\b(?:required|minimum|must)\b/gi;
+  const generic = [...description.matchAll(genericPattern)].flatMap<ScreeningRequirement>((match) => {
+    if (!isMandatory(description, match.index!, match[0])) return [];
+    const level = match[1]!.toLocaleLowerCase();
+    const fact = profile.facts.find((value) => value.kind === "education" && educationLevelMatches(level, value.value));
+    const requirement = level.startsWith("bachelor") ? "Bachelor's degree" : level.startsWith("master") ? "Master's degree" : "Doctoral degree";
+    return [{ kind: "education", requirement, status: fact ? "supported" : "unsupported", factIds: fact ? [fact.id] : [] }];
+  });
+  return [...fieldSpecific, ...generic];
+}
+
+function isMandatory(description: string, start: number, value: string): boolean {
+  const clauseEnd = description.slice(start).search(/[.\n]/);
+  const clause = description.slice(start, clauseEnd < 0 ? undefined : start + clauseEnd);
+  const rawPrefix = description.slice(Math.max(0, start - 40), start);
+  const prefix = rawPrefix.slice(Math.max(rawPrefix.lastIndexOf("."), rawPrefix.lastIndexOf("\n")) + 1);
+  const context = `${prefix} ${value} ${clause}`;
+  if (/\b(?:preferred|optional|desired|nice to have|not required|up to)\b/i.test(context)) return false;
+  if (/\b(?:required|must|minimum|at least)\b/i.test(context)) return true;
+  return lastSectionKind(description.slice(0, start)) === "required";
+}
+
+function lastSectionKind(value: string): "required" | "optional" | undefined {
+  let kind: "required" | "optional" | undefined;
+  for (const rawLine of value.split("\n")) {
+    const line = rawLine.trim().replace(/:$/, "");
+    if (/^(?:preferred qualifications?|optional requirements?|nice to have|extra awesome|optional)$/i.test(line)) kind = "optional";
+    else if (/^(?:what(?:'|’)s required|requirements?|qualifications?|required qualifications|minimum qualifications?|essentials?|essential qualifications?|must haves?|your experience includes)$/i.test(line)) kind = "required";
+  }
+  return kind;
+}
+
+function clean(value: string): string { return value.trim().replace(/\s+/g, " "); }
+
+function educationLevelMatches(level: string, value: string): boolean {
+  if (level.startsWith("bachelor")) return /\b(?:bachelor|b\.?\s*(?:tech|e|sc|a|com|ba|eng))\b/i.test(value);
+  if (level.startsWith("master")) return /\b(?:master|m\.?\s*(?:tech|e|sc|a|com|ba|eng))\b/i.test(value);
+  return /\b(?:ph\.?d|doctorate)\b/i.test(value);
+}
+
+function educationFieldMatches(requiredField: string, value: string): boolean {
+  const candidate = value.toLocaleLowerCase();
+  if (/\b(?:computer science|computing|software|information technology|engineering)\b/i.test(requiredField) && /\b(?:computer science|computing|software|information technology|engineering|b\.?tech|b\.?e\.?|m\.?tech|m\.?e\.?)\b/i.test(candidate)) return true;
+  if (/\b(?:technical|business)\b/i.test(requiredField) && /\b(?:technology|technical|engineering|computer|software|information systems|business|commerce|bba|mba|b\.?tech|m\.?tech)\b/i.test(candidate)) return true;
+  return false;
+}

package/src/selected-job-lookup.ts

@@ -0,0 +1,14 @@
+import type { Job } from "./types.ts";
+
+export interface SelectedJobLookupOptions {
+  getSnapshotJob(id: string): Promise<Job | null>;
+  getDetailedJob(id: string): Promise<Job | null>;
+}
+
+export function createSelectedJobLookup(options: SelectedJobLookupOptions) {
+  return async (id: string): Promise<Job | null> => {
+    const snapshot = await options.getSnapshotJob(id);
+    if (snapshot?.description.trim()) return snapshot;
+    return options.getDetailedJob(id);
+  };
+}

package/src/snapshot-catalog.ts

@@ -0,0 +1,21 @@
+import { searchJobs, type Catalog } from "./catalog.ts";
+import type { SnapshotStore } from "./crawler.ts";
+
+export function createSnapshotCatalog(store: SnapshotStore): Catalog {
+  return {
+    async search(query) {
+      const snapshot = await store.read();
+      if (!snapshot) return [];
+      return searchJobs(Object.values(snapshot.partitions).flatMap((partition) => partition.jobs), query);
+    },
+    async get(id) {
+      const snapshot = await store.read();
+      if (!snapshot) return null;
+      for (const partition of Object.values(snapshot.partitions)) {
+        const job = partition.jobs.find((candidate) => candidate.id === id);
+        if (job) return job;
+      }
+      return null;
+    },
+  };
+}

package/src/snapshot-export.ts

@@ -0,0 +1,66 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { atomicWrite } from "./atomic-file.ts";
+import type { JobSnapshot } from "./types.ts";
+
+export interface SnapshotExportReport {
+  outputDir: string;
+  sources: number;
+  jobs: number;
+  countries: Record<string, number>;
+  manifestPath: string;
+}
+
+export async function exportSnapshot(inputPath: string, outputDir: string): Promise<SnapshotExportReport> {
+  const snapshot = JSON.parse(await readFile(inputPath, "utf8")) as JobSnapshot;
+  if (snapshot.version !== 1 || !snapshot.partitions || typeof snapshot.partitions !== "object") throw new Error(`Unsupported snapshot format: ${inputPath}`);
+  const slugs = Object.keys(snapshot.partitions);
+  if (slugs.some((slug) => !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(slug))) throw new Error("Snapshot contains an unsafe source slug");
+  const sourceDir = join(outputDir, "sources");
+  await mkdir(sourceDir, { recursive: true });
+  const countries: Record<string, number> = {};
+  const expectedFiles = new Set(slugs.map((slug) => `${slug}.json`));
+  for (const file of await previousPartitionFiles(outputDir)) if (!expectedFiles.has(file)) await unlink(join(sourceDir, file)).catch(() => undefined);
+  const partitions = [];
+  let jobCount = 0;
+
+  for (const slug of Object.keys(snapshot.partitions).sort()) {
+    const partition = snapshot.partitions[slug]!;
+    const value = { source: slug, fetchedAt: partition.fetchedAt, jobs: [...partition.jobs].sort((left, right) => left.id.localeCompare(right.id)) };
+    const content = stableJson(value);
+    const path = `sources/${slug}.json`;
+    await atomicWrite(join(outputDir, path), content);
+    jobCount += value.jobs.length;
+    for (const job of value.jobs) for (const country of job.eligibleCountries) countries[country] = (countries[country] ?? 0) + 1;
+    partitions.push({ source: slug, path, jobs: value.jobs.length, sha256: sha256(content) });
+  }
+
+  const sortedCountries = Object.fromEntries(Object.entries(countries).sort(([left], [right]) => left.localeCompare(right)));
+  const manifest = {
+    version: 1,
+    updatedAt: snapshot.updatedAt,
+    sources: partitions.length,
+    jobs: jobCount,
+    countries: sortedCountries,
+    partitions,
+  };
+  const manifestPath = join(outputDir, "manifest.json");
+  await atomicWrite(manifestPath, stableJson(manifest));
+  return { outputDir, sources: partitions.length, jobs: jobCount, countries: sortedCountries, manifestPath };
+}
+
+async function previousPartitionFiles(outputDir: string): Promise<string[]> {
+  try {
+    const value: unknown = JSON.parse(await readFile(join(outputDir, "manifest.json"), "utf8"));
+    if (!value || typeof value !== "object" || !("partitions" in value) || !Array.isArray(value.partitions)) return [];
+    return value.partitions.flatMap((partition) => {
+      if (!partition || typeof partition !== "object" || !("path" in partition) || typeof partition.path !== "string") return [];
+      const match = /^sources\/([a-z0-9]+(?:-[a-z0-9]+)*\.json)$/.exec(partition.path);
+      return match ? [match[1]!] : [];
+    });
+  } catch { return []; }
+}
+
+function stableJson(value: unknown): string { return `${JSON.stringify(value, null, 2)}\n`; }
+function sha256(value: string): string { return createHash("sha256").update(value).digest("hex"); }

package/src/snapshot-store.ts

@@ -0,0 +1,31 @@
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import type { SnapshotStore } from "./crawler.ts";
+import type { JobSnapshot } from "./types.ts";
+
+export function createFileSnapshotStore(path: string): SnapshotStore {
+  return {
+    async read() {
+      try {
+        const parsed = JSON.parse(await readFile(path, "utf8")) as JobSnapshot;
+        if (parsed.version !== 1 || typeof parsed.partitions !== "object") {
+          throw new Error(`Unsupported snapshot format: ${path}`);
+        }
+        return parsed;
+      } catch (error) {
+        if (isMissing(error)) return null;
+        throw error;
+      }
+    },
+    async write(snapshot) {
+      await mkdir(dirname(path), { recursive: true });
+      const temporary = `${path}.${process.pid}.tmp`;
+      await writeFile(temporary, `${JSON.stringify(snapshot)}\n`, "utf8");
+      await rename(temporary, path);
+    },
+  };
+}
+
+function isMissing(error: unknown): boolean {
+  return error instanceof Error && "code" in error && error.code === "ENOENT";
+}

package/src/source-discovery-pipeline.ts

@@ -0,0 +1,26 @@
+import type { SourceDiscoveryReport } from "./source-discovery.ts";
+import { runSourceVerification, type SourcePipelineReport } from "./source-pipeline.ts";
+
+interface PromotionOptions {
+  fetch?: (input: string | URL, init?: RequestInit) => Promise<Response>;
+  concurrency?: number;
+  timeoutMs?: number;
+  now?: () => Date;
+  registryPath?: string;
+}
+
+export interface DiscoveryPromotionResult {
+  discovery: SourceDiscoveryReport;
+  promotion: SourcePipelineReport;
+}
+
+export async function discoverAndPromote(
+  discover: () => Promise<SourceDiscoveryReport>,
+  candidatesPath: string,
+  catalogPath: string,
+  options: PromotionOptions = {},
+): Promise<DiscoveryPromotionResult> {
+  const discovery = await discover();
+  const promotion = await runSourceVerification(candidatesPath, catalogPath, options);
+  return { discovery, promotion };
+}

package/src/source-discovery.ts

@@ -0,0 +1,231 @@
+import { mkdir, readFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { atomicJson } from "./atomic-file.ts";
+import { stampReport, type ReportMeta } from "./report-meta.ts";
+import { withFileLock } from "./file-lock.ts";
+import { resolveSource } from "./source-verification.ts";
+import type { DiscoveryChannel, SourceCandidate } from "./types.ts";
+import { mergeEnrichmentLeads, type EnrichmentLead, type IdentityEvidence } from "./enrichment-registry.ts";
+
+type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
+
+interface DiscoveryOptions {
+  country?: string;
+  fetch?: Fetch;
+  concurrency?: number;
+  timeoutMs?: number;
+  registryPath?: string;
+}
+
+interface FeedEntry { sourceUrl: string; companyName?: string; companyDomain?: string; reference?: string; channel?: DiscoveryChannel; domainEvidence?: "authoritative_dataset" | "company_registry" | "company_redirect" }
+interface DiscoveryIssue { sourceUrl: string; reason: string; detail: string; companyName?: string; reference?: string }
+
+export interface SourceDiscoveryReport extends ReportMeta {
+  discovered: number;
+  ready: number;
+  alreadyKnown: number;
+  needsDomain: number;
+  rejected: number;
+  candidatesPath: string;
+  reportPath: string;
+  unresolved: DiscoveryIssue[];
+  rejections: DiscoveryIssue[];
+  registryPath?: string;
+  registryAdded: number;
+}
+
+export async function runSourceDiscovery(feedPath: string, candidatesPath: string, reportPath: string, options: DiscoveryOptions = {}): Promise<SourceDiscoveryReport> {
+  const feed = await readFeed(feedPath);
+  return discoverEntries(feed, candidatesPath, reportPath, options, feedPath, false);
+}
+
+export async function runYcSourceDiscovery(candidatesPath: string, reportPath: string, options: DiscoveryOptions & { country: string }): Promise<SourceDiscoveryReport> {
+  const fetcher = options.fetch ?? globalThis.fetch;
+  const response = await fetcher("https://yc-oss.github.io/api/companies/all.json");
+  if (!response.ok) throw new Error(`YC company API returned HTTP ${response.status}`);
+  const value: unknown = await response.json();
+  if (!Array.isArray(value) || !value.every(isRecord)) throw new Error("YC company API returned an invalid payload");
+  const country = options.country.toUpperCase();
+  const countryName = new Intl.DisplayNames(["en"], { type: "region" }).of(country);
+  if (!countryName || countryName === country) throw new Error("country must be a valid two-letter code");
+  const countryPattern = new RegExp(`\\b${escapeRegExp(countryName)}\\b`, "i");
+  const feed: FeedEntry[] = value.flatMap((company) => {
+    if (typeof company.slug !== "string" || typeof company.website !== "string" || typeof company.all_locations !== "string" || !countryPattern.test(company.all_locations)) return [];
+    let companyDomain: string;
+    try { companyDomain = new URL(company.website).hostname.replace(/^www\./, ""); } catch { return []; }
+    return [{
+      sourceUrl: `https://job-boards.greenhouse.io/${company.slug}`,
+      companyName: typeof company.name === "string" ? company.name : company.slug,
+      companyDomain,
+      reference: `https://www.ycombinator.com/companies/${company.slug}`,
+      domainEvidence: "authoritative_dataset",
+    }];
+  });
+  return discoverEntries(feed, candidatesPath, reportPath, { ...options, fetch: fetcher, country }, "YC public company API", true);
+}
+
+async function discoverEntries(feed: FeedEntry[], candidatesPath: string, reportPath: string, options: DiscoveryOptions, feedReference: string, trustDomainEvidence: boolean): Promise<SourceDiscoveryReport> {
+  const existing = await readCandidates(candidatesPath);
+  const concurrency = Math.max(1, Math.trunc(options.concurrency ?? 10));
+  const country = options.country?.toUpperCase();
+  if (country && !/^[A-Z]{2}$/.test(country)) throw new Error("country must be a two-letter code");
+  const ready: Array<{ index: number; candidate: SourceCandidate }> = [];
+  const unresolved: Array<{ index: number; issue: DiscoveryIssue }> = [];
+  const rejected: Array<{ index: number; issue: DiscoveryIssue }> = [];
+  const registryRows: EnrichmentLead[] = [];
+  const existingSources = new Set(existing.map((candidate) => sourceKey(candidate.sourceUrl)).filter(Boolean));
+  let alreadyKnown = 0;
+  let cursor = 0;
+
+  async function worker() {
+    while (cursor < feed.length) {
+      const index = cursor++;
+      const entry = feed[index];
+      if (!entry || typeof entry.sourceUrl !== "string") {
+        rejected.push({ index, issue: { sourceUrl: "", reason: "invalid_entry", detail: "sourceUrl must be a string" } });
+        continue;
+      }
+      if (entry.companyDomain !== undefined && (typeof entry.companyDomain !== "string" || !/^[a-z0-9.-]+\.[a-z]{2,}$/i.test(entry.companyDomain))) {
+        rejected.push({ index, issue: issue({ sourceUrl: entry.sourceUrl }, "invalid_domain", "companyDomain must be a hostname") });
+        continue;
+      }
+      if (entry.companyName !== undefined && (typeof entry.companyName !== "string" || !entry.companyName.trim())) {
+        rejected.push({ index, issue: issue({ sourceUrl: entry.sourceUrl }, "invalid_name", "companyName must be a non-empty string") });
+        continue;
+      }
+      if (entry.reference !== undefined && typeof entry.reference !== "string") {
+        rejected.push({ index, issue: issue({ sourceUrl: entry.sourceUrl }, "invalid_reference", "reference must be a string") });
+        continue;
+      }
+      const source = resolveSource(entry.sourceUrl);
+      if (!source) {
+        rejected.push({ index, issue: issue(entry, "unsupported_source", "Discovery accepts Greenhouse, Lever, Ashby, Workday, and Recruitee source URLs") });
+        continue;
+      }
+      if (entry.channel !== undefined && !["search", "career_page", "provider_directory", "community", "dataset"].includes(entry.channel)) {
+        rejected.push({ index, issue: issue(entry, "invalid_channel", `Unsupported discovery channel: ${entry.channel}`) });
+        continue;
+      }
+      if (entry.domainEvidence !== undefined && !["authoritative_dataset", "company_registry", "company_redirect"].includes(entry.domainEvidence)) {
+        rejected.push({ index, issue: issue(entry, "invalid_domain_evidence", `Unsupported domain evidence: ${entry.domainEvidence}`) });
+        continue;
+      }
+      const key = `${source.ats}:${source.token.toLocaleLowerCase()}`;
+      const companyName = typeof entry.companyName === "string" ? entry.companyName.trim() : "";
+      const reference = entry.reference?.trim() || feedReference;
+      const match = entry.companyDomain && companyName ? [{ companyName, companyDomain: entry.companyDomain.toLowerCase(), method: "normalized_token" as const, reference }] : [];
+      const redirectTrusted = entry.domainEvidence === "company_redirect" && entry.companyDomain && referenceBelongsToDomain(reference, entry.companyDomain);
+      const datasetTrusted = trustDomainEvidence && entry.domainEvidence && entry.domainEvidence !== "company_redirect";
+      const evidence: IdentityEvidence[] = (redirectTrusted || datasetTrusted) && entry.companyDomain && companyName ? [{ companyName, companyDomain: entry.companyDomain.toLowerCase(), kind: entry.domainEvidence!, reference, observedAt: new Date().toISOString() }] : [];
+      registryRows.push({ sourceKey: key, sourceUrl: source.canonicalSourceUrl, ats: source.ats, token: source.token,
+        discoveredFrom: [{ channel: entry.channel ?? "dataset", reference }], companyMatches: match, identityEvidence: evidence, attempts: [] });
+      if (existingSources.has(key)) {
+        alreadyKnown++;
+        continue;
+      }
+      if (!entry.companyDomain || !companyName) {
+        unresolved.push({ index, issue: { ...issue(entry, "needs_identity", "companyName and companyDomain are required before verification"), companyName: companyName || undefined } });
+        continue;
+      }
+      const providerCanAcquireIdentity = source.ats === "recruitee";
+      if (!redirectTrusted && !datasetTrusted && !providerCanAcquireIdentity) {
+        unresolved.push({ index, issue: { ...issue(entry, "needs_domain_evidence", "The entry needs trusted dataset evidence or a company-owned redirect"), companyName } });
+        continue;
+      }
+      ready.push({ index, candidate: {
+        companyName, companyDomain: entry.companyDomain.toLocaleLowerCase(), sourceUrl: source.canonicalSourceUrl,
+        cohorts: country ? [country] : undefined,
+        discoveredFrom: { channel: entry.channel ?? "dataset", reference },
+        ...((redirectTrusted || datasetTrusted) ? { domainEvidence: { kind: entry.domainEvidence!, reference } } : {}),
+      } });
+    }
+  }
+
+  await Promise.all(Array.from({ length: Math.min(concurrency, feed.length) }, worker));
+  const registry = options.registryPath ? await mergeEnrichmentLeads(options.registryPath, registryRows) : { added: 0 };
+  const additions: SourceCandidate[] = [];
+  const acceptedSources = new Set<string>();
+  for (const row of ready.sort((a, b) => a.index - b.index)) {
+    const key = sourceKey(row.candidate.sourceUrl)!;
+    if (acceptedSources.has(key)) rejected.push({ index: row.index, issue: issue({ sourceUrl: row.candidate.sourceUrl, reference: row.candidate.discoveredFrom.reference }, "duplicate_source", `Duplicate of ${key}`) });
+    else { acceptedSources.add(key); additions.push(row.candidate); }
+  }
+  const filteredUnresolved = unresolved.filter((row) => {
+    const key = sourceKey(row.issue.sourceUrl);
+    if (!key || !acceptedSources.has(key)) return true;
+    rejected.push({ index: row.index, issue: { ...row.issue, reason: "duplicate_source", detail: `Duplicate of ${key}` } });
+    return false;
+  });
+  const unresolvedSources = new Set<string>();
+  const uniqueUnresolved = filteredUnresolved.filter((row) => {
+    const key = sourceKey(row.issue.sourceUrl);
+    if (!key || !unresolvedSources.has(key)) { if (key) unresolvedSources.add(key); return true; }
+    rejected.push({ index: row.index, issue: { ...row.issue, reason: "duplicate_source", detail: `Duplicate of ${key}` } });
+    return false;
+  });
+  const appended = await mergeSourceCandidates(candidatesPath, additions);
+  const report: SourceDiscoveryReport = stampReport("source-discovery:1", 1, {
+    discovered: feed.length, ready: appended, alreadyKnown, needsDomain: uniqueUnresolved.length, rejected: rejected.length,
+    candidatesPath, reportPath, registryPath: options.registryPath, registryAdded: registry.added,
+    unresolved: uniqueUnresolved.sort((a, b) => a.index - b.index).map((row) => row.issue),
+    rejections: rejected.sort((a, b) => a.index - b.index).map((row) => row.issue),
+  });
+  await atomicJson(reportPath, report);
+  return report;
+}
+
+async function readFeed(path: string): Promise<FeedEntry[]> {
+  const value: unknown = JSON.parse(await readFile(path, "utf8"));
+  if (!Array.isArray(value) || !value.every(isRecord)) throw new Error("Discovery feed must be a JSON array of objects");
+  return value as unknown as FeedEntry[];
+}
+
+async function readCandidates(path: string): Promise<SourceCandidate[]> {
+  try {
+    const value: unknown = JSON.parse(await readFile(path, "utf8"));
+    if (!Array.isArray(value)) throw new Error("Candidate file must contain an array");
+    return value as SourceCandidate[];
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") return [];
+    throw error;
+  }
+}
+
+export async function mergeSourceCandidates(path: string, additions: SourceCandidate[]): Promise<number> {
+  await mkdir(dirname(path), { recursive: true });
+  return withFileLock(path, async () => {
+    const current = await readCandidates(path);
+    const positions = new Map(current.flatMap((candidate, index) => {
+      const key = sourceKey(candidate.sourceUrl);
+      return key ? [[key, index] as const] : [];
+    }));
+    const seen = new Set(positions.keys());
+    const unique = additions.filter((candidate) => {
+      const key = sourceKey(candidate.sourceUrl);
+      if (!key) return false;
+      const existingIndex = positions.get(key);
+      if (existingIndex !== undefined) {
+        const existing = current[existingIndex]!;
+        const cohorts = [...new Set([...(existing.cohorts ?? []), ...(candidate.cohorts ?? [])])].sort();
+        current[existingIndex] = {
+          ...existing,
+          ...(cohorts.length ? { cohorts } : {}),
+          ...(candidate.domainEvidence && !existing.domainEvidence ? { domainEvidence: candidate.domainEvidence, discoveredFrom: candidate.discoveredFrom } : {}),
+        };
+        return false;
+      }
+      if (seen.has(key)) return false;
+      seen.add(key);
+      return true;
+    });
+    await atomicJson(path, [...current, ...unique]);
+    return unique.length;
+  }, { operation: "merge source candidates" });
+}
+
+
+function sourceKey(value: string): string | undefined { const source = resolveSource(value); return source ? `${source.ats}:${source.token.toLocaleLowerCase()}` : undefined; }
+function issue(entry: FeedEntry, reason: string, detail: string): DiscoveryIssue { return { sourceUrl: entry.sourceUrl, reference: entry.reference, reason, detail }; }
+function referenceBelongsToDomain(reference: string, domain: string): boolean { try { const host = new URL(reference).hostname.toLowerCase().replace(/^www\./, ""); const expected = domain.toLowerCase().replace(/^www\./, ""); return host === expected || host.endsWith(`.${expected}`); } catch { return false; } }
+function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }
+function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); }