openings 0.1.0

package/src/common-crawl-discovery.ts

@@ -0,0 +1,137 @@
+import { readFile } from "node:fs/promises";
+import { atomicJson } from "./atomic-file.ts";
+import { stampReport, type ReportMeta } from "./report-meta.ts";
+import { mergeEnrichmentLeads, type EnrichmentLead } from "./enrichment-registry.ts";
+import { resolveSource } from "./source-verification.ts";
+import type { Ats } from "./types.ts";
+import { assertArtifactFile } from "./artifact-path.ts";
+
+type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
+interface Options { country?: string; fetch?: Fetch; indexUrl?: string; registryPath?: string; provider?: Ats; indexRecordLimit?: number; sampleTokenLimit?: number; excludeTokens?: string[] }
+interface Lead { sourceUrl: string; ats: string; token: string; discoveredFrom: { channel: "dataset"; reference: string } }
+interface Collection { id?: unknown; "cdx-api"?: unknown }
+
+export interface CommonCrawlDiscoveryReport extends ReportMeta {
+  country?: string;
+  provider?: Ats;
+  index: string;
+  indexRecordsExamined: number;
+  sampleTokenLimit?: number;
+  sampleShortfall: number;
+  urlsSeen: number;
+  sourcesFound: number;
+  alreadyKnown: number;
+  unresolved: number;
+  rejected: number;
+  truncated: boolean;
+  truncatedPatterns: string[];
+  candidatesPath: string;
+  reportPath: string;
+  registryPath?: string;
+  registryAdded: number;
+  registryBytes?: number;
+  registryLockHeldMs?: number;
+  availableLeads: Lead[];
+  leads: Lead[];
+  rejections: Array<{ value: string; reason: string }>;
+}
+
+const patterns = ["job-boards.greenhouse.io/*", "boards.greenhouse.io/*", "jobs.lever.co/*", "jobs.ashbyhq.com/*", "*.myworkdayjobs.com/*", "*.recruitee.com/*"];
+const recordsPerPattern = 10_000;
+const providerPatterns: Record<Ats, string[]> = {
+  greenhouse: patterns.slice(0, 2), lever: [patterns[2]!], ashby: [patterns[3]!], workday: [patterns[4]!], recruitee: [patterns[5]!],
+};
+
+export async function discoverCommonCrawlSources(candidatesPath: string, reportPath: string, options: Options = {}): Promise<CommonCrawlDiscoveryReport> {
+  if (options.provider === "recruitee") await assertArtifactFile(reportPath);
+  const country = options.country?.toUpperCase();
+  if (country && !/^[A-Z]{2}$/.test(country)) throw new Error("country must be a two-letter code");
+  const indexRecordLimit = options.indexRecordLimit ?? recordsPerPattern;
+  if (!Number.isInteger(indexRecordLimit) || indexRecordLimit < 1 || indexRecordLimit > recordsPerPattern) throw new Error(`indexRecordLimit must be an integer from 1 to ${recordsPerPattern}`);
+  const sampleTokenLimit = options.sampleTokenLimit;
+  if (sampleTokenLimit !== undefined && (!Number.isInteger(sampleTokenLimit) || sampleTokenLimit < 1 || sampleTokenLimit > indexRecordLimit)) throw new Error("sampleTokenLimit must be a positive integer no greater than indexRecordLimit");
+  if (options.provider === "recruitee" && (indexRecordLimit > 2_000 || sampleTokenLimit === undefined || sampleTokenLimit > 60 || options.registryPath)) throw new Error("Bounded Recruitee discovery requires at most 2,000 records, at most 60 sampled tokens, and report-only output");
+  const fetcher = options.fetch ?? globalThis.fetch;
+  const index = options.indexUrl ?? await latestIndex(fetcher);
+  const existing = await readExistingKeys(candidatesPath);
+  const found = new Map<string, ReturnType<typeof resolveSource>>();
+  const seenUrls = new Set<string>();
+  const rejections: Array<{ value: string; reason: string }> = [];
+  const truncatedPatterns: string[] = [];
+  let indexRecordsExamined = 0;
+
+  for (const pattern of options.provider ? providerPatterns[options.provider] : patterns) {
+    const query = new URL(index);
+    query.searchParams.set("url", pattern);
+    query.searchParams.set("output", "json");
+    query.searchParams.set("filter", "status:200");
+    query.searchParams.set("collapse", "urlkey");
+    query.searchParams.set("fl", "url");
+    query.searchParams.set("limit", String(indexRecordLimit));
+    const response = await fetcher(query);
+    if (!response.ok) throw new Error(`Common Crawl index returned HTTP ${response.status} for ${pattern}`);
+    const lines = (await response.text()).split(/\r?\n/).filter(Boolean).slice(0, indexRecordLimit);
+    indexRecordsExamined += lines.length;
+    if (lines.length >= indexRecordLimit) truncatedPatterns.push(pattern);
+    for (const line of lines) {
+      let value: unknown;
+      try { value = JSON.parse(line); } catch { rejections.push({ value: line, reason: "invalid_index_record" }); continue; }
+      if (!isRecord(value) || typeof value.url !== "string") { rejections.push({ value: line, reason: "missing_url" }); continue; }
+      seenUrls.add(value.url);
+      const source = resolveSource(value.url);
+      if (!source) { rejections.push({ value: value.url, reason: "unsupported_source" }); continue; }
+      found.set(`${source.ats}:${source.token.toLowerCase()}`, source);
+    }
+  }
+
+  const availableLeads: Lead[] = [];
+  const excludedTokens = new Set((options.excludeTokens ?? []).map((token) => token.toLowerCase()));
+  let alreadyKnown = 0;
+  for (const [key, source] of [...found].sort(([a], [b]) => a.localeCompare(b))) {
+    if (!source) continue;
+    if (existing.has(key)) { alreadyKnown++; continue; }
+    if (excludedTokens.has(source.token.toLowerCase())) continue;
+    availableLeads.push({
+      sourceUrl: source.canonicalSourceUrl, ats: source.ats, token: source.token,
+      discoveredFrom: { channel: "dataset", reference: index },
+    });
+  }
+  const leads = sampleTokenLimit === undefined ? availableLeads : availableLeads.slice(0, sampleTokenLimit);
+  const registryLeads: EnrichmentLead[] = leads.map((lead) => ({
+      sourceKey: `${lead.ats}:${lead.token.toLowerCase()}`, sourceUrl: lead.sourceUrl, ats: lead.ats as Ats, token: lead.token,
+      discoveredFrom: [lead.discoveredFrom], companyMatches: [], identityEvidence: [], attempts: [],
+    }));
+  const registry = options.registryPath ? await mergeEnrichmentLeads(options.registryPath, registryLeads) : { added: 0, bytes: undefined, lockHeldMs: undefined };
+  const report: CommonCrawlDiscoveryReport = stampReport("common-crawl-discovery:1", 1, {
+    country, provider: options.provider, index, indexRecordsExamined, sampleTokenLimit, sampleShortfall: sampleTokenLimit === undefined ? 0 : Math.max(0, sampleTokenLimit - leads.length), urlsSeen: seenUrls.size, sourcesFound: found.size, alreadyKnown, unresolved: leads.length, rejected: rejections.length,
+    truncated: truncatedPatterns.length > 0, truncatedPatterns,
+    candidatesPath, reportPath, registryPath: options.registryPath, registryAdded: registry.added, registryBytes: registry.bytes, registryLockHeldMs: registry.lockHeldMs, availableLeads, leads, rejections,
+  });
+  await atomicJson(reportPath, report);
+  return report;
+}
+
+async function latestIndex(fetcher: Fetch): Promise<string> {
+  const response = await fetcher("https://index.commoncrawl.org/collinfo.json");
+  if (!response.ok) throw new Error(`Common Crawl collection list returned HTTP ${response.status}`);
+  const value: unknown = await response.json();
+  if (!Array.isArray(value)) throw new Error("Common Crawl collection list is invalid");
+  const collection = value.find((entry): entry is Collection => isRecord(entry) && typeof entry["cdx-api"] === "string");
+  if (!collection || typeof collection["cdx-api"] !== "string") throw new Error("Common Crawl collection list contains no queryable index");
+  return collection["cdx-api"];
+}
+
+async function readExistingKeys(path: string): Promise<Set<string>> {
+  try {
+    const value: unknown = JSON.parse(await readFile(path, "utf8"));
+    if (!Array.isArray(value)) throw new Error("Candidate file must contain an array");
+    return new Set(value.flatMap((candidate) => isRecord(candidate) && typeof candidate.sourceUrl === "string" ? [resolveSource(candidate.sourceUrl)] : [])
+      .filter((source): source is NonNullable<ReturnType<typeof resolveSource>> => Boolean(source))
+      .map((source) => `${source.ats}:${source.token.toLowerCase()}`));
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") return new Set();
+    throw error;
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }

package/src/company-seeds.ts

@@ -0,0 +1,43 @@
+import { atomicJson } from "./atomic-file.ts";
+import { withFileLock } from "./file-lock.ts";
+
+type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
+interface Options { country: string; fetch?: Fetch }
+interface CompanySeed { companyName: string; companyDomain: string }
+
+export interface CompanySeedReport {
+  country: string;
+  matched: number;
+  written: number;
+  skipped: number;
+  outputPath: string;
+}
+
+export async function generateYcCompanySeeds(outputPath: string, options: Options): Promise<CompanySeedReport> {
+  const country = options.country.toUpperCase();
+  const countryName = new Intl.DisplayNames(["en"], { type: "region" }).of(country);
+  if (!/^[A-Z]{2}$/.test(country) || !countryName || countryName === country) throw new Error("country must be a valid two-letter code");
+  const response = await (options.fetch ?? globalThis.fetch)("https://yc-oss.github.io/api/companies/all.json");
+  if (!response.ok) throw new Error(`YC company API returned HTTP ${response.status}`);
+  const value: unknown = await response.json();
+  if (!Array.isArray(value) || !value.every(isRecord)) throw new Error("YC company API returned an invalid payload");
+  const pattern = new RegExp(`\\b${escapeRegExp(countryName)}\\b`, "i");
+  const matched = value.filter((company) => typeof company.all_locations === "string" && pattern.test(company.all_locations));
+  const seen = new Set<string>();
+  const seeds: CompanySeed[] = [];
+  for (const company of matched) {
+    if (typeof company.name !== "string" || !company.name.trim() || typeof company.website !== "string") continue;
+    let url: URL;
+    try { url = new URL(company.website); } catch { continue; }
+    if (url.protocol !== "https:" && url.protocol !== "http:") continue;
+    const companyDomain = url.hostname.toLowerCase().replace(/^www\./, "");
+    if (!/^[a-z0-9.-]+\.[a-z]{2,}$/i.test(companyDomain) || seen.has(companyDomain)) continue;
+    seen.add(companyDomain);
+    seeds.push({ companyName: company.name.trim(), companyDomain });
+  }
+  await withFileLock(outputPath, () => atomicJson(outputPath, seeds), { operation: "write company seeds" });
+  return { country, matched: matched.length, written: seeds.length, skipped: matched.length - seeds.length, outputPath };
+}
+
+function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); }
+function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }

package/src/country-coverage.ts

@@ -0,0 +1,203 @@
+import { readFile } from "node:fs/promises";
+import { atomicJson } from "./atomic-file.ts";
+import { deriveLeadState, readEnrichmentRegistry, type EnrichmentState } from "./enrichment-registry.ts";
+import { isEligibleForCountry } from "./locations.ts";
+import { stampReport, type ReportMeta } from "./report-meta.ts";
+import { resolveSource } from "./source-verification.ts";
+import type { Ats, EligibilityConfidence, JobSnapshot, SourceCandidate, VerifiedCompany } from "./types.ts";
+import { projectJobCoverage } from "./job-coverage.ts";
+
+export interface CountryCoveragePaths {
+  catalog: string;
+  candidates: string;
+  registry: string;
+  snapshot: string;
+  output?: string;
+}
+
+export interface CountryCoverageReport extends ReportMeta {
+  country: string;
+  catalog: {
+    verifiedSources: number; countryCohortSources: number;
+    providers: Partial<Record<Ats, number>>; countryCohortProviders: Partial<Record<Ats, number>>;
+  };
+  snapshot: {
+    updatedAt: string; indexedSources: number; countryCohortIndexedSources: number; catalogCoveragePercent: number; orphanedSources: string[];
+    snapshotJobs: number; indexedJobs: number; indexedSourcesWithEligibleJobs: number; eligibleJobs: number; distinctEligibleEmployers: number;
+    indexedProviders: Partial<Record<Ats, number>>; countryCohortIndexedProviders: Partial<Record<Ats, number>>;
+    allIndexedJobsByConfidence: Record<EligibilityConfidence, number>; eligibleJobsByConfidence: Record<EligibilityConfidence, number>;
+  };
+  sourceHealth: { latestBatch: { selected: number; succeeded: number; failed: number; successPercent: number | null } };
+  freshness: {
+    referenceAt: string; fresh24Hours: number; age1To7Days: number; age8To14Days: number; olderThan14Days: number;
+    futureTimestamp: number; neverIndexedSources: string[]; oldestFetchedAt: string | null; newestFetchedAt: string | null;
+    sources: { fresh24Hours: string[]; age1To7Days: string[]; age8To14Days: string[]; olderThan14Days: string[]; futureTimestamp: string[] };
+  };
+  discovery: {
+    registryLeadsGlobal: number; registryStatesGlobal: Record<EnrichmentState, number>; verifiedRegistryLeadsGlobal: number; registryVerificationYieldPercentGlobal: number;
+    candidatesGlobal: number; promotedCandidatesGlobal: number; candidatePromotionPercentGlobal: number;
+    countryCohortCandidates: number; promotedCountryCohortCandidates: number; countryCohortCandidatePromotionPercent: number;
+  };
+}
+
+export async function generateCountryCoverageReport(paths: CountryCoveragePaths, options: { country: string; asOf?: Date }): Promise<CountryCoverageReport> {
+  const country = options.country.toUpperCase();
+  if (!/^[A-Z]{2}$/u.test(country)) throw new Error("Country coverage requires a two-letter country code");
+  const asOf = options.asOf ?? new Date();
+  if (!Number.isFinite(asOf.getTime())) throw new Error("Country coverage requires a valid reference time");
+  const [catalog, candidates, registry, snapshot] = await Promise.all([
+    readCatalog(paths.catalog), readCandidates(paths.candidates), readEnrichmentRegistry(paths.registry), readSnapshot(paths.snapshot),
+  ]);
+  const slugs = Object.keys(catalog).sort();
+  const countryCohortSlugs = slugs.filter((slug) => catalog[slug]!.cohorts?.includes(country));
+  const providers = countProviders(slugs.map((slug) => catalog[slug]!.ats));
+  const indexedSlugs = slugs.filter((slug) => snapshot.partitions[slug]);
+  const countryCohortIndexedSlugs = countryCohortSlugs.filter((slug) => snapshot.partitions[slug]);
+  const neverIndexedSources = slugs.filter((slug) => !snapshot.partitions[slug]);
+  const orphanedSources = Object.keys(snapshot.partitions).filter((slug) => !catalog[slug]).sort();
+  const indexedJobs = indexedSlugs.flatMap((slug) => snapshot.partitions[slug]!.jobs);
+  const eligibleJobs = indexedJobs.filter((job) => isEligibleForCountry(job, country));
+  const candidateCoverage = projectJobCoverage(slugs.map((slug) => ({ slug, ...catalog[slug]! })), snapshot, [country]).countries[0]!;
+  const allConfidence = confidenceCounts(indexedJobs);
+  const eligibleConfidence = confidenceCounts(eligibleJobs);
+  const states = { unresolved: 0, matched: 0, evidence_ready: 0, verified: 0, rejected: 0 } satisfies Record<EnrichmentState, number>;
+  for (const lead of registry.leads) states[deriveLeadState(lead)] += 1;
+  const promotedCandidates = candidates.filter((candidate) => candidateIsPromoted(candidate, catalog)).length;
+  const countryCandidates = candidates.filter((candidate) => candidate.cohorts?.includes(country));
+  const promotedCountryCandidates = countryCandidates.filter((candidate) => candidateIsPromoted(candidate, catalog)).length;
+  const selected = snapshot.lastCrawl.selected;
+  const report = stampReport("country-coverage:1", 1, {
+    country,
+    catalog: {
+      verifiedSources: slugs.length, countryCohortSources: countryCohortSlugs.length, providers,
+      countryCohortProviders: countProviders(countryCohortSlugs.map((slug) => catalog[slug]!.ats)),
+    },
+    snapshot: {
+      updatedAt: snapshot.updatedAt,
+      indexedSources: indexedSlugs.length,
+      countryCohortIndexedSources: countryCohortIndexedSlugs.length,
+      catalogCoveragePercent: percent(indexedSlugs.length, slugs.length),
+      orphanedSources,
+      snapshotJobs: Object.values(snapshot.partitions).reduce((sum, partition) => sum + partition.jobs.length, 0),
+      indexedJobs: indexedJobs.length,
+      indexedSourcesWithEligibleJobs: candidateCoverage.indexedSourcesWithEligibleJobs,
+      eligibleJobs: candidateCoverage.eligibleJobs,
+      distinctEligibleEmployers: candidateCoverage.distinctEligibleEmployers,
+      indexedProviders: countProviders(indexedSlugs.map((slug) => catalog[slug]!.ats)),
+      countryCohortIndexedProviders: countProviders(countryCohortIndexedSlugs.map((slug) => catalog[slug]!.ats)),
+      allIndexedJobsByConfidence: allConfidence,
+      eligibleJobsByConfidence: eligibleConfidence,
+    },
+    sourceHealth: { latestBatch: {
+      selected, succeeded: snapshot.lastCrawl.succeeded, failed: snapshot.lastCrawl.failed.length,
+      successPercent: selected > 0 ? percent(snapshot.lastCrawl.succeeded, selected) : null,
+    } },
+    freshness: freshness(indexedSlugs.map((slug) => [slug, snapshot.partitions[slug]!.fetchedAt] as const), neverIndexedSources, asOf),
+    discovery: {
+      registryLeadsGlobal: registry.leads.length, registryStatesGlobal: states, verifiedRegistryLeadsGlobal: states.verified,
+      registryVerificationYieldPercentGlobal: percent(states.verified, registry.leads.length),
+      candidatesGlobal: candidates.length, promotedCandidatesGlobal: promotedCandidates,
+      candidatePromotionPercentGlobal: percent(promotedCandidates, candidates.length),
+      countryCohortCandidates: countryCandidates.length, promotedCountryCohortCandidates: promotedCountryCandidates,
+      countryCohortCandidatePromotionPercent: percent(promotedCountryCandidates, countryCandidates.length),
+    },
+  }, asOf);
+  if (paths.output) await atomicJson(paths.output, report);
+  return report;
+}
+
+async function readCatalog(path: string): Promise<Record<string, Omit<VerifiedCompany, "slug">>> {
+  const value: unknown = JSON.parse(await readFile(path, "utf8"));
+  if (!isRecord(value)) throw new Error(`Invalid verified catalog: ${path}`);
+  for (const [slug, company] of Object.entries(value)) {
+    if (!isRecord(company) || !["greenhouse", "lever", "ashby", "workday", "recruitee"].includes(String(company.ats)) || typeof company.token !== "string"
+      || typeof company.companyDomain !== "string" || typeof company.sourceUrl !== "string"
+      || company.cohorts !== undefined && (!Array.isArray(company.cohorts) || !company.cohorts.every(validCountryCode))
+      || !validVerification(company.verification)) throw new Error(`Invalid verified catalog source: ${slug}`);
+  }
+  return value as Record<string, Omit<VerifiedCompany, "slug">>;
+}
+
+async function readCandidates(path: string): Promise<SourceCandidate[]> {
+  const value: unknown = JSON.parse(await readFile(path, "utf8"));
+  if (!Array.isArray(value) || !value.every((candidate) => isRecord(candidate) && typeof candidate.companyName === "string"
+    && typeof candidate.companyDomain === "string" && typeof candidate.sourceUrl === "string" && isRecord(candidate.discoveredFrom)
+    && (candidate.cohorts === undefined || Array.isArray(candidate.cohorts) && candidate.cohorts.every((country) => typeof country === "string")))) throw new Error(`Invalid source candidates: ${path}`);
+  return value as SourceCandidate[];
+}
+
+async function readSnapshot(path: string): Promise<JobSnapshot> {
+  const value: unknown = JSON.parse(await readFile(path, "utf8"));
+  if (!isRecord(value) || value.version !== 1 || !validDate(value.updatedAt) || !isRecord(value.partitions) || !isRecord(value.lastCrawl)
+    || !Object.values(value.partitions).every(validPartition) || !validLastCrawl(value.lastCrawl)) throw new Error(`Invalid job snapshot: ${path}`);
+  return value as unknown as JobSnapshot;
+}
+
+function candidateIsPromoted(candidate: SourceCandidate, catalog: Record<string, Omit<VerifiedCompany, "slug">>): boolean {
+  const source = resolveSource(candidate.sourceUrl);
+  if (!source) return false;
+  return Object.values(catalog).some((company) => company.ats === source.ats && company.token.toLowerCase() === source.token.toLowerCase()
+    && company.companyDomain.toLowerCase() === candidate.companyDomain.toLowerCase());
+}
+
+function countProviders(values: Ats[]): Partial<Record<Ats, number>> {
+  const counts: Partial<Record<Ats, number>> = {};
+  for (const provider of [...values].sort()) counts[provider] = (counts[provider] ?? 0) + 1;
+  return counts;
+}
+
+function freshness(values: ReadonlyArray<readonly [string, string]>, neverIndexedSources: string[], asOf: Date): CountryCoverageReport["freshness"] {
+  const result: CountryCoverageReport["freshness"] = {
+    referenceAt: asOf.toISOString(), fresh24Hours: 0, age1To7Days: 0, age8To14Days: 0, olderThan14Days: 0,
+    futureTimestamp: 0, neverIndexedSources, oldestFetchedAt: null, newestFetchedAt: null,
+    sources: { fresh24Hours: [], age1To7Days: [], age8To14Days: [], olderThan14Days: [], futureTimestamp: [] },
+  };
+  const valid: number[] = [];
+  for (const [slug, value] of values) {
+    const timestamp = Date.parse(value);
+    valid.push(timestamp);
+    const ageMs = asOf.getTime() - timestamp;
+    if (ageMs < 0) { result.futureTimestamp += 1; result.sources.futureTimestamp.push(slug); }
+    else if (ageMs <= 86_400_000) { result.fresh24Hours += 1; result.sources.fresh24Hours.push(slug); }
+    else if (ageMs <= 7 * 86_400_000) { result.age1To7Days += 1; result.sources.age1To7Days.push(slug); }
+    else if (ageMs <= 14 * 86_400_000) { result.age8To14Days += 1; result.sources.age8To14Days.push(slug); }
+    else { result.olderThan14Days += 1; result.sources.olderThan14Days.push(slug); }
+  }
+  if (valid.length) {
+    result.oldestFetchedAt = new Date(Math.min(...valid)).toISOString();
+    result.newestFetchedAt = new Date(Math.max(...valid)).toISOString();
+  }
+  return result;
+}
+
+function confidenceCounts(jobs: JobSnapshot["partitions"][string]["jobs"]): Record<EligibilityConfidence, number> {
+  const counts = { explicit: 0, inferred: 0, unknown: 0 } satisfies Record<EligibilityConfidence, number>;
+  for (const job of jobs) counts[job.eligibilityConfidence] += 1;
+  return counts;
+}
+
+function validPartition(value: unknown): boolean { return isRecord(value) && validDate(value.fetchedAt) && Array.isArray(value.jobs) && value.jobs.every(validJob); }
+function validJob(value: unknown): boolean {
+  return isRecord(value) && typeof value.id === "string" && typeof value.company === "string"
+    && (typeof value.title === "string" || value.title === undefined)
+    && typeof value.location === "string" && typeof value.remote === "boolean" && ["remote", "hybrid", "onsite", "unknown"].includes(String(value.workMode))
+    && arrayOfStrings(value.eligibleCountries) && arrayOfStrings(value.excludedCountries) && arrayOfStrings(value.eligibleRegions)
+    && ["explicit", "inferred", "unknown"].includes(String(value.eligibilityConfidence)) && typeof value.url === "string" && typeof value.description === "string";
+}
+function validLastCrawl(value: Record<string, unknown>): boolean {
+  if (!validDate(value.startedAt) || !validDate(value.finishedAt) || !nonNegativeInteger(value.selected) || !nonNegativeInteger(value.succeeded)
+    || !Array.isArray(value.failed) || !value.failed.every((failure) => isRecord(failure) && typeof failure.source === "string" && typeof failure.error === "string")) return false;
+  return value.succeeded + value.failed.length <= value.selected;
+}
+function validVerification(value: unknown): boolean {
+  return isRecord(value) && validDate(value.checkedAt) && typeof value.canonicalSourceUrl === "string" && typeof value.observedCompanyName === "string"
+    && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect"].includes(String(value.identityEvidence))
+    && typeof value.contentType === "string" && typeof value.payloadVersion === "string" && nonNegativeInteger(value.jobCount);
+}
+function validDate(value: unknown): value is string { return typeof value === "string" && Number.isFinite(Date.parse(value)); }
+function validCountryCode(value: unknown): value is string { return typeof value === "string" && /^[A-Z]{2}$/u.test(value); }
+function arrayOfStrings(value: unknown): value is string[] { return Array.isArray(value) && value.every((item) => typeof item === "string"); }
+function nonNegativeInteger(value: unknown): value is number { return Number.isInteger(value) && (value as number) >= 0; }
+
+function percent(numerator: number, denominator: number): number { return denominator > 0 ? Math.round(numerator / denominator * 10_000) / 100 : 0; }
+function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }

package/src/crawl-reporting.ts

@@ -0,0 +1,56 @@
+import type { Company, Job, JobPartition, JobSnapshot } from "./types.ts";
+
+type Fetch = typeof globalThis.fetch;
+
+/** Wire format for one crawled source partition posted to the aggregator. */
+export interface CrawlReportPayload {
+  version: 1;
+  source: { slug: string; ats: string; token: string };
+  fetchedAt: string;
+  jobs: Job[];
+}
+
+function endpoint(base: string, path: string): string {
+  return new URL(path, base.endsWith("/") ? base : `${base}/`).toString();
+}
+
+/** Returns a usable aggregator base URL or undefined; anything malformed is ignored so a bad env var can never break startup. */
+export function resolveAggregatorUrl(value: string | undefined): string | undefined {
+  const trimmed = value?.trim();
+  if (!trimmed) return undefined;
+  try {
+    const url = new URL(trimmed);
+    return url.protocol === "http:" || url.protocol === "https:" ? url.toString() : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Posts each successfully crawled partition to the aggregator. Only job data is sent, never resume content. */
+export function createCrawlReporter(options: { url: string; fetcher?: Fetch; timeoutMs?: number }) {
+  const fetcher = options.fetcher ?? globalThis.fetch;
+  const target = endpoint(options.url, "v1/crawls");
+  return async (source: Company, partition: JobPartition): Promise<void> => {
+    const payload: CrawlReportPayload = { version: 1, source: { slug: source.slug, ats: source.ats, token: source.token }, fetchedAt: partition.fetchedAt, jobs: partition.jobs };
+    const response = await fetcher(target, {
+      method: "POST",
+      headers: { "content-type": "application/json", "content-encoding": "gzip" },
+      body: Bun.gzipSync(JSON.stringify(payload)),
+      signal: AbortSignal.timeout(options.timeoutMs ?? 10_000),
+    });
+    if (!response.ok) throw new Error(`Aggregator rejected crawl report: HTTP ${response.status}`);
+  };
+}
+
+/** Downloads the aggregator's published snapshot; returns null on any failure so setup falls back to crawling. */
+export async function fetchSeedSnapshot(url: string, fetcher: Fetch = globalThis.fetch, timeoutMs = 20_000): Promise<JobSnapshot | null> {
+  try {
+    const response = await fetcher(endpoint(url, "v1/snapshot"), { signal: AbortSignal.timeout(timeoutMs) });
+    if (!response.ok) return null;
+    const snapshot = await response.json() as JobSnapshot;
+    if (snapshot?.version !== 1 || !snapshot.partitions || typeof snapshot.partitions !== "object" || !snapshot.lastCrawl) return null;
+    return snapshot;
+  } catch {
+    return null;
+  }
+}

package/src/crawler.ts

@@ -0,0 +1,146 @@
+import type { FetchJobsObserver } from "./catalog.ts";
+import type { Company, CrawlFailure, CrawlReport, CrawlSourceResult, Job, JobPartition, JobSnapshot } from "./types.ts";
+
+export interface SnapshotStore {
+  read(): Promise<JobSnapshot | null>;
+  write(snapshot: JobSnapshot): Promise<void>;
+}
+
+interface CrawlerOptions {
+  store: SnapshotStore;
+  fetchJobs(source: Company, signal?: AbortSignal, observer?: FetchJobsObserver): Promise<Job[]>;
+  concurrency?: number;
+  timeoutMs?: number;
+  maxAttempts?: number;
+  sourceStartDelayMs?: number;
+  sourceFreshnessMs?: number;
+  sourceLimit?: number;
+  workdayPageDelayMs?: number;
+  pacingNow?: () => number;
+  pacingSleep?: (delayMs: number) => Promise<void>;
+  now?: () => Date;
+  /** Called once per succeeded source after the snapshot is written; failures are swallowed so reporting never blocks a crawl. */
+  onCrawled?(source: Company, partition: JobPartition): void | Promise<void>;
+}
+
+export interface Crawler {
+  crawl(sources: Company[], settings?: { prune?: boolean }): Promise<CrawlReport>;
+}
+
+export function createCrawler(options: CrawlerOptions): Crawler {
+  const concurrency = Math.max(1, Math.trunc(options.concurrency ?? 10));
+  const timeoutMs = Math.max(1, Math.trunc(options.timeoutMs ?? 120_000));
+  const maxAttempts = Math.max(1, Math.trunc(options.maxAttempts ?? 2));
+  const sourceStartDelayMs = Math.max(0, Math.trunc(options.sourceStartDelayMs ?? 0));
+  const sourceFreshnessMs = Math.max(0, Math.trunc(options.sourceFreshnessMs ?? 0));
+  const sourceLimit = Math.max(0, Math.trunc(options.sourceLimit ?? 0));
+  const now = options.now ?? (() => new Date());
+  const pacingNow = options.pacingNow ?? Date.now;
+  const pacingSleep = options.pacingSleep ?? ((delayMs: number) => new Promise<void>((resolve) => setTimeout(resolve, delayMs)));
+  let previousStart: number | undefined;
+  let pacingGate = Promise.resolve();
+
+  async function paceSourceStart() {
+    if (!sourceStartDelayMs) return;
+    let release!: () => void;
+    const previous = pacingGate;
+    pacingGate = new Promise<void>((resolve) => { release = resolve; });
+    await previous;
+    try {
+      const remaining = previousStart === undefined ? 0 : previousStart + sourceStartDelayMs - pacingNow();
+      if (remaining > 0) await pacingSleep(remaining);
+      previousStart = pacingNow();
+    } finally { release(); }
+  }
+
+  return {
+    async crawl(sources, settings) {
+      const startedAt = now().toISOString();
+      const previous = await options.store.read();
+      const partitions = { ...(previous?.partitions ?? {}) };
+      if (settings?.prune) {
+        const current = new Set(sources.map((source) => source.slug));
+        for (const slug of Object.keys(partitions)) if (!current.has(slug)) delete partitions[slug];
+      }
+      const considered = sources.length;
+      const cutoff = Date.parse(startedAt) - sourceFreshnessMs;
+      const eligibleSources = (sourceFreshnessMs === 0 ? sources : sources.filter((source) => {
+        const fetchedAt = Date.parse(partitions[source.slug]?.fetchedAt ?? "");
+        return !Number.isFinite(fetchedAt) || fetchedAt <= cutoff;
+      })).sort((left, right) => partitionTime(partitions[left.slug]?.fetchedAt) - partitionTime(partitions[right.slug]?.fetchedAt));
+      const selectedSources = sourceLimit > 0 ? eligibleSources.slice(0, sourceLimit) : eligibleSources;
+      let pending = selectedSources;
+      let finalFailures: CrawlFailure[] = [];
+      let succeeded = 0;
+      const metrics = new Map<string, CrawlSourceResult>();
+      for (const source of selectedSources) metrics.set(source.slug, { source: source.slug, status: "failed", attempts: 0, durationMs: 0, jobs: 0, countryJobs: {}, throttles: 0, backoffMs: 0 });
+
+      for (let attempt = 1; attempt <= maxAttempts && pending.length; attempt += 1) {
+        let cursor = 0;
+        const retry: Company[] = [];
+        const failures: CrawlFailure[] = [];
+
+        async function worker() {
+          while (cursor < pending.length) {
+            const source = pending[cursor++];
+            if (!source) continue;
+            await paceSourceStart();
+            const metric = metrics.get(source.slug)!;
+            metric.attempts += 1;
+            const attemptStartedAt = Date.now();
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${timeoutMs}ms`)), timeoutMs);
+            try {
+              const jobs = await options.fetchJobs(source, controller.signal, {
+                onBackoff: ({ status, delayMs }) => { metric.backoffMs += delayMs; if (status === 429) metric.throttles += 1; },
+                workdayPageDelayMs: options.workdayPageDelayMs,
+              });
+              partitions[source.slug] = { fetchedAt: now().toISOString(), jobs };
+              succeeded += 1;
+              metric.status = "succeeded";
+              metric.jobs = jobs.length;
+              metric.countryJobs = countCountries(jobs);
+              delete metric.error;
+            } catch (error) {
+              retry.push(source);
+              const message = error instanceof Error ? error.message : String(error);
+              failures.push({ source: source.slug, error: message });
+              metric.error = message;
+            } finally {
+              metric.durationMs += Date.now() - attemptStartedAt;
+              clearTimeout(timer);
+            }
+          }
+        }
+
+        await Promise.all(Array.from({ length: Math.min(concurrency, pending.length) }, worker));
+        pending = retry;
+        finalFailures = failures;
+      }
+
+      const finishedAt = now().toISOString();
+      const report: CrawlReport = {
+        startedAt, finishedAt, considered, selected: selectedSources.length,
+        cached: considered - eligibleSources.length, deferred: eligibleSources.length - selectedSources.length,
+        succeeded, failed: finalFailures, sources: selectedSources.map((source) => metrics.get(source.slug)!),
+      };
+      await options.store.write({ version: 1, updatedAt: finishedAt, partitions, lastCrawl: report });
+      if (options.onCrawled) {
+        const crawled = selectedSources.filter((source) => metrics.get(source.slug)!.status === "succeeded");
+        await Promise.all(crawled.map((source) => Promise.resolve().then(() => options.onCrawled!(source, partitions[source.slug]!)).catch(() => undefined)));
+      }
+      return report;
+    },
+  };
+}
+
+function partitionTime(value: string | undefined): number {
+  const timestamp = Date.parse(value ?? "");
+  return Number.isFinite(timestamp) ? timestamp : Number.NEGATIVE_INFINITY;
+}
+
+function countCountries(jobs: Job[]): Record<string, number> {
+  const counts: Record<string, number> = {};
+  for (const job of jobs) for (const country of job.eligibleCountries) counts[country] = (counts[country] ?? 0) + 1;
+  return Object.fromEntries(Object.entries(counts).sort(([left], [right]) => left.localeCompare(right)));
+}