openings 0.1.0

package/src/job-search-preparation.ts

@@ -0,0 +1,116 @@
+import type { SnapshotStore } from "./crawler.ts";
+import { projectJobCoverage, type JobCoverageSummary } from "./job-coverage.ts";
+import type { CrawlScope } from "./local-jobs.ts";
+import type { Company, CrawlReport, JobSnapshot } from "./types.ts";
+
+export interface PrepareJobSearchResult {
+  status: "ready" | "partial";
+  nextAction: "ready" | "call_again" | "retry_later";
+  continuation?: string;
+  networkAttempted: boolean;
+  sources: { catalog: number; indexed: number; fresh: number; stale: number; missing: number; pending: number };
+  coverage: JobCoverageSummary;
+  crawl?: Pick<CrawlReport, "selected" | "succeeded" | "failed">;
+}
+
+export function createJobSearchPreparer(options: {
+  sources: Company[];
+  store: SnapshotStore;
+  crawl(scope: CrawlScope): Promise<CrawlReport>;
+  /** Optional published snapshot used instead of crawling when no local snapshot exists yet. */
+  seed?(): Promise<JobSnapshot | null>;
+  now?: () => Date;
+  freshnessDays?: number;
+  batchSize?: number;
+}) {
+  const now = options.now ?? (() => new Date());
+  const freshnessMs = (options.freshnessDays ?? 14) * 86_400_000;
+  const batchSize = Math.max(1, Math.min(10, Math.trunc(options.batchSize ?? 10)));
+  return {
+    async prepare(value: unknown): Promise<PrepareJobSearchResult> {
+      const input = validateInput(value, options.sources);
+      const { countries } = input;
+      let snapshot = await options.store.read();
+      if (!snapshot && options.seed) {
+        const seeded = await options.seed().catch(() => null);
+        if (seeded) { await options.store.write(seeded); snapshot = seeded; }
+      }
+      const pendingBefore = pendingSources(options.sources, snapshot, now(), freshnessMs);
+      const selected = pendingBefore.filter((source) => !input.attempted.has(source.slug)).slice(0, batchSize).map((source) => source.slug);
+      const crawl = selected.length ? await options.crawl({ slugs: selected }) : undefined;
+      if (crawl) snapshot = await options.store.read();
+      if (!snapshot) throw new Error("Job search preparation did not produce a local snapshot");
+
+      const state = sourceState(options.sources, snapshot, now(), freshnessMs);
+      const attempted = new Set([...input.attempted, ...selected]);
+      const hasUnattemptedPending = pendingSources(options.sources, snapshot, now(), freshnessMs).some((source) => !attempted.has(source.slug));
+      const nextAction = state.pending === 0 ? "ready" : hasUnattemptedPending ? "call_again" : "retry_later";
+      return {
+        status: state.pending === 0 ? "ready" : "partial",
+        nextAction,
+        ...(nextAction === "call_again" ? { continuation: encodeContinuation(countries, attempted) } : {}),
+        networkAttempted: Boolean(crawl),
+        sources: { catalog: options.sources.length, ...state },
+        coverage: projectJobCoverage(options.sources, snapshot, countries),
+        ...(crawl ? { crawl: { selected: crawl.selected, succeeded: crawl.succeeded, failed: crawl.failed } } : {}),
+      };
+    },
+  };
+}
+
+function pendingSources(sources: Company[], snapshot: Awaited<ReturnType<SnapshotStore["read"]>>, now: Date, freshnessMs: number): Company[] {
+  const cutoff = now.getTime() - freshnessMs;
+  return sources.filter((source) => {
+    const fetchedAt = Date.parse(snapshot?.partitions[source.slug]?.fetchedAt ?? "");
+    return !Number.isFinite(fetchedAt) || fetchedAt < cutoff;
+  });
+}
+
+function sourceState(sources: Company[], snapshot: NonNullable<Awaited<ReturnType<SnapshotStore["read"]>>>, now: Date, freshnessMs: number) {
+  const cutoff = now.getTime() - freshnessMs;
+  let missing = 0;
+  let stale = 0;
+  for (const source of sources) {
+    const partition = snapshot.partitions[source.slug];
+    if (!partition) missing += 1;
+    else {
+      const fetchedAt = Date.parse(partition.fetchedAt);
+      if (!Number.isFinite(fetchedAt) || fetchedAt < cutoff) stale += 1;
+    }
+  }
+  const indexed = sources.length - missing;
+  return { indexed, fresh: indexed - stale, stale, missing, pending: stale + missing };
+}
+
+function validateInput(value: unknown, sources: Company[]): { countries: string[]; attempted: Set<string> } {
+  if (!isRecord(value)) throw new Error("prepare_job_search input must be an object");
+  const unknown = Object.keys(value).find((key) => key !== "countries" && key !== "continuation");
+  if (unknown) throw new Error(`prepare_job_search does not accept field: ${unknown}`);
+  if (!Array.isArray(value.countries) || value.countries.length === 0 || value.countries.length > 20) {
+    throw new Error("prepare_job_search requires between 1 and 20 countries");
+  }
+  if (!value.countries.every((country) => typeof country === "string" && /^[a-z]{2}$/iu.test(country))) {
+    throw new Error("countries must contain only two-letter country codes");
+  }
+  const countries = [...new Set(value.countries.map((country) => String(country).toUpperCase()))];
+  if (value.continuation === undefined) return { countries, attempted: new Set() };
+  if (typeof value.continuation !== "string" || value.continuation.length > 16_384) throw new Error("continuation must be a valid preparation token");
+  try {
+    const parsed = JSON.parse(Buffer.from(value.continuation, "base64url").toString("utf8")) as unknown;
+    if (!isRecord(parsed) || parsed.version !== 1 || !Array.isArray(parsed.countries) || !Array.isArray(parsed.attempted)) throw new Error("invalid");
+    if (JSON.stringify(parsed.countries) !== JSON.stringify(countries)) throw new Error("country mismatch");
+    const known = new Set(sources.map((source) => source.slug));
+    if (!parsed.attempted.every((slug) => typeof slug === "string" && known.has(slug))) throw new Error("unknown source");
+    return { countries, attempted: new Set(parsed.attempted as string[]) };
+  } catch {
+    throw new Error("continuation must be a valid preparation token for the requested countries");
+  }
+}
+
+function encodeContinuation(countries: string[], attempted: Set<string>): string {
+  return Buffer.from(JSON.stringify({ version: 1, countries, attempted: [...attempted].sort() }), "utf8").toString("base64url");
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/jobposting-probe.ts

@@ -0,0 +1,167 @@
+import { lstat, mkdir, readFile, realpath } from "node:fs/promises";
+import { dirname, relative, resolve } from "node:path";
+import { atomicJson } from "./atomic-file.ts";
+import { fetchSafeGet, SafeGetError } from "./safe-get.ts";
+import { stampReport, type ReportMeta } from "./report-meta.ts";
+
+export interface ProbePage { status: number; finalUrl: string; contentType: string; body: string; requestCount?: number }
+interface ProbeOptions { fetchPage?: (url: string, maxRequests: number, companyDomain: string) => Promise<ProbePage>; companyLimit?: number; now?: Date; delayMs?: number; sleep?: (ms: number) => Promise<void>; reportRoot?: string }
+interface CompanyResult { companyName: string; companyDomain: string; originHost: string; inputReference: string; status: "qualified" | "unresolved" | "failed"; acceptedJobs: number; attemptedDetailPages: number; discoveredPostings: number; admissionFailures: number; requestAttempts: number; failedRequests: number; jobs: AcceptedJob[]; issues: string[] }
+interface AcceptedJob { key: string; signature: string; identifier: boolean; explicitCountry: boolean }
+export interface JobPostingProbeReport extends ReportMeta { sample: Array<{ companyName: string; companyDomain: string; inputReference: string }>; companiesChecked: number; acceptedJobs: number; jobsWithIdentifier: number; identifierPresencePercent: number; jobsWithExplicitCountry: number; explicitCountryPercent: number; requests: number; stopped: boolean; stopReason?: string; viability: "pending_second_phase" | "passed" | "failed"; companies: Array<Omit<CompanyResult, "jobs" | "originHost">> }
+
+export async function probeJobPostingJsonLd(inputPath: string, catalogPath: string, reportPath: string, options: ProbeOptions = {}): Promise<JobPostingProbeReport> {
+  await assertReportPath(reportPath, options.reportRoot ?? ".openings");
+  const fetchPage = options.fetchPage ?? ((url, maxRequests, domain) => fetchSafeGet(url, { maxRequests, allowedDomain: domain, allowedOrigin: new URL(url).hostname }));
+  const [markdown, catalogText] = await Promise.all([readFile(inputPath, "utf8"), readFile(catalogPath, "utf8")]);
+  const catalog = JSON.parse(catalogText) as Record<string, { companyDomain?: string }>;
+  const known = [...new Set(Object.values(catalog).map((company) => normalizeDomain(company.companyDomain ?? "")).filter(Boolean))];
+  const seeds = parseSeeds(markdown).filter((seed) => !known.some((domain) => seed.companyDomain === domain || seed.companyDomain.endsWith(`.${domain}`))).slice(0, Math.min(20, options.companyLimit ?? 20));
+  let requests = 0;
+  const delayMs = options.delayMs ?? 500;
+  const sleep = options.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  let requestGate = Promise.resolve();
+  let fatalReason: string | undefined;
+  const get = async (url: string, domain: string) => {
+    let release!: () => void; const previous = requestGate; requestGate = new Promise<void>((resolve) => { release = resolve; }); await previous;
+    try {
+      if (fatalReason) throw new SafetyViolation(fatalReason);
+      const remaining = 320 - requests; if (remaining <= 0) { fatalReason = "global_request_cap"; throw new SafetyViolation(fatalReason); }
+      if (requests && delayMs) await sleep(delayMs);
+      try { const page = await fetchPage(url, remaining, domain); requests += page.requestCount ?? 1; if (requests > 320) { fatalReason = "global_request_cap"; throw new SafetyViolation(fatalReason); } return page; }
+      catch (error) { if (error instanceof SafeGetError) { requests += error.requestCount; if (error.safety) fatalReason = error.message; } else if (!(error instanceof SafetyViolation)) requests += 1; throw error; }
+    } finally { release(); }
+  };
+  const processCompany = async (seed: ReturnType<typeof parseSeeds>[number]): Promise<CompanyResult> => {
+    const issues: string[] = [];
+    let attemptedDetailPages = 0;
+    let discoveredPostings = 0;
+    let admissionFailures = 0;
+    let requestAttempts = 0;
+    let failedRequests = 0;
+    const jobs = new Map<string, AcceptedJob>();
+    const conflicts = new Set<string>();
+    try {
+      const companyGet = async (url: string) => { try { const page = await get(url, seed.companyDomain); requestAttempts += page.requestCount ?? 1; if (page.status === 429 || page.status >= 500) { failedRequests += 1; throw new Error(`transport_http_${page.status}`); } return page; } catch (error) { if (error instanceof SafeGetError) { requestAttempts += error.requestCount; failedRequests += error.failedRequestCount; if (error.safety) throw new SafetyViolation(error.message); } throw error; } };
+      const policies = new Map<string, ReturnType<typeof parseRobots>>();
+      const policyFor = async (host: string) => { const normalized = normalizeDomain(host); const existing = policies.get(normalized); if (existing) return existing; const robots = await companyGet(`https://${normalized}/robots.txt`); const policy = robotsPolicy(robots); policies.set(normalized, policy); return policy; };
+      const rules = await policyFor(seed.originHost);
+      const sitemapUrls = rules.sitemaps.length ? rules.sitemaps : [`https://${seed.originHost}/sitemap.xml`];
+      const detailUrls: string[] = [];
+      const sitemapQueue = sitemapUrls.map((url) => ({ url, depth: 0 }));
+      const seenSitemaps = new Set<string>();
+      while (sitemapQueue.length && seenSitemaps.size < 5) {
+        const item = sitemapQueue.shift()!;
+        const sitemapUrl = item.url;
+        if (seenSitemaps.has(sitemapUrl)) continue;
+        if (!allowedUrl(sitemapUrl, seed.companyDomain)) continue;
+        const sitemapParsed = new URL(sitemapUrl); const sitemapRules = await policyFor(sitemapParsed.hostname);
+        if (!robotsAllows(sitemapParsed, sitemapRules)) continue;
+        seenSitemaps.add(sitemapUrl);
+        const sitemap = await companyGet(sitemapUrl);
+        if (sitemap.status !== 200 || !xmlMediaTypes.has(mediaType(sitemap.contentType))) continue;
+        if (/<!DOCTYPE|<!ENTITY/i.test(sitemap.body)) { fatalReason = "unsafe_xml"; throw new SafetyViolation(fatalReason); }
+        for (const url of sitemapLocations(sitemap.body)) {
+          if (!allowedUrl(url, seed.companyDomain)) continue;
+          const parsed = new URL(url); const urlRules = await policyFor(parsed.hostname); if (!robotsAllows(parsed, urlRules)) continue;
+          if (item.depth < 1 && /\.xml(?:$|\?)/i.test(url)) sitemapQueue.push({ url, depth: item.depth + 1 });
+          else detailUrls.push(url);
+        }
+      }
+      for (const url of [...new Set(detailUrls)].slice(0, 10)) {
+        attemptedDetailPages += 1; const page = await companyGet(url);
+        if (page.status !== 200 || mediaType(page.contentType) !== "text/html") continue;
+        const postings = extractJobPostings(page.body);
+        discoveredPostings += postings.length;
+        if (postings.length > 1) { fatalReason = "multiple_jobpostings"; throw new SafetyViolation(fatalReason); }
+        for (const job of postings) {
+          const result = validateJob(job, seed.companyDomain, url, options.now ?? new Date());
+          if (!result.accepted) { issues.push(result.reason); admissionFailures += 1; continue; }
+          const previous = jobs.get(result.job.key);
+          if (previous && previous.signature !== result.job.signature) { conflicts.add(result.job.key); jobs.delete(result.job.key); issues.push("identity_conflict"); }
+          else if (!conflicts.has(result.job.key)) jobs.set(result.job.key, result.job);
+        }
+      }
+      return { ...seed, status: jobs.size ? "qualified" : "unresolved", acceptedJobs: jobs.size, attemptedDetailPages, discoveredPostings, admissionFailures, requestAttempts, failedRequests, jobs: [...jobs.values()], issues: [...new Set(issues)] };
+    } catch (error) {
+      if (error instanceof SafetyViolation) throw error;
+      if (!failedRequests) failedRequests = 1;
+      return { ...seed, status: "failed", acceptedJobs: 0, attemptedDetailPages, discoveredPostings, admissionFailures, requestAttempts, failedRequests, jobs: [], issues: [error instanceof Error ? error.message : String(error)] };
+    }
+  };
+  const companies: CompanyResult[] = [];
+  let stopReason: string | undefined;
+  try {
+    companies.push(...await runPhase(seeds.slice(0, 10), processCompany));
+    const firstGate = healthStop(companies);
+    if (firstGate) stopReason = firstGate;
+    else if (seeds.length > 10) companies.push(...await runPhase(seeds.slice(10, 20), processCompany));
+  } catch (error) { stopReason = error instanceof Error ? error.message : String(error); fatalReason = stopReason; }
+  const jobs = companies.flatMap((company) => company.jobs);
+  const acceptedJobs = jobs.length;
+  const jobsWithIdentifier = jobs.filter((job) => job.identifier).length;
+  const jobsWithExplicitCountry = jobs.filter((job) => job.explicitCountry).length;
+  const viability: JobPostingProbeReport["viability"] = stopReason ? "failed" : seeds.length < 20 || companies.length < 20 ? "pending_second_phase" : companies.filter((company) => company.status === "qualified").length >= 5 && acceptedJobs >= 20 && percent(jobsWithIdentifier, acceptedJobs) >= 80 && percent(jobsWithExplicitCountry, acceptedJobs) >= 95 ? "passed" : "failed";
+  const publicCompanies = companies.map(({ jobs: _jobs, originHost: _originHost, ...company }) => company);
+  const sample = seeds.map(({ companyName, companyDomain, inputReference }) => ({ companyName, companyDomain, inputReference }));
+  const report = stampReport("jobposting-jsonld-probe:1", 1, { sample, companiesChecked: companies.length, acceptedJobs, jobsWithIdentifier, identifierPresencePercent: percent(jobsWithIdentifier, acceptedJobs), jobsWithExplicitCountry, explicitCountryPercent: percent(jobsWithExplicitCountry, acceptedJobs), requests, stopped: Boolean(stopReason), ...(stopReason ? { stopReason } : {}), viability, companies: publicCompanies }, options.now);
+  await atomicJson(reportPath, report);
+  return report;
+}
+
+function parseSeeds(markdown: string) {
+  const rows: Array<{ companyName: string; companyDomain: string; originHost: string; inputReference: string }> = [];
+  const pattern = /<li>\s*<a\s+href="(https:\/\/[^"#]+)[^"]*"[^>]*>([^<]+)<\/a>/gi;
+  for (const match of markdown.matchAll(pattern)) { const url = new URL(match[1]!); const domain = normalizeDomain(url.hostname); if (!thirdParty(domain)) rows.push({ companyName: decode(match[2]!).trim(), companyDomain: domain, originHost: url.hostname.toLowerCase(), inputReference: match[1]! }); }
+  return [...new Map(rows.map((row) => [row.companyDomain, row])).values()];
+}
+interface RobotsRule { allow: boolean; pattern: string }
+function robotsPolicy(page: ProbePage) { if (page.status === 401 || page.status === 403) return { sitemaps: [], rules: [{ allow: false, pattern: "/" }] }; if (page.status === 404 || page.status === 410) return { sitemaps: [], rules: [] as RobotsRule[] }; if (page.status < 200 || page.status >= 300) throw new Error(`robots_unavailable_${page.status}`); return parseRobots(page.body); }
+function parseRobots(body: string) {
+  const sitemaps: string[] = []; const groups: Array<{ agents: string[]; rules: RobotsRule[] }> = []; let group: { agents: string[]; rules: RobotsRule[] } | undefined; let sawRule = false;
+  for (const raw of body.split(/\r?\n/)) { const line = raw.replace(/#.*/, "").trim(); if (!line) continue; const split = line.indexOf(":"); if (split < 0) continue; const key = line.slice(0, split).trim().toLowerCase(); const value = line.slice(split + 1).trim();
+    if (key === "sitemap" && value) { sitemaps.push(value); continue; }
+    if (key === "user-agent") { if (!group || sawRule) { group = { agents: [], rules: [] }; groups.push(group); sawRule = false; } group.agents.push(value.toLowerCase()); continue; }
+    if ((key === "allow" || key === "disallow") && group) { sawRule = true; if (value) group.rules.push({ allow: key === "allow", pattern: value }); }
+  }
+  const exact = groups.filter((item) => item.agents.some((agent) => "openings".startsWith(agent) && agent !== "*")); const selected = exact.length ? exact : groups.filter((item) => item.agents.includes("*"));
+  return { sitemaps, rules: selected.flatMap((item) => item.rules) };
+}
+function robotsAllows(url: URL, policy: ReturnType<typeof parseRobots>) { const target = `${url.pathname}${url.search}`; const matches = policy.rules.flatMap((rule) => { const source = `^${rule.pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\\\$$/, "$")}`; return new RegExp(source).test(target) ? [{ ...rule, length: rule.pattern.replace(/[*$]/g, "").length }] : []; }); matches.sort((a, b) => b.length - a.length || Number(b.allow) - Number(a.allow)); return matches[0]?.allow ?? true; }
+function sitemapLocations(xml: string) { return [...xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map((match) => decode(match[1]!)); }
+function allowedUrl(value: string, domain: string) { try { const url = new URL(value); const host = normalizeDomain(url.hostname); return url.protocol === "https:" && (!url.port || url.port === "443") && (host === domain || host.endsWith(`.${domain}`)); } catch { return false; } }
+function extractJobPostings(html: string): Record<string, unknown>[] { const results: Record<string, unknown>[] = []; for (const match of html.matchAll(/<script\b[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) { try { const value: unknown = JSON.parse(match[1]!.trim()); visit(value, results); } catch { /* malformed JSON-LD is inert */ } } return results; }
+function visit(value: unknown, output: Record<string, unknown>[]) { if (Array.isArray(value)) { value.forEach((item) => visit(item, output)); return; } if (!record(value)) return; const type = value["@type"]; if (type === "JobPosting" || Array.isArray(type) && type.includes("JobPosting")) output.push(value); const graph = value["@graph"]; if (graph) visit(graph, output); }
+function validateJob(job: Record<string, unknown>, domain: string, pageUrl: string, now: Date): { accepted: false; reason: string } | { accepted: true; reason: ""; job: AcceptedJob } { const org = record(job.hiringOrganization) ? job.hiringOrganization : undefined; if (org?.sameAs !== undefined && !organizationDomainsMatch(org.sameAs, domain)) return fail("organization_domain_conflict"); if (!text(job.title) || !text(job.description) || !validDate(job.datePosted) || !text(org?.name)) return fail("missing_required_field"); if (job.validThrough !== undefined && (!validDate(job.validThrough) || Date.parse(String(job.validThrough)) < now.getTime())) return fail("expired_or_invalid"); const physicalCountry = hasPhysicalCountry(job.jobLocation); const remote = job.jobLocationType === "TELECOMMUTE"; const explicitCountry = physicalCountry || remote && hasApplicantCountry(job.applicantLocationRequirements); if (!explicitCountry) return fail("missing_country"); const identifierValue = record(job.identifier) && text(job.identifier.value) ? job.identifier.value.trim() : undefined; const url = typeof job.url === "string" ? job.url : pageUrl; if (!identifierValue && !allowedUrl(url, domain)) return fail("missing_identity"); const key = identifierValue ? `${domain}:id:${identifierValue}` : `${domain}:url:${normalizeJobUrl(url)}`; const signature = stableStringify(job); return { accepted: true, reason: "", job: { key, signature, identifier: Boolean(identifierValue), explicitCountry } }; }
+function organizationDomainsMatch(value: unknown, domain: string): boolean { const values = Array.isArray(value) ? value : [value]; return values.length > 0 && values.every((item) => typeof item === "string" && allowedUrl(item, domain)); }
+function hasPhysicalCountry(value: unknown): boolean { if (Array.isArray(value)) return value.some(hasPhysicalCountry); return record(value) && record(value.address) && validCountry(value.address.addressCountry); }
+function hasApplicantCountry(value: unknown): boolean { if (Array.isArray(value)) return value.some(hasApplicantCountry); if (!record(value)) return false; const country = record(value.address) ? value.address.addressCountry : value.name; return validCountry(country); }
+function fail(reason: string): { accepted: false; reason: string } { return { accepted: false, reason }; }
+function thirdParty(domain: string) { return ["linkedin.com", "indeed.com", "github.com", "twitter.com", "angel.co", "wellfound.com", "lever.co", "ashbyhq.com", "greenhouse.io", "myworkdayjobs.com", "recruitee.com"].some((value) => domain === value || domain.endsWith(`.${value}`)); }
+function normalizeDomain(value: string) { return value.trim().toLowerCase().replace(/^www\./, ""); }
+function normalizeJobUrl(value: string) { const url = new URL(value); url.hash = ""; if (url.port === "443") url.port = ""; return url.href; }
+function stableStringify(value: unknown): string { if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`; if (record(value)) return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`; return JSON.stringify(value); }
+function decode(value: string) { return value.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'"); }
+function validDate(value: unknown) { return typeof value === "string" && Number.isFinite(Date.parse(value)); }
+function text(value: unknown): value is string { return typeof value === "string" && Boolean(value.trim()); }
+function record(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }
+function percent(value: number, total: number) { return total ? Math.round(value / total * 10_000) / 100 : 0; }
+async function runPhase<T, R>(values: T[], operation: (value: T) => Promise<R>): Promise<R[]> { const output = new Array<R>(values.length); let cursor = 0; let failure: unknown; async function worker() { while (cursor < values.length && !failure) { const index = cursor++; try { output[index] = await operation(values[index]!); } catch (error) { failure = error; } } } await Promise.all(Array.from({ length: Math.min(3, values.length) }, worker)); if (failure) throw failure; return output; }
+function healthStop(companies: CompanyResult[]): string | undefined { if (companies.filter((company) => company.status === "qualified").length < 2) return "phase_one_low_yield"; const attempts = companies.reduce((sum, company) => sum + company.requestAttempts, 0); const failed = companies.reduce((sum, company) => sum + company.failedRequests, 0); if (attempts && failed / attempts > 0.1) return "phase_one_transport_failures"; const discovered = companies.reduce((sum, company) => sum + company.discoveredPostings, 0); const admissionFailures = companies.reduce((sum, company) => sum + company.admissionFailures, 0); if (discovered && admissionFailures / discovered > 0.5) return "phase_one_schema_failures"; return undefined; }
+class SafetyViolation extends Error {}
+const countryNames = new Set(["india", "united states", "usa", "united kingdom", "canada", "germany", "france", "australia", "singapore", "japan", "netherlands", "ireland", "spain", "italy", "brazil", "mexico"]);
+function validCountry(value: unknown): boolean { return typeof value === "string" && (/^[A-Z]{2}$/i.test(value.trim()) || countryNames.has(value.trim().toLowerCase())); }
+function mediaType(value: string) { return value.split(";", 1)[0]!.trim().toLowerCase(); }
+const xmlMediaTypes = new Set(["application/xml", "text/xml", "application/sitemap+xml"]);
+async function assertReportPath(path: string, root: string) {
+  const expectedRoot = resolve(root);
+  const target = resolve(path);
+  const lexicalRelation = relative(expectedRoot, target);
+  if (!lexicalRelation || lexicalRelation.startsWith("..")) throw new Error("JobPosting probe report must be a file under .openings");
+  await mkdir(expectedRoot, { recursive: true });
+  if ((await lstat(expectedRoot)).isSymbolicLink()) throw new Error("JobPosting probe report must be a file under .openings");
+  await mkdir(dirname(target), { recursive: true });
+  const [actualRoot, actualParent] = await Promise.all([realpath(expectedRoot), realpath(dirname(target))]);
+  const relation = relative(actualRoot, actualParent);
+  if (relation.startsWith("..") || resolve(actualRoot, relation) !== actualParent) throw new Error("JobPosting probe report must be a file under .openings");
+}

package/src/local-jobs.ts

@@ -0,0 +1,113 @@
+import type { FetchJobsObserver } from "./catalog.ts";
+import { createCrawler, type SnapshotStore } from "./crawler.ts";
+import { createSnapshotCatalog } from "./snapshot-catalog.ts";
+import type { Company, CrawlReport, Job, JobPartition, JobSnapshot, JobSummary, SearchQuery } from "./types.ts";
+
+interface LocalJobsOptions {
+  sources: Company[];
+  store: SnapshotStore;
+  fetchJobs(source: Company, signal?: AbortSignal, observer?: FetchJobsObserver): Promise<Job[]>;
+  concurrency?: number;
+  timeoutMs?: number;
+  maxAttempts?: number;
+  sourceStartDelayMs?: number;
+  sourceFreshnessMs?: number;
+  sourceLimit?: number;
+  workdayPageDelayMs?: number;
+  now?: () => Date;
+  onCrawled?(source: Company, partition: JobPartition): void | Promise<void>;
+}
+
+export interface CrawlScope {
+  country?: string;
+  countries?: string[];
+  slugs?: string[];
+}
+
+export interface SnapshotStatus {
+  updatedAt: string;
+  ageDays: number;
+  stale: boolean;
+  refreshed: boolean;
+  sources: number;
+  jobs: number;
+  failures: number;
+}
+
+export function createLocalJobs(options: LocalJobsOptions) {
+  const now = options.now ?? (() => new Date());
+  const crawler = createCrawler({ ...options, now });
+  const catalog = createSnapshotCatalog(options.store);
+
+  async function crawl(scope: CrawlScope = {}): Promise<CrawlReport> {
+    const selected = selectSources(options.sources, scope);
+    return crawler.crawl(selected, { prune: !scope.country && !scope.countries && !scope.slugs });
+  }
+
+  async function ensureFresh(offline: boolean, staleDays: number): Promise<{ snapshot: JobSnapshot; refreshed: boolean }> {
+    let snapshot = await options.store.read();
+    const stale = !snapshot || snapshotIsStale(snapshot, staleDays, now());
+    if (stale && !offline) {
+      await crawl();
+      snapshot = await options.store.read();
+      if (!snapshot) throw new Error("Crawl completed without producing a snapshot");
+      return { snapshot, refreshed: true };
+    }
+    if (!snapshot) throw new Error("No local snapshot. Run `openings crawl` or search without --offline.");
+    return { snapshot, refreshed: false };
+  }
+
+  return {
+    crawl,
+    async search(query: SearchQuery, settings: { offline: boolean; staleDays: number }): Promise<{ jobs: JobSummary[]; snapshot: SnapshotStatus }> {
+      const ready = await ensureFresh(settings.offline, settings.staleDays);
+      return { jobs: await catalog.search(query), snapshot: snapshotStatus(ready.snapshot, settings.staleDays, now(), ready.refreshed) };
+    },
+    async get(id: string, settings: { offline: boolean; staleDays: number }): Promise<{ job: Job | null; snapshot: SnapshotStatus }> {
+      const ready = await ensureFresh(settings.offline, settings.staleDays);
+      return { job: await catalog.get(id), snapshot: snapshotStatus(ready.snapshot, settings.staleDays, now(), ready.refreshed) };
+    },
+    ensureFresh,
+    catalog,
+  };
+}
+
+function selectSources(sources: Company[], scope: CrawlScope): Company[] {
+  if (scope.slugs) {
+    const wanted = new Set(scope.slugs);
+    const selected = sources.filter((source) => wanted.has(source.slug));
+    const missing = [...wanted].filter((slug) => !selected.some((source) => source.slug === slug));
+    if (missing.length) throw new Error(`Unknown company source(s): ${missing.join(", ")}`);
+    return selected;
+  }
+  if (scope.country) {
+    const country = scope.country.toUpperCase();
+    return sources.filter((source) => source.cohorts?.includes(country));
+  }
+  if (scope.countries) {
+    const countries = new Set(scope.countries.map((country) => country.toUpperCase()));
+    return sources.filter((source) => source.cohorts?.some((country) => countries.has(country)));
+  }
+  return sources;
+}
+
+export function snapshotIsStale(snapshot: JobSnapshot, staleDays: number, now: Date): boolean {
+  const timestamps = Object.values(snapshot.partitions).map((partition) => Date.parse(partition.fetchedAt));
+  if (timestamps.length === 0) return true;
+  const oldest = Math.min(...timestamps);
+  return now.getTime() - oldest > staleDays * 86_400_000;
+}
+
+export function snapshotStatus(snapshot: JobSnapshot, staleDays: number, now: Date, refreshed: boolean): SnapshotStatus {
+  const timestamps = Object.values(snapshot.partitions).map((partition) => Date.parse(partition.fetchedAt));
+  const oldest = timestamps.length ? Math.min(...timestamps) : Date.parse(snapshot.updatedAt);
+  return {
+    updatedAt: snapshot.updatedAt,
+    ageDays: Math.max(0, Math.floor((now.getTime() - oldest) / 86_400_000)),
+    stale: snapshotIsStale(snapshot, staleDays, now),
+    refreshed,
+    sources: Object.keys(snapshot.partitions).length,
+    jobs: Object.values(snapshot.partitions).reduce((sum, partition) => sum + partition.jobs.length, 0),
+    failures: snapshot.lastCrawl.failed.length,
+  };
+}

package/src/locations.ts

@@ -0,0 +1,180 @@
+import type { Job } from "./types.ts";
+
+const INDIA_PLACES = [
+  "india", "bharat",
+  "andhra pradesh", "arunachal pradesh", "assam", "bihar", "chhattisgarh", "goa", "gujarat",
+  "haryana", "himachal pradesh", "jharkhand", "karnataka", "kerala", "madhya pradesh", "maharashtra",
+  "manipur", "meghalaya", "mizoram", "nagaland", "odisha", "orissa", "punjab", "rajasthan", "sikkim",
+  "tamil nadu", "telangana", "tripura", "uttar pradesh", "uttarakhand", "west bengal",
+  "andaman", "chandigarh", "dadra", "daman", "delhi", "jammu", "kashmir", "ladakh", "lakshadweep", "puducherry",
+  "bengaluru", "bangalore", "hyderabad", "pune", "chennai", "mumbai", "gurugram", "gurgaon", "noida",
+  "kolkata", "ahmedabad", "kochi", "cochin", "jaipur", "coimbatore", "indore", "thiruvananthapuram",
+  "mysuru", "mysore", "bhubaneswar", "lucknow", "nagpur", "visakhapatnam", "vizag", "vadodara", "surat",
+];
+
+const INDIA_PATTERN = new RegExp(`\\b(${INDIA_PLACES.map(escapeRegExp).join("|")})\\b`, "i");
+const INDIA_INCLUSIVE_REGION_PATTERN = /\b(apac|asia|asia[ -]pacific|worldwide|anywhere|global)\b/i;
+const INDIA_EXCLUSION_PATTERN = /\b(not available|unavailable|excluding|except|cannot hire|can't hire|unable to hire|do not hire|does not hire)\b.{0,80}\b(india|apac|asia)\b|\b(india|apac|asia)\b.{0,40}\b(excluded|not eligible|not supported)\b/i;
+const INDIA_ELIGIBILITY_PATTERN = /\b(remote (?:in|from)|available (?:in|to)|open to|hiring (?:in|from)|candidates? (?:in|from)|applicants? (?:in|from)|work (?:in|from)|based in)\b.{0,80}\b(india|apac|asia)\b|\b(india|apac|asia)\b.{0,40}\b(remote|candidates?|applicants?|eligible|hiring)\b/i;
+
+export function normalizeLocation(value: string): string {
+  const aliases: Record<string, string> = {
+    bangalore: "bengaluru",
+    gurgaon: "gurugram",
+    bombay: "mumbai",
+    calcutta: "kolkata",
+    madras: "chennai",
+    mysore: "mysuru",
+    orissa: "odisha",
+  };
+  return value.trim().toLocaleLowerCase().replace(/\b(bangalore|gurgaon|bombay|calcutta|madras|mysore|orissa)\b/g, (name) => aliases[name] ?? name);
+}
+
+export function isExplicitlyIndiaEligible(job: Job): boolean {
+  if (job.remote && INDIA_EXCLUSION_PATTERN.test(`${job.location}\n${job.description}`)) return false;
+  if (INDIA_PATTERN.test(job.location)) return true;
+  if (!job.remote) return false;
+  return INDIA_INCLUSIVE_REGION_PATTERN.test(job.location) || INDIA_ELIGIBILITY_PATTERN.test(job.description);
+}
+
+export function classifyJob(job: Job): Job {
+  const workMode = inferWorkMode(job.location, job.workMode);
+  const evidence = `${job.location}\n${job.description}`;
+  const excludedCountries = detectExcludedCountries(evidence);
+  const eligibleCountries = detectCountries(job.location);
+  if (workMode === "remote") eligibleCountries.push(...detectEligibleCountries(job.description));
+  const uniqueEligibleCountries = [...new Set(eligibleCountries)].filter((code) => !excludedCountries.includes(code));
+  const eligibleRegions = workMode === "remote" ? detectRegions(job.location, job.description) : [];
+  const withMode = { ...job, remote: workMode === "remote", workMode };
+  const indiaLocation = INDIA_PATTERN.test(job.location);
+  const indiaDescription = workMode === "remote" && INDIA_ELIGIBILITY_PATTERN.test(job.description);
+  if ((indiaLocation || indiaDescription) && !excludedCountries.includes("IN") && !uniqueEligibleCountries.includes("IN")) uniqueEligibleCountries.push("IN");
+  return {
+    ...withMode,
+    eligibleCountries: uniqueEligibleCountries.sort(),
+    excludedCountries,
+    eligibleRegions,
+    eligibilityConfidence: uniqueEligibleCountries.length > 0 ? "explicit" : eligibleRegions.length > 0 ? "inferred" : "unknown",
+  };
+}
+
+export function isEligibleForCountry(job: Job, country: string): boolean {
+  const code = country.toUpperCase();
+  if (job.excludedCountries.includes(code)) return false;
+  if (job.eligibleCountries.includes(code)) return true;
+  if (job.eligibleRegions.includes("worldwide")) return true;
+  return job.eligibleRegions.some((region) => regionIncludes(region, code));
+}
+
+function detectExcludedCountries(value: string): string[] {
+  const excluded: string[] = [];
+  for (const rule of countryRules()) {
+    if (rule.exclusion.test(value)) excluded.push(rule.code);
+  }
+  return excluded.sort();
+}
+
+function inferWorkMode(location: string, supplied: Job["workMode"]): Job["workMode"] {
+  if (supplied !== "unknown") return supplied;
+  if (/\bremote\b/i.test(location)) return "remote";
+  if (/\bhybrid\b/i.test(location)) return "hybrid";
+  if (/\b(on[ -]?site|office[ -]?based)\b/i.test(location)) return "onsite";
+  return "unknown";
+}
+
+function detectRegions(location: string, description: string): string[] {
+  const regions: Array<[string, RegExp]> = [
+    ["worldwide", /\b(worldwide|anywhere|global)\b/i],
+    ["APAC", /\b(APAC|Asia[ -]Pacific)\b/i],
+    ["Asia", /\bAsia\b/i],
+    ["EMEA", /\bEMEA\b/i],
+    ["LATAM", /\b(LATAM|Latin America)\b/i],
+  ];
+  const fromLocation = regions.filter(([, pattern]) => pattern.test(location)).map(([name]) => name);
+  const eligibilityPhrase = /\b(open to|hiring (?:in|from|across)|(?:candidates?|applicants?) (?:in|from|across)|eligible (?:in|for|across)|work from|available (?:in|to))\b.{0,60}\b(worldwide|anywhere|global|APAC|Asia[ -]Pacific|Asia|EMEA|LATAM|Latin America)\b/gi;
+  const fromDescription: string[] = [];
+  for (const match of description.matchAll(eligibilityPhrase)) {
+    const value = match[2] ?? "";
+    const region = regions.find(([, pattern]) => pattern.test(value))?.[0];
+    if (region) fromDescription.push(region);
+  }
+  return [...new Set([...fromLocation, ...fromDescription])];
+}
+
+function detectCountries(location: string): string[] {
+  const matches: string[] = [];
+  for (const rule of countryRules()) {
+    if (rule.location.test(location) || rule.codeLocation.test(location)) matches.push(rule.code);
+  }
+  return matches;
+}
+
+function detectEligibleCountries(description: string): string[] {
+  const matches: string[] = [];
+  for (const rule of countryRules()) {
+    if (rule.eligibility.test(description)) matches.push(rule.code);
+  }
+  if (/\b(open to|hiring|candidates?|applicants?|eligible|remote (?:in|from)|work (?:in|from)|based in|available (?:in|to))\b.{0,80}\bGeorgia\b/i.test(description)) matches.push("GE");
+  return matches;
+}
+
+let cachedCountryNames: Array<[string, string]> | undefined;
+function countryNames(): Array<[string, string]> {
+  if (cachedCountryNames) return cachedCountryNames;
+  const display = new Intl.DisplayNames(["en"], { type: "region" });
+  const names: Array<[string, string]> = [];
+  for (let first = 65; first <= 90; first += 1) {
+    for (let second = 65; second <= 90; second += 1) {
+      const code = String.fromCharCode(first, second);
+      const name = display.of(code);
+      if (name && name !== code && !name.startsWith("Unknown Region")) names.push([code, name]);
+    }
+  }
+  cachedCountryNames = names;
+  return names;
+}
+
+let cachedCountryMatchers: Array<[string, string]> | undefined;
+function countryMatchers(): Array<[string, string]> {
+  if (cachedCountryMatchers) return cachedCountryMatchers;
+  const aliases: Record<string, string[]> = { US: ["United States", "USA", "U\\.S\\.A\\.?", "U\\.S\\.?"], GB: ["United Kingdom", "UK", "U\\.K\\.?"], AE: ["United Arab Emirates", "UAE"] };
+  cachedCountryMatchers = countryNames().map(([code, name]) => {
+    if (code === "GE") return [code, "(?:\\b(?:Tbilisi|Batumi|Kutaisi|Rustavi|Gori|Zugdidi),?\\s+Georgia\\b|^Georgia$|\\bGeorgia,?\\s+(?:Country|Europe)\\b)"];
+    const names = [name, ...(aliases[code] ?? [])];
+    return [code, names.length ? `\\b(?:${names.map(escapeRegExpUnlessPattern).join("|")})\\b` : "(?!)"];
+  });
+  return cachedCountryMatchers;
+}
+
+interface CountryRule { code: string; location: RegExp; codeLocation: RegExp; exclusion: RegExp; eligibility: RegExp }
+let cachedCountryRules: CountryRule[] | undefined;
+function countryRules(): CountryRule[] {
+  if (cachedCountryRules) return cachedCountryRules;
+  cachedCountryRules = countryMatchers().map(([code, country]) => ({
+    code,
+    location: new RegExp(country, "i"),
+    codeLocation: new RegExp(`(?:^|[,(/-]\\s*)${code}(?=\\s*(?:$|[,)/-]))`, "i"),
+    exclusion: new RegExp(`\\b(not available|unavailable|excluding|except|cannot hire|can't hire|unable to hire|do not hire|does not hire)\\b.{0,80}(?:${country})|(?:${country}).{0,40}\\b(excluded|not eligible|not supported)\\b`, "i"),
+    eligibility: new RegExp(`\\b(open to|hiring|candidates?|applicants?|eligible|remote (?:in|from)|work (?:in|from)|based in|available (?:in|to))\\b.{0,80}(?:${country})`, "i"),
+  }));
+  return cachedCountryRules;
+}
+
+function regionIncludes(region: string, code: string): boolean {
+  if (region === "worldwide") return true;
+  const groups: Record<string, string> = {
+    APAC: "AU BN CN FJ HK ID IN JP KH KR LA MM MN MO MY NP NZ PH PK SG TH TW VN",
+    Asia: "AE AF AM AZ BD BH BN BT CN CY GE HK ID IL IN IQ IR JO JP KG KH KP KR KW KZ LA LB LK MM MN MO MV MY NP OM PH PK PS QA SA SG SY TH TJ TL TM TR TW UZ VN YE",
+    EMEA: "AD AE AF AL AM AO AT AZ BA BE BF BG BH BI BJ BW BY CD CF CG CH CI CM CV CY CZ DE DJ DK DZ EE EG ER ES ET FI FR GA GB GE GH GM GN GQ GR GW HR HU IE IL IQ IR IS IT JO KE KG KM KW KZ LB LI LR LS LT LU LV LY MA MC MD ME MG MK ML MR MT MU MW MZ NA NE NG NL NO OM PL PS PT QA RO RS RU RW SA SC SD SE SI SK SL SM SN SO SS ST SY SZ TD TG TJ TM TN TR TZ UA UG UZ VA YE ZA ZM ZW",
+    LATAM: "AR BO BR BZ CL CO CR CU DO EC GT GY HN HT MX NI PA PE PR PY SR SV UY VE",
+  };
+  return (` ${groups[region] ?? ""} `).includes(` ${code} `);
+}
+
+function escapeRegExpUnlessPattern(value: string): string {
+  return value.includes("\\") ? value : escapeRegExp(value);
+}
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}