openings 0.1.11 → 0.1.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.11",
+  "version": "0.1.13",
   "description": "A free, candidate-safe job-search substrate for AI agents",
   "license": "MIT",
   "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },

package/src/career-tracing.ts

@@ -3,7 +3,7 @@ import { atomicJson } from "./atomic-file.ts";
 import { assertCompatibleReport, stampReport, type ReportMeta } from "./report-meta.ts";
 import { mergeSourceCandidates } from "./source-discovery.ts";
 import { resolveSource } from "./source-verification.ts";
-import { fetchSafeHead, type HeadTransport, type ResolveHost } from "./safe-head.ts";
+import { extractLinks, fetchSafeHead, fetchSafePage, robotsAllows, type HeadTransport, type PageTransport, type ResolveHost } from "./safe-head.ts";
 import type { SourceCandidate } from "./types.ts";
 import { deriveLeadState, mergeEnrichmentLeads, type EnrichmentLead, type IdentityEvidence } from "./enrichment-registry.ts";
 
@@ -11,7 +11,7 @@ type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
 
 interface CompanySeed { companyName: string; companyDomain: string; careerUrl?: string }
 interface TraceIssue { companyName?: string; companyDomain?: string; careerUrls?: string[]; reason: string; detail: string }
-interface TraceOptions { country?: string; fetch?: Fetch; concurrency?: number; timeoutMs?: number; searchKey?: string; commonCrawlReportPath?: string; resolveHost?: ResolveHost; headTransport?: HeadTransport; registryPath?: string }
+interface TraceOptions { country?: string; fetch?: Fetch; concurrency?: number; timeoutMs?: number; searchKey?: string; commonCrawlReportPath?: string; resolveHost?: ResolveHost; headTransport?: HeadTransport; pageTransport?: PageTransport; pageLinks?: boolean; registryPath?: string }
 
 export interface CareerTraceReport extends ReportMeta {
   companiesChecked: number;
@@ -64,6 +64,8 @@ export async function traceCareerSources(inputPath: string, candidatesPath: stri
           failures.push({ index, issue: { ...identity(seed), reason: "search_failed", detail: error instanceof Error ? error.message : String(error) } });
         }
       }
+      let evidenceKind: "company_redirect" | "company_page_link" = "company_redirect";
+      const reachablePages: string[] = [];
       if (!found) {
         for (const careerUrl of careerUrls) {
           try {
@@ -71,23 +73,39 @@ export async function traceCareerSources(inputPath: string, candidatesPath: stri
             if (!result.response.ok) continue;
             const source = resolveSource(result.finalUrl);
             if (source) { found = source; reference = careerUrl; break; }
+            reachablePages.push(result.finalUrl);
           } catch (error) {
             failures.push({ index, issue: { ...identity(seed), careerUrls: [careerUrl], reason: "career_request_failed", detail: error instanceof Error ? error.message : String(error) } });
           }
         }
       }
+      // No redirect: read the company's own careers page once, within robots rules, for a link to a supported board. Only hrefs are inspected.
+      if (!found && options.pageLinks !== false) {
+        for (const pageUrl of reachablePages) {
+          try {
+            if (!(await robotsAllows(pageUrl, { resolveHost: options.resolveHost, transport: options.pageTransport, timeoutMs }))) continue;
+            const page = await fetchSafePage(pageUrl, { resolveHost: options.resolveHost, transport: options.pageTransport, timeoutMs });
+            if (page.status !== 200) continue;
+            const boards = extractLinks(page.html, page.finalUrl).map((link) => resolveSource(link)).filter((source): source is NonNullable<typeof source> => Boolean(source));
+            const best = boards.find((source) => sourceMatchesCompany(source.token, seed)) ?? boards[0];
+            if (best) { found = best; reference = page.finalUrl; evidenceKind = "company_page_link"; break; }
+          } catch (error) {
+            failures.push({ index, issue: { ...identity(seed), careerUrls: [pageUrl], reason: "career_page_failed", detail: error instanceof Error ? error.message : String(error) } });
+          }
+        }
+      }
       if (!found) {
         const lead = commonCrawlLeads.find((item) => sourceMatchesCompany(item.source.token, seed));
         if (lead) { found = lead.source; reference = lead.reference; channel = "dataset"; }
       }
       if (!found) {
-        unresolved.push({ index, issue: { ...identity(seed), careerUrls, reason: "ats_not_resolved", detail: "Career URLs did not redirect to a supported structured job source" } });
+        unresolved.push({ index, issue: { ...identity(seed), careerUrls, reason: "ats_not_resolved", detail: "Career URLs neither redirected to nor linked a supported structured job source" } });
         continue;
       }
       const companyName = seed.companyName.trim();
       const companyDomain = normalizeDomain(seed.companyDomain);
       const companyMatch = { companyName, companyDomain, method: "normalized_token" as const, reference };
-      const identityEvidence: IdentityEvidence[] = channel === "career_page" ? [{ companyName, companyDomain, kind: "company_redirect", reference, observedAt: new Date().toISOString() }] : [];
+      const identityEvidence: IdentityEvidence[] = channel === "career_page" ? [{ companyName, companyDomain, kind: evidenceKind, reference, observedAt: new Date().toISOString() }] : [];
       const lead: EnrichmentLead = { sourceKey: `${found.ats}:${found.token.toLowerCase()}`, sourceUrl: found.canonicalSourceUrl, ats: found.ats, token: found.token,
         discoveredFrom: [{ channel, reference }], companyMatches: [companyMatch], identityEvidence, attempts: [] };
       registryRows.push(lead);
@@ -96,7 +114,7 @@ export async function traceCareerSources(inputPath: string, candidatesPath: stri
         companyName, companyDomain, sourceUrl: found.canonicalSourceUrl,
         cohorts: country ? [country] : undefined,
         discoveredFrom: { channel, reference },
-        domainEvidence: channel === "career_page" ? { kind: "company_redirect", reference } : undefined,
+        domainEvidence: channel === "career_page" ? { kind: evidenceKind, reference } : undefined,
       } });
     }
   }

package/src/catalog.ts

@@ -116,6 +116,8 @@ export function searchJobs(jobs: Job[], query: SearchQuery): JobSummary[] {
 export interface FetchJobsObserver {
   onBackoff?(event: { status: number; delayMs: number }): void;
   workdayPageDelayMs?: number;
+  /** Countries to fetch with Workday's country facet after a capped crawl, so roles beyond the 2,000-posting cap are not lost. */
+  workdayCountries?: string[];
   pacingNow?: () => number;
   pacingSleep?: (delayMs: number, signal?: AbortSignal) => Promise<void>;
 }
@@ -147,6 +149,11 @@ export async function fetchSourceJobs(company: Company, fetcher: Fetch = globalT
         : (body as { offers: RecruiteeJob[] }).offers.map((job) => normalizeRecruitee(company, job));
 }
 
+const WORKDAY_LISTING_CAP = 2000;
+/** Workday's country facet ids are the same on every tenant. Verified live on 2026-09-07. */
+const WORKDAY_COUNTRY_FACETS: Record<string, string> = { IN: "c4f78be1a8f14da0ab49ce1162348a5e", US: "bc33aa3152ec42d4995f4791a106ed09" };
+function workdayJobKey(job: WorkdayJob): string { return job.bulletFields?.[0] ?? job.externalPath; }
+
 async function fetchWorkdayJobs(company: Company, fetcher: Fetch, signal?: AbortSignal, observer?: FetchJobsObserver): Promise<Job[]> {
   const source = parseWorkdayToken(company.token);
   const endpoint = `https://${source.host}/wday/cxs/${encodeURIComponent(source.tenant)}/${encodeURIComponent(source.site)}/jobs`;
@@ -168,10 +175,10 @@ async function fetchWorkdayJobs(company: Company, fetcher: Fetch, signal?: Abort
       previousPageStart = pacingNow();
     } finally { release(); }
   }
-  async function page(offset: number): Promise<{ total: number; jobs: WorkdayJob[] }> {
+  async function page(offset: number, appliedFacets: Record<string, string[]> = {}): Promise<{ total: number; jobs: WorkdayJob[] }> {
     for (let attempt = 0; attempt < 3; attempt += 1) {
       await pacePageStart();
-      const response = await fetcher(endpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ appliedFacets: {}, limit, offset, searchText: "" }), signal });
+      const response = await fetcher(endpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ appliedFacets, limit, offset, searchText: "" }), signal });
       if (response.ok) {
         const body = await response.json() as { total?: unknown; jobPostings?: unknown };
         if (!Number.isInteger(body.total) || !Array.isArray(body.jobPostings)) throw new Error(`${company.name} Workday source returned an invalid payload`);
@@ -203,6 +210,18 @@ async function fetchWorkdayJobs(company: Company, fetcher: Fetch, signal?: Abort
   await Promise.all(Array.from({ length: workerCount }, worker));
   const jobs = [first.jobs, ...pages].flat().slice(0, first.total);
   if (jobs.length !== first.total) throw new Error(`${company.name} Workday source returned ${jobs.length} of ${first.total} jobs`);
+  // Workday stops an unfiltered listing at 2,000 postings. For capped tenants, a per-country pass sees past the cap.
+  if (first.total >= WORKDAY_LISTING_CAP) {
+    const seen = new Set(jobs.map((job) => workdayJobKey(job)));
+    for (const code of observer?.workdayCountries ?? []) {
+      const facet = WORKDAY_COUNTRY_FACETS[code.toUpperCase()];
+      if (!facet) continue;
+      const head = await page(0, { locationCountry: [facet] });
+      const extra = [head.jobs];
+      for (let offset = limit; offset < Math.min(head.total, WORKDAY_LISTING_CAP); offset += limit) extra.push((await page(offset, { locationCountry: [facet] })).jobs);
+      for (const job of extra.flat()) { const key = workdayJobKey(job); if (!seen.has(key)) { seen.add(key); jobs.push(job); } }
+    }
+  }
   return jobs.map((job) => normalizeWorkday(company, source, job));
 }
 

package/src/cli.ts

@@ -22,7 +22,7 @@ import { mergeAttemptedRoundLeads, prepareRecruiteeRoundArtifacts } from "./recr
 const HELP = `Openings — search public company job boards
 
 Usage:
-  openings crawl [--country CODE | --companies FILE] [--concurrency N] [--source-cache-hours N] [--source-limit N] [--delay-ms N] [--workday-page-delay-ms N] [--data-dir PATH]
+  openings crawl [--workday-countries IN,US] [--country CODE | --companies FILE] [--concurrency N] [--source-cache-hours N] [--source-limit N] [--delay-ms N] [--workday-page-delay-ms N] [--data-dir PATH]
   openings snapshot export [--input FILE] [--output-dir PATH]
   openings coverage report --country CODE [--snapshot FILE] [--catalog FILE] [--candidates FILE] [--registry FILE] [--output FILE] [--as-of ISO]
   openings sources discover-subdomains SEEDS.json [--output FILE] [--limit N] [--delay-ms N] [--report FILE]
@@ -69,7 +69,7 @@ export async function run(args: string[]): Promise<number> {
     if (typeof parsed === "string") return fail(parsed);
     const runtime = createRuntime({
       dataDir: parsed.dataDir, concurrency: parsed.concurrency, sourceCacheHours: parsed.sourceCacheHours,
-      sourceLimit: parsed.sourceLimit, crawlDelayMs: parsed.delayMs, workdayPageDelayMs: parsed.workdayPageDelayMs,
+      sourceLimit: parsed.sourceLimit, crawlDelayMs: parsed.delayMs, workdayPageDelayMs: parsed.workdayPageDelayMs, workdayCountries: parsed.workdayCountries,
     });
     const slugs = parsed.companiesFile ? await readCompanyFile(parsed.companiesFile) : undefined;
     console.log(JSON.stringify(await runtime.crawl({ country: parsed.country, slugs }), null, 2));
@@ -514,6 +514,7 @@ function parseCrawl(args: string[]) {
   let concurrency = 10;
   let delayMs = 0;
   let workdayPageDelayMs = 0;
+  let workdayCountries: string[] | undefined;
   let sourceCacheHours = 24;
   let sourceLimit = 0;
   for (let index = 0; index < args.length; index += 1) {
@@ -526,6 +527,10 @@ function parseCrawl(args: string[]) {
     else if (arg === "--concurrency") {
       concurrency = Number(args[++index]);
       if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 100) return "--concurrency must be an integer from 1 to 100";
+    } else if (arg === "--workday-countries") {
+      const value = args[++index] ?? "";
+      workdayCountries = value.split(",").map((code) => code.trim().toUpperCase()).filter(Boolean);
+      if (!workdayCountries.every((code) => /^[A-Z]{2}$/.test(code))) return "--workday-countries must be a comma-separated list of two-letter codes";
     } else if (arg === "--delay-ms") {
       delayMs = Number(args[++index]);
       if (!Number.isInteger(delayMs) || delayMs < 0 || delayMs > 60_000) return "--delay-ms must be an integer from 0 to 60000";
@@ -543,7 +548,7 @@ function parseCrawl(args: string[]) {
   if (country && companiesFile) return "Use either --country or --companies, not both";
   if (args.includes("--companies") && !companiesFile) return "--companies requires a file";
   if (args.includes("--data-dir") && !dataDir) return "--data-dir requires a value";
-  return { country, companiesFile, dataDir, concurrency, delayMs, workdayPageDelayMs, sourceCacheHours, sourceLimit };
+  return { country, companiesFile, dataDir, concurrency, delayMs, workdayPageDelayMs, sourceCacheHours, sourceLimit , workdayCountries };
 }
 
 function parseCountry(value: string | undefined): string | undefined {

package/src/crawler.ts

@@ -16,6 +16,7 @@ interface CrawlerOptions {
   sourceFreshnessMs?: number;
   sourceLimit?: number;
   workdayPageDelayMs?: number;
+  workdayCountries?: string[];
   pacingNow?: () => number;
   pacingSleep?: (delayMs: number) => Promise<void>;
   now?: () => Date;
@@ -94,6 +95,7 @@ export function createCrawler(options: CrawlerOptions): Crawler {
               const jobs = await options.fetchJobs(source, controller.signal, {
                 onBackoff: ({ status, delayMs }) => { metric.backoffMs += delayMs; if (status === 429) metric.throttles += 1; },
                 workdayPageDelayMs: options.workdayPageDelayMs,
+                workdayCountries: options.workdayCountries,
               });
               partitions[source.slug] = { fetchedAt: now().toISOString(), jobs };
               succeeded += 1;

package/src/enrichment-registry.ts

@@ -24,6 +24,7 @@ export interface EnrichmentRegistry { version: 1; updatedAt: string; leads: Enri
 const evidenceRank: Record<IdentityEvidence["kind"], number> = {
   provider_structured_domain: 4,
   company_redirect: 3,
+  company_page_link: 3,
   company_registry: 2,
   authoritative_dataset: 1,
 };
@@ -32,7 +33,7 @@ export function deriveLeadState(lead: EnrichmentLead): EnrichmentState {
   const latestAttempt = lead.attempts.at(-1);
   if (latestAttempt?.outcome === "permanent_failure" && !supersededFailure(lead, latestAttempt)) return "rejected";
   if (!lead.companyMatches.length) return lead.promotedAt && latestAttempt?.outcome === "success" ? "verified" : "unresolved";
-  const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect"].includes(evidence.kind));
+  const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect", "company_page_link"].includes(evidence.kind));
   if (!qualifying.length) return lead.promotedAt && latestAttempt?.outcome === "success" ? "verified" : "matched";
   const winningRank = Math.max(...qualifying.map((evidence) => evidenceRank[evidence.kind]));
   const winningDomains = new Set(qualifying.filter((evidence) => evidenceRank[evidence.kind] === winningRank).map((evidence) => normalizeDomain(evidence.companyDomain)));
@@ -125,7 +126,7 @@ function isRegistry(value: unknown): value is EnrichmentRegistry {
 function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }
 function validProvenance(value: unknown): boolean { return isRecord(value) && ["search", "career_page", "provider_directory", "community", "dataset", "legacy"].includes(String(value.channel)) && typeof value.reference === "string"; }
 function validMatch(value: unknown): boolean { return isRecord(value) && typeof value.companyName === "string" && typeof value.companyDomain === "string" && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(value.companyDomain) && ["normalized_token", "normalized_name", "normalized_domain", "search_result"].includes(String(value.method)) && typeof value.reference === "string"; }
-function validEvidence(value: unknown): boolean { return isRecord(value) && typeof value.companyName === "string" && typeof value.companyDomain === "string" && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(value.companyDomain) && ["provider_structured_domain", "company_redirect", "company_registry", "authoritative_dataset"].includes(String(value.kind)) && typeof value.reference === "string" && typeof value.observedAt === "string" && Number.isFinite(Date.parse(value.observedAt)); }
+function validEvidence(value: unknown): boolean { return isRecord(value) && typeof value.companyName === "string" && typeof value.companyDomain === "string" && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(value.companyDomain) && ["provider_structured_domain", "company_redirect", "company_page_link", "company_registry", "authoritative_dataset"].includes(String(value.kind)) && typeof value.reference === "string" && typeof value.observedAt === "string" && Number.isFinite(Date.parse(value.observedAt)); }
 function validAttempt(value: unknown): boolean { return isRecord(value) && typeof value.attemptedAt === "string" && Number.isFinite(Date.parse(value.attemptedAt)) && ["success", "transient_failure", "permanent_failure"].includes(String(value.outcome)) && (value.nextEligibleAt === undefined || typeof value.nextEligibleAt === "string" && Number.isFinite(Date.parse(value.nextEligibleAt))) && (value.category === undefined || typeof value.category === "string") && (value.detail === undefined || typeof value.detail === "string") && (value.evidenceRank === undefined || typeof value.evidenceRank === "number" && Number.isFinite(value.evidenceRank)); }
 function sourceIdentityMatches(value: Record<string, unknown>): boolean {
   const source = resolveSource(String(value.sourceUrl));

package/src/local-jobs.ts

@@ -14,6 +14,7 @@ interface LocalJobsOptions {
   sourceFreshnessMs?: number;
   sourceLimit?: number;
   workdayPageDelayMs?: number;
+  workdayCountries?: string[];
   now?: () => Date;
   onCrawled?(source: Company, partition: JobPartition): void | Promise<void>;
 }

package/src/providers.ts

@@ -204,8 +204,9 @@ export function countryLabel(value: string): string {
   try { const name = regionNames.of(code); return name && name !== code ? name : value.trim(); } catch { return value.trim(); }
 }
 
+const RESERVED_TOKENS = new Set(["my-applications", "sign-in", "signin", "login", "register", "privacy", "cookie-policy", "job-alerts", "search"]);
 function validToken(value: string | undefined): string | null {
-  return value && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) ? value : null;
+  return value && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) && !RESERVED_TOKENS.has(value.toLowerCase()) ? value : null;
 }
 
 export function plainText(value: string): string {

package/src/runtime.ts

@@ -13,7 +13,8 @@ import { createFileSnapshotStore } from "./snapshot-store.ts";
 import { createJobCoverageReader } from "./job-coverage.ts";
 import { createJobSearchPreparer } from "./job-search-preparation.ts";
 
-export function createRuntime(options: { dataDir?: string; concurrency?: number; crawlDelayMs?: number; workdayPageDelayMs?: number; sourceCacheHours?: number; sourceLimit?: number } = {}) {
+export function createRuntime(options: { dataDir?: string; concurrency?: number; crawlDelayMs?: number; workdayPageDelayMs?: number; workdayCountries?: string[]; sourceCacheHours?: number; sourceLimit?: number } = {}) {
+  const workdayCountries = options.workdayCountries ?? (process.env.OPENINGS_WORKDAY_COUNTRIES ?? "IN,US").split(",").map((code) => code.trim().toUpperCase()).filter((code) => /^[A-Z]{2}$/.test(code));
   const dataDir = options.dataDir ?? process.env.OPENINGS_DATA_DIR ?? join(process.cwd(), ".openings");
   const store = createFileSnapshotStore(join(dataDir, "snapshot.json"));
   const aggregatorUrl = resolveAggregatorUrl(process.env.OPENINGS_AGGREGATOR_URL);
@@ -28,6 +29,7 @@ export function createRuntime(options: { dataDir?: string; concurrency?: number;
     sourceFreshnessMs: (options.sourceCacheHours ?? 0) * 60 * 60 * 1000,
     sourceLimit: options.sourceLimit,
     workdayPageDelayMs: options.workdayPageDelayMs,
+    workdayCountries,
     onCrawled,
   });
   const recommender = createJobRecommender({ sources: companies, store, crawl: local.crawl });
@@ -41,6 +43,7 @@ export function createRuntime(options: { dataDir?: string; concurrency?: number;
     maxAttempts: 1,
     sourceStartDelayMs: options.crawlDelayMs,
     workdayPageDelayMs: options.workdayPageDelayMs,
+    workdayCountries,
     onCrawled,
   });
   const preparation = createJobSearchPreparer({ sources: companies, store, crawl: preparationLocal.crawl, seed: aggregatorUrl ? (countries) => fetchSeedSnapshot(aggregatorUrl, globalThis.fetch, 20_000, countries) : undefined });

package/src/safe-head.ts

@@ -6,6 +6,63 @@ import { connect as connectTls } from "node:tls";
 export type ResolveHost = (hostname: string) => Promise<string[]>;
 export type HeadTransport = (url: URL, address: string, signal: AbortSignal) => Promise<Response>;
 export interface SafeHeadResult { response: Response; finalUrl: string }
+export type PageTransport = (url: URL, address: string, signal: AbortSignal, maxBytes: number) => Promise<Response>;
+export interface SafePageResult { status: number; finalUrl: string; html: string }
+
+/**
+ * One bounded GET of a company-owned page, with the same DNS pinning and HTTPS-only redirect rules as fetchSafeHead.
+ * Reads at most `maxBytes` and only ever hands back the body for link extraction; nothing else is retained.
+ */
+export async function fetchSafePage(url: string, options: { resolveHost?: ResolveHost; transport?: PageTransport; timeoutMs?: number; maxBytes?: number } = {}): Promise<SafePageResult> {
+  const maxBytes = options.maxBytes ?? 1_000_000;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${options.timeoutMs ?? 15_000}ms`)), options.timeoutMs ?? 15_000);
+  try {
+    let current = new URL(url);
+    for (let redirects = 0; redirects <= 5; redirects += 1) {
+      const address = await withAbort(publicAddress(current, options.resolveHost ?? resolveAddresses), controller.signal);
+      const response = await withAbort((options.transport ?? pinnedGet)(current, address, controller.signal, maxBytes), controller.signal);
+      if ([301, 302, 303, 307, 308].includes(response.status) && response.headers.get("location")) {
+        current = new URL(response.headers.get("location")!, current);
+        if (current.protocol !== "https:") throw new Error("Career redirect must use HTTPS");
+        continue;
+      }
+      const html = (await response.text()).slice(0, maxBytes);
+      return { status: response.status, finalUrl: current.href, html };
+    }
+    throw new Error("Career redirect limit exceeded");
+  } finally { clearTimeout(timer); }
+}
+
+/** True unless robots.txt for the page's origin disallows the path for everyone or for this crawler. Fetch failures count as allowed, an absent file does. */
+export async function robotsAllows(pageUrl: string, options: { resolveHost?: ResolveHost; transport?: PageTransport; timeoutMs?: number } = {}): Promise<boolean> {
+  const url = new URL(pageUrl);
+  let text = "";
+  try {
+    const robots = await fetchSafePage(`${url.origin}/robots.txt`, { ...options, maxBytes: 64_000 });
+    if (robots.status !== 200) return true;
+    text = robots.html;
+  } catch { return true; }
+  let applies = false;
+  const disallowed: string[] = [];
+  for (const raw of text.split(/\r?\n/)) {
+    const line = raw.replace(/#.*$/, "").trim();
+    const agent = /^user-agent:\s*(.+)$/i.exec(line);
+    if (agent) { const name = agent[1]!.trim().toLowerCase(); applies = name === "*" || name.includes("openings"); continue; }
+    const rule = /^disallow:\s*(.*)$/i.exec(line);
+    if (rule && applies && rule[1]!.trim()) disallowed.push(rule[1]!.trim());
+  }
+  return !disallowed.some((prefix) => url.pathname.startsWith(prefix));
+}
+
+/** Absolute https links found in a page's anchors. Only hrefs are read; the page text is never inspected. */
+export function extractLinks(html: string, base: string): string[] {
+  const links = new Set<string>();
+  for (const match of html.matchAll(/href\s*=\s*["']([^"'#\s]+)["']/gi)) {
+    try { const url = new URL(match[1]!, base); if (url.protocol === "https:") links.add(url.href); } catch { /* not a url */ }
+  }
+  return [...links];
+}
 
 export async function fetchSafeHead(url: string, options: { resolveHost?: ResolveHost; transport?: HeadTransport; timeoutMs?: number } = {}): Promise<SafeHeadResult> {
   const controller = new AbortController();
@@ -66,6 +123,28 @@ function pinnedHead(url: URL, address: string, signal: AbortSignal): Promise<Res
   });
 }
 
+function pinnedGet(url: URL, address: string, signal: AbortSignal, maxBytes: number): Promise<Response> {
+  return new Promise((resolve, reject) => {
+    const req = request(url, {
+      method: "GET", signal, servername: url.hostname, agent: false, headers: { "user-agent": "openings-discovery/0.1 (+https://avagama.co/openings/)", accept: "text/html" },
+      createConnection: () => {
+        const socket = connectTls({ host: address, port: Number(url.port || 443), servername: url.hostname });
+        const destroy = () => socket.destroy(signal.reason instanceof Error ? signal.reason : undefined);
+        if (signal.aborted) destroy();
+        else signal.addEventListener("abort", destroy, { once: true });
+        return socket;
+      },
+    }, (response) => {
+      const chunks: Buffer[] = []; let size = 0;
+      response.on("data", (chunk: Buffer) => { if (size < maxBytes) { chunks.push(chunk); size += chunk.length; } if (size >= maxBytes) response.destroy(); });
+      const finish = () => resolve(new Response(Buffer.concat(chunks).subarray(0, maxBytes), { status: response.statusCode ?? 500, headers: response.headers as HeadersInit }));
+      response.once("end", finish); response.once("close", finish); response.once("error", finish);
+    });
+    req.once("error", reject);
+    req.end();
+  });
+}
+
 function isPublicAddress(address: string): boolean {
   if (isIP(address) === 4) {
     const [a, b] = address.split(".").map(Number);

package/src/source-discovery.ts

@@ -17,7 +17,7 @@ interface DiscoveryOptions {
   registryPath?: string;
 }
 
-interface FeedEntry { sourceUrl: string; companyName?: string; companyDomain?: string; reference?: string; channel?: DiscoveryChannel; domainEvidence?: "authoritative_dataset" | "company_registry" | "company_redirect" }
+interface FeedEntry { sourceUrl: string; companyName?: string; companyDomain?: string; reference?: string; channel?: DiscoveryChannel; domainEvidence?: "authoritative_dataset" | "company_registry" | "company_redirect" | "company_page_link" }
 interface DiscoveryIssue { sourceUrl: string; reason: string; detail: string; companyName?: string; reference?: string }
 
 export interface SourceDiscoveryReport extends ReportMeta {
@@ -106,7 +106,7 @@ async function discoverEntries(feed: FeedEntry[], candidatesPath: string, report
         rejected.push({ index, issue: issue(entry, "invalid_channel", `Unsupported discovery channel: ${entry.channel}`) });
         continue;
       }
-      if (entry.domainEvidence !== undefined && !["authoritative_dataset", "company_registry", "company_redirect"].includes(entry.domainEvidence)) {
+      if (entry.domainEvidence !== undefined && !["authoritative_dataset", "company_registry", "company_redirect", "company_page_link"].includes(entry.domainEvidence)) {
         rejected.push({ index, issue: issue(entry, "invalid_domain_evidence", `Unsupported domain evidence: ${entry.domainEvidence}`) });
         continue;
       }
@@ -114,8 +114,8 @@ async function discoverEntries(feed: FeedEntry[], candidatesPath: string, report
       const companyName = typeof entry.companyName === "string" ? entry.companyName.trim() : "";
       const reference = entry.reference?.trim() || feedReference;
       const match = entry.companyDomain && companyName ? [{ companyName, companyDomain: entry.companyDomain.toLowerCase(), method: "normalized_token" as const, reference }] : [];
-      const redirectTrusted = entry.domainEvidence === "company_redirect" && entry.companyDomain && referenceBelongsToDomain(reference, entry.companyDomain);
-      const datasetTrusted = trustDomainEvidence && entry.domainEvidence && entry.domainEvidence !== "company_redirect";
+      const redirectTrusted = (entry.domainEvidence === "company_redirect" || entry.domainEvidence === "company_page_link") && entry.companyDomain && referenceBelongsToDomain(reference, entry.companyDomain);
+      const datasetTrusted = trustDomainEvidence && entry.domainEvidence && entry.domainEvidence !== "company_redirect" && entry.domainEvidence !== "company_page_link";
       const evidence: IdentityEvidence[] = (redirectTrusted || datasetTrusted) && entry.companyDomain && companyName ? [{ companyName, companyDomain: entry.companyDomain.toLowerCase(), kind: entry.domainEvidence!, reference, observedAt: new Date().toISOString() }] : [];
       registryRows.push({ sourceKey: key, sourceUrl: source.canonicalSourceUrl, ats: source.ats, token: source.token,
         discoveredFrom: [{ channel: entry.channel ?? "dataset", reference }], companyMatches: match, identityEvidence: evidence, attempts: [] });

package/src/source-enrichment.ts

@@ -54,8 +54,8 @@ export async function enrichSourcesFromCompanies(registryPath: string, companies
 }
 
 function winningIdentity(lead: EnrichmentLead): IdentityEvidence | undefined {
-  const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect"].includes(evidence.kind));
-  const rank = { provider_structured_domain: 4, company_redirect: 3, company_registry: 2, authoritative_dataset: 1 } as const;
+  const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect", "company_page_link"].includes(evidence.kind));
+  const rank = { provider_structured_domain: 4, company_redirect: 3, company_page_link: 3, company_registry: 2, authoritative_dataset: 1 } as const;
   return [...qualifying].sort((left, right) => rank[right.kind] - rank[left.kind] || left.companyDomain.localeCompare(right.companyDomain))[0];
 }
 function countStates(leads: EnrichmentLead[]): Record<string, number> { const counts: Record<string, number> = {}; for (const lead of leads) { const state = deriveLeadState(lead); counts[state] = (counts[state] ?? 0) + 1; } return counts; }
@@ -84,7 +84,7 @@ function isVerifiedCatalogRecord(company: unknown): boolean {
     && verification.canonicalSourceUrl === source.canonicalSourceUrl
     && typeof verification.checkedAt === "string" && Number.isFinite(Date.parse(verification.checkedAt))
     && typeof verification.observedCompanyName === "string" && verification.observedCompanyName.length > 0
-    && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect"].includes(String(verification.identityEvidence))
+    && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect", "company_page_link"].includes(String(verification.identityEvidence))
     && typeof verification.contentType === "string" && typeof verification.payloadVersion === "string" && verification.payloadVersion.length > 0
     && Number.isInteger(verification.jobCount) && Number(verification.jobCount) > 0;
 }

package/src/source-pipeline.ts

@@ -195,6 +195,8 @@ async function recordRegistryOutcomes(registryPath: string, result: Awaited<Retu
       ? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "provider_structured_domain", reference: company.sourceUrl, observedAt: now.toISOString() }]
       : company.verification.identityEvidence === "company_redirect" && company.domainEvidence?.kind === "company_redirect"
         ? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "company_redirect", reference: company.domainEvidence.reference, observedAt: now.toISOString() }]
+      : company.verification.identityEvidence === "company_page_link" && company.domainEvidence?.kind === "company_page_link"
+        ? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "company_page_link", reference: company.domainEvidence.reference, observedAt: now.toISOString() }]
         : [];
     additions.push({ ...lead, identityEvidence: evidence, attempts: [attempt], promotedAt: now.toISOString() });
   }

package/src/source-verification.ts

@@ -1,5 +1,5 @@
 import type { Ats, RejectedSource, SourceCandidate, SourceRejectionReason, SourceVerificationResult, VerifiedCompany } from "./types.ts";
-import { fetchSafeHead, type HeadTransport, type ResolveHost } from "./safe-head.ts";
+import { extractLinks, fetchSafeHead, fetchSafePage, robotsAllows, type HeadTransport, type PageTransport, type ResolveHost } from "./safe-head.ts";
 import { abortableDelay, fetchSourceJobs, isTransientStatus, retryDelayMs } from "./catalog.ts";
 import { isEligibleForCountry } from "./locations.ts";
 import { providerSpec, resolveProviderSource } from "./providers.ts";
@@ -14,6 +14,7 @@ interface VerificationOptions {
   timeoutMs?: number;
   resolveHost?: ResolveHost;
   headTransport?: HeadTransport;
+  pageTransport?: PageTransport;
   requireCountry?: string;
   countryGateTimeoutMs?: number;
   providerConcurrency?: Partial<Record<Ats, number>>;
@@ -56,8 +57,9 @@ export async function verifyCandidates(candidates: SourceCandidate[], options: V
       const release = await providerLimits.get(source.ats)!.acquire();
       const providerFetch = providerCooldowns.get(source.ats)!.wrap(fetcher);
       try {
-        const evidence = await probe(candidate, source, providerFetch, timeoutMs, options.resolveHost, options.headTransport);
-        if (!identityMatches(candidate.companyName, candidate.companyDomain, evidence.observedCompanyName, source.token)) {
+        const evidence = await probe(candidate, source, providerFetch, timeoutMs, options.resolveHost, options.headTransport, options.pageTransport);
+        const replayed = evidence.identityEvidence === "company_redirect" || evidence.identityEvidence === "company_page_link";
+        if (!replayed && !identityMatches(candidate.companyName, candidate.companyDomain, evidence.observedCompanyName, source.token)) {
           rejected.push({ index, value: rejection(candidate, "identity_mismatch", `Expected ${candidate.companyName}; observed ${evidence.observedCompanyName}`) });
           continue;
         }
@@ -149,7 +151,7 @@ class ProviderCooldown {
   }
 }
 
-async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher: Fetch, timeoutMs: number, resolveHost?: ResolveHost, headTransport?: HeadTransport) {
+async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher: Fetch, timeoutMs: number, resolveHost?: ResolveHost, headTransport?: HeadTransport, pageTransport?: PageTransport) {
   const controller = new AbortController();
   const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${timeoutMs}ms`)), timeoutMs);
   const workday = source.ats === "workday" ? parseWorkdayToken(source.token) : undefined;
@@ -169,14 +171,16 @@ async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher
     if (!jobs) throw new VerificationError("invalid_payload", "Payload does not contain the expected jobs array");
     if (jobs.length === 0) throw new VerificationError("empty_board", "Source has no jobs, so identity cannot be verified");
     const providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
-    const namedProvider = source.ats === "greenhouse" || source.ats === "workday" || Boolean(spec && providerName);
+    const tenantMatches = source.ats === "workday" && identityMatches(candidate.companyName, candidate.companyDomain, workday!.tenant, source.token);
+    const namedProvider = source.ats === "greenhouse" || tenantMatches || Boolean(spec && providerName);
     const hasDomainLink = !namedProvider && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
     const hasRedirectEvidence = !namedProvider && await verifiedCompanyRedirect(candidate, source, resolveHost, headTransport, timeoutMs);
-    if (!namedProvider && !hasDomainLink && !hasRedirectEvidence) throw new VerificationError("identity_mismatch", `Neither structured identity fields nor a verified company redirect link to ${candidate.companyDomain}`);
-    const observedCompanyName = providerName || candidate.companyName;
+    const hasPageLink = !namedProvider && !hasDomainLink && !hasRedirectEvidence && await verifiedCompanyPageLink(candidate, source, resolveHost, pageTransport, timeoutMs);
+    if (!namedProvider && !hasDomainLink && !hasRedirectEvidence && !hasPageLink) throw new VerificationError("identity_mismatch", `Neither structured identity fields, a verified company redirect, nor a company careers-page link point to ${candidate.companyDomain}`);
+    const observedCompanyName = namedProvider && providerName ? providerName : candidate.companyName;
     return {
       observedCompanyName,
-      identityEvidence: source.ats === "workday" ? "provider_tenant" as const : (providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : "company_redirect" as const),
+      identityEvidence: tenantMatches ? "provider_tenant" as const : (namedProvider && providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : hasRedirectEvidence ? "company_redirect" as const : "company_page_link" as const),
       contentType,
       payloadVersion: spec ? spec.payloadVersion(body) : source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
       jobCount: source.ats === "workday" && isRecord(body) && typeof body.total === "number" ? body.total : jobs.length,
@@ -252,7 +256,7 @@ function validateCandidate(candidate: SourceCandidate): string | null {
   if (!isRecord(candidate.discoveredFrom) || typeof candidate.discoveredFrom.channel !== "string" || !channels.has(candidate.discoveredFrom.channel) || typeof candidate.discoveredFrom.reference !== "string" || !candidate.discoveredFrom.reference.trim()) return "discoveredFrom must contain a supported channel and reference";
   if (candidate.cohorts !== undefined && (!Array.isArray(candidate.cohorts) || !candidate.cohorts.every((code) => typeof code === "string"))) return "cohorts must be an array of country codes";
   if (candidate.domainEvidence !== undefined) {
-    if (!isRecord(candidate.domainEvidence) || !["authoritative_dataset", "company_registry", "company_redirect"].includes(String(candidate.domainEvidence.kind)) || typeof candidate.domainEvidence.reference !== "string" || !candidate.domainEvidence.reference) return "domainEvidence must contain a supported kind and reference";
+    if (!isRecord(candidate.domainEvidence) || !["authoritative_dataset", "company_registry", "company_redirect", "company_page_link"].includes(String(candidate.domainEvidence.kind)) || typeof candidate.domainEvidence.reference !== "string" || !candidate.domainEvidence.reference) return "domainEvidence must contain a supported kind and reference";
   }
   return null;
 }
@@ -300,4 +304,18 @@ async function verifiedCompanyRedirect(candidate: SourceCandidate, expected: Res
 }
 
 
+/** Replays company_page_link evidence: the company's own careers page, fetched once within robots rules, still links to the exact board. */
+async function verifiedCompanyPageLink(candidate: SourceCandidate, expected: ResolvedSource, resolveHost?: ResolveHost, pageTransport?: PageTransport, timeoutMs?: number): Promise<boolean> {
+  if (candidate.domainEvidence?.kind !== "company_page_link") return false;
+  try {
+    const host = new URL(candidate.domainEvidence.reference).hostname.toLowerCase().replace(/^www\./, "");
+    const domain = candidate.companyDomain.toLowerCase().replace(/^www\./, "");
+    if (host !== domain && !host.endsWith(`.${domain}`)) return false;
+    if (!(await robotsAllows(candidate.domainEvidence.reference, { resolveHost, transport: pageTransport, timeoutMs }))) return false;
+    const page = await fetchSafePage(candidate.domainEvidence.reference, { resolveHost, transport: pageTransport, timeoutMs });
+    if (page.status !== 200) return false;
+    return extractLinks(page.html, page.finalUrl).some((link) => { const observed = resolveSource(link); return observed?.ats === expected.ats && observed.token.toLowerCase() === expected.token.toLowerCase(); });
+  } catch { return false; }
+}
+
 class VerificationError extends Error { constructor(readonly reason: SourceRejectionReason, message: string) { super(message); } }

package/src/types.ts

@@ -2,7 +2,7 @@ export const ALL_PROVIDERS = ["greenhouse", "lever", "ashby", "workday", "recrui
 export type Ats = (typeof ALL_PROVIDERS)[number];
 
 export interface DomainEvidence {
-  kind: "authoritative_dataset" | "company_registry" | "company_redirect";
+  kind: "authoritative_dataset" | "company_registry" | "company_redirect" | "company_page_link";
   reference: string;
 }
 
@@ -40,7 +40,7 @@ export interface SourceVerification {
   checkedAt: string;
   canonicalSourceUrl: string;
   observedCompanyName: string;
-  identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect" | "provider_board";
+  identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect" | "company_page_link" | "provider_board";
   contentType: string;
   payloadVersion: string;
   jobCount: number;

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = "0.1.11";
+export const VERSION = "0.1.13";