openings 0.1.12 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "openings",
3
- "version": "0.1.12",
3
+ "version": "0.1.13",
4
4
  "description": "A free, candidate-safe job-search substrate for AI agents",
5
5
  "license": "MIT",
6
6
  "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },
@@ -3,7 +3,7 @@ import { atomicJson } from "./atomic-file.ts";
3
3
  import { assertCompatibleReport, stampReport, type ReportMeta } from "./report-meta.ts";
4
4
  import { mergeSourceCandidates } from "./source-discovery.ts";
5
5
  import { resolveSource } from "./source-verification.ts";
6
- import { fetchSafeHead, type HeadTransport, type ResolveHost } from "./safe-head.ts";
6
+ import { extractLinks, fetchSafeHead, fetchSafePage, robotsAllows, type HeadTransport, type PageTransport, type ResolveHost } from "./safe-head.ts";
7
7
  import type { SourceCandidate } from "./types.ts";
8
8
  import { deriveLeadState, mergeEnrichmentLeads, type EnrichmentLead, type IdentityEvidence } from "./enrichment-registry.ts";
9
9
 
@@ -11,7 +11,7 @@ type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
11
11
 
12
12
  interface CompanySeed { companyName: string; companyDomain: string; careerUrl?: string }
13
13
  interface TraceIssue { companyName?: string; companyDomain?: string; careerUrls?: string[]; reason: string; detail: string }
14
- interface TraceOptions { country?: string; fetch?: Fetch; concurrency?: number; timeoutMs?: number; searchKey?: string; commonCrawlReportPath?: string; resolveHost?: ResolveHost; headTransport?: HeadTransport; registryPath?: string }
14
+ interface TraceOptions { country?: string; fetch?: Fetch; concurrency?: number; timeoutMs?: number; searchKey?: string; commonCrawlReportPath?: string; resolveHost?: ResolveHost; headTransport?: HeadTransport; pageTransport?: PageTransport; pageLinks?: boolean; registryPath?: string }
15
15
 
16
16
  export interface CareerTraceReport extends ReportMeta {
17
17
  companiesChecked: number;
@@ -64,6 +64,8 @@ export async function traceCareerSources(inputPath: string, candidatesPath: stri
64
64
  failures.push({ index, issue: { ...identity(seed), reason: "search_failed", detail: error instanceof Error ? error.message : String(error) } });
65
65
  }
66
66
  }
67
+ let evidenceKind: "company_redirect" | "company_page_link" = "company_redirect";
68
+ const reachablePages: string[] = [];
67
69
  if (!found) {
68
70
  for (const careerUrl of careerUrls) {
69
71
  try {
@@ -71,23 +73,39 @@ export async function traceCareerSources(inputPath: string, candidatesPath: stri
71
73
  if (!result.response.ok) continue;
72
74
  const source = resolveSource(result.finalUrl);
73
75
  if (source) { found = source; reference = careerUrl; break; }
76
+ reachablePages.push(result.finalUrl);
74
77
  } catch (error) {
75
78
  failures.push({ index, issue: { ...identity(seed), careerUrls: [careerUrl], reason: "career_request_failed", detail: error instanceof Error ? error.message : String(error) } });
76
79
  }
77
80
  }
78
81
  }
82
+ // No redirect: read the company's own careers page once, within robots rules, for a link to a supported board. Only hrefs are inspected.
83
+ if (!found && options.pageLinks !== false) {
84
+ for (const pageUrl of reachablePages) {
85
+ try {
86
+ if (!(await robotsAllows(pageUrl, { resolveHost: options.resolveHost, transport: options.pageTransport, timeoutMs }))) continue;
87
+ const page = await fetchSafePage(pageUrl, { resolveHost: options.resolveHost, transport: options.pageTransport, timeoutMs });
88
+ if (page.status !== 200) continue;
89
+ const boards = extractLinks(page.html, page.finalUrl).map((link) => resolveSource(link)).filter((source): source is NonNullable<typeof source> => Boolean(source));
90
+ const best = boards.find((source) => sourceMatchesCompany(source.token, seed)) ?? boards[0];
91
+ if (best) { found = best; reference = page.finalUrl; evidenceKind = "company_page_link"; break; }
92
+ } catch (error) {
93
+ failures.push({ index, issue: { ...identity(seed), careerUrls: [pageUrl], reason: "career_page_failed", detail: error instanceof Error ? error.message : String(error) } });
94
+ }
95
+ }
96
+ }
79
97
  if (!found) {
80
98
  const lead = commonCrawlLeads.find((item) => sourceMatchesCompany(item.source.token, seed));
81
99
  if (lead) { found = lead.source; reference = lead.reference; channel = "dataset"; }
82
100
  }
83
101
  if (!found) {
84
- unresolved.push({ index, issue: { ...identity(seed), careerUrls, reason: "ats_not_resolved", detail: "Career URLs did not redirect to a supported structured job source" } });
102
+ unresolved.push({ index, issue: { ...identity(seed), careerUrls, reason: "ats_not_resolved", detail: "Career URLs neither redirected to nor linked a supported structured job source" } });
85
103
  continue;
86
104
  }
87
105
  const companyName = seed.companyName.trim();
88
106
  const companyDomain = normalizeDomain(seed.companyDomain);
89
107
  const companyMatch = { companyName, companyDomain, method: "normalized_token" as const, reference };
90
- const identityEvidence: IdentityEvidence[] = channel === "career_page" ? [{ companyName, companyDomain, kind: "company_redirect", reference, observedAt: new Date().toISOString() }] : [];
108
+ const identityEvidence: IdentityEvidence[] = channel === "career_page" ? [{ companyName, companyDomain, kind: evidenceKind, reference, observedAt: new Date().toISOString() }] : [];
91
109
  const lead: EnrichmentLead = { sourceKey: `${found.ats}:${found.token.toLowerCase()}`, sourceUrl: found.canonicalSourceUrl, ats: found.ats, token: found.token,
92
110
  discoveredFrom: [{ channel, reference }], companyMatches: [companyMatch], identityEvidence, attempts: [] };
93
111
  registryRows.push(lead);
@@ -96,7 +114,7 @@ export async function traceCareerSources(inputPath: string, candidatesPath: stri
96
114
  companyName, companyDomain, sourceUrl: found.canonicalSourceUrl,
97
115
  cohorts: country ? [country] : undefined,
98
116
  discoveredFrom: { channel, reference },
99
- domainEvidence: channel === "career_page" ? { kind: "company_redirect", reference } : undefined,
117
+ domainEvidence: channel === "career_page" ? { kind: evidenceKind, reference } : undefined,
100
118
  } });
101
119
  }
102
120
  }
@@ -24,6 +24,7 @@ export interface EnrichmentRegistry { version: 1; updatedAt: string; leads: Enri
24
24
  const evidenceRank: Record<IdentityEvidence["kind"], number> = {
25
25
  provider_structured_domain: 4,
26
26
  company_redirect: 3,
27
+ company_page_link: 3,
27
28
  company_registry: 2,
28
29
  authoritative_dataset: 1,
29
30
  };
@@ -32,7 +33,7 @@ export function deriveLeadState(lead: EnrichmentLead): EnrichmentState {
32
33
  const latestAttempt = lead.attempts.at(-1);
33
34
  if (latestAttempt?.outcome === "permanent_failure" && !supersededFailure(lead, latestAttempt)) return "rejected";
34
35
  if (!lead.companyMatches.length) return lead.promotedAt && latestAttempt?.outcome === "success" ? "verified" : "unresolved";
35
- const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect"].includes(evidence.kind));
36
+ const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect", "company_page_link"].includes(evidence.kind));
36
37
  if (!qualifying.length) return lead.promotedAt && latestAttempt?.outcome === "success" ? "verified" : "matched";
37
38
  const winningRank = Math.max(...qualifying.map((evidence) => evidenceRank[evidence.kind]));
38
39
  const winningDomains = new Set(qualifying.filter((evidence) => evidenceRank[evidence.kind] === winningRank).map((evidence) => normalizeDomain(evidence.companyDomain)));
@@ -125,7 +126,7 @@ function isRegistry(value: unknown): value is EnrichmentRegistry {
125
126
  function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }
126
127
  function validProvenance(value: unknown): boolean { return isRecord(value) && ["search", "career_page", "provider_directory", "community", "dataset", "legacy"].includes(String(value.channel)) && typeof value.reference === "string"; }
127
128
  function validMatch(value: unknown): boolean { return isRecord(value) && typeof value.companyName === "string" && typeof value.companyDomain === "string" && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(value.companyDomain) && ["normalized_token", "normalized_name", "normalized_domain", "search_result"].includes(String(value.method)) && typeof value.reference === "string"; }
128
- function validEvidence(value: unknown): boolean { return isRecord(value) && typeof value.companyName === "string" && typeof value.companyDomain === "string" && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(value.companyDomain) && ["provider_structured_domain", "company_redirect", "company_registry", "authoritative_dataset"].includes(String(value.kind)) && typeof value.reference === "string" && typeof value.observedAt === "string" && Number.isFinite(Date.parse(value.observedAt)); }
129
+ function validEvidence(value: unknown): boolean { return isRecord(value) && typeof value.companyName === "string" && typeof value.companyDomain === "string" && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(value.companyDomain) && ["provider_structured_domain", "company_redirect", "company_page_link", "company_registry", "authoritative_dataset"].includes(String(value.kind)) && typeof value.reference === "string" && typeof value.observedAt === "string" && Number.isFinite(Date.parse(value.observedAt)); }
129
130
  function validAttempt(value: unknown): boolean { return isRecord(value) && typeof value.attemptedAt === "string" && Number.isFinite(Date.parse(value.attemptedAt)) && ["success", "transient_failure", "permanent_failure"].includes(String(value.outcome)) && (value.nextEligibleAt === undefined || typeof value.nextEligibleAt === "string" && Number.isFinite(Date.parse(value.nextEligibleAt))) && (value.category === undefined || typeof value.category === "string") && (value.detail === undefined || typeof value.detail === "string") && (value.evidenceRank === undefined || typeof value.evidenceRank === "number" && Number.isFinite(value.evidenceRank)); }
130
131
  function sourceIdentityMatches(value: Record<string, unknown>): boolean {
131
132
  const source = resolveSource(String(value.sourceUrl));
package/src/providers.ts CHANGED
@@ -204,8 +204,9 @@ export function countryLabel(value: string): string {
204
204
  try { const name = regionNames.of(code); return name && name !== code ? name : value.trim(); } catch { return value.trim(); }
205
205
  }
206
206
 
207
+ const RESERVED_TOKENS = new Set(["my-applications", "sign-in", "signin", "login", "register", "privacy", "cookie-policy", "job-alerts", "search"]);
207
208
  function validToken(value: string | undefined): string | null {
208
- return value && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) ? value : null;
209
+ return value && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) && !RESERVED_TOKENS.has(value.toLowerCase()) ? value : null;
209
210
  }
210
211
 
211
212
  export function plainText(value: string): string {
package/src/safe-head.ts CHANGED
@@ -6,6 +6,63 @@ import { connect as connectTls } from "node:tls";
6
6
  export type ResolveHost = (hostname: string) => Promise<string[]>;
7
7
  export type HeadTransport = (url: URL, address: string, signal: AbortSignal) => Promise<Response>;
8
8
  export interface SafeHeadResult { response: Response; finalUrl: string }
9
+ export type PageTransport = (url: URL, address: string, signal: AbortSignal, maxBytes: number) => Promise<Response>;
10
+ export interface SafePageResult { status: number; finalUrl: string; html: string }
11
+
12
+ /**
13
+ * One bounded GET of a company-owned page, with the same DNS pinning and HTTPS-only redirect rules as fetchSafeHead.
14
+ * Reads at most `maxBytes` and only ever hands back the body for link extraction; nothing else is retained.
15
+ */
16
+ export async function fetchSafePage(url: string, options: { resolveHost?: ResolveHost; transport?: PageTransport; timeoutMs?: number; maxBytes?: number } = {}): Promise<SafePageResult> {
17
+ const maxBytes = options.maxBytes ?? 1_000_000;
18
+ const controller = new AbortController();
19
+ const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${options.timeoutMs ?? 15_000}ms`)), options.timeoutMs ?? 15_000);
20
+ try {
21
+ let current = new URL(url);
22
+ for (let redirects = 0; redirects <= 5; redirects += 1) {
23
+ const address = await withAbort(publicAddress(current, options.resolveHost ?? resolveAddresses), controller.signal);
24
+ const response = await withAbort((options.transport ?? pinnedGet)(current, address, controller.signal, maxBytes), controller.signal);
25
+ if ([301, 302, 303, 307, 308].includes(response.status) && response.headers.get("location")) {
26
+ current = new URL(response.headers.get("location")!, current);
27
+ if (current.protocol !== "https:") throw new Error("Career redirect must use HTTPS");
28
+ continue;
29
+ }
30
+ const html = (await response.text()).slice(0, maxBytes);
31
+ return { status: response.status, finalUrl: current.href, html };
32
+ }
33
+ throw new Error("Career redirect limit exceeded");
34
+ } finally { clearTimeout(timer); }
35
+ }
36
+
37
+ /** True unless robots.txt for the page's origin disallows the path for everyone or for this crawler. Fetch failures count as allowed, an absent file does. */
38
+ export async function robotsAllows(pageUrl: string, options: { resolveHost?: ResolveHost; transport?: PageTransport; timeoutMs?: number } = {}): Promise<boolean> {
39
+ const url = new URL(pageUrl);
40
+ let text = "";
41
+ try {
42
+ const robots = await fetchSafePage(`${url.origin}/robots.txt`, { ...options, maxBytes: 64_000 });
43
+ if (robots.status !== 200) return true;
44
+ text = robots.html;
45
+ } catch { return true; }
46
+ let applies = false;
47
+ const disallowed: string[] = [];
48
+ for (const raw of text.split(/\r?\n/)) {
49
+ const line = raw.replace(/#.*$/, "").trim();
50
+ const agent = /^user-agent:\s*(.+)$/i.exec(line);
51
+ if (agent) { const name = agent[1]!.trim().toLowerCase(); applies = name === "*" || name.includes("openings"); continue; }
52
+ const rule = /^disallow:\s*(.*)$/i.exec(line);
53
+ if (rule && applies && rule[1]!.trim()) disallowed.push(rule[1]!.trim());
54
+ }
55
+ return !disallowed.some((prefix) => url.pathname.startsWith(prefix));
56
+ }
57
+
58
+ /** Absolute https links found in a page's anchors. Only hrefs are read; the page text is never inspected. */
59
+ export function extractLinks(html: string, base: string): string[] {
60
+ const links = new Set<string>();
61
+ for (const match of html.matchAll(/href\s*=\s*["']([^"'#\s]+)["']/gi)) {
62
+ try { const url = new URL(match[1]!, base); if (url.protocol === "https:") links.add(url.href); } catch { /* not a url */ }
63
+ }
64
+ return [...links];
65
+ }
9
66
 
10
67
  export async function fetchSafeHead(url: string, options: { resolveHost?: ResolveHost; transport?: HeadTransport; timeoutMs?: number } = {}): Promise<SafeHeadResult> {
11
68
  const controller = new AbortController();
@@ -66,6 +123,28 @@ function pinnedHead(url: URL, address: string, signal: AbortSignal): Promise<Res
66
123
  });
67
124
  }
68
125
 
126
+ function pinnedGet(url: URL, address: string, signal: AbortSignal, maxBytes: number): Promise<Response> {
127
+ return new Promise((resolve, reject) => {
128
+ const req = request(url, {
129
+ method: "GET", signal, servername: url.hostname, agent: false, headers: { "user-agent": "openings-discovery/0.1 (+https://avagama.co/openings/)", accept: "text/html" },
130
+ createConnection: () => {
131
+ const socket = connectTls({ host: address, port: Number(url.port || 443), servername: url.hostname });
132
+ const destroy = () => socket.destroy(signal.reason instanceof Error ? signal.reason : undefined);
133
+ if (signal.aborted) destroy();
134
+ else signal.addEventListener("abort", destroy, { once: true });
135
+ return socket;
136
+ },
137
+ }, (response) => {
138
+ const chunks: Buffer[] = []; let size = 0;
139
+ response.on("data", (chunk: Buffer) => { if (size < maxBytes) { chunks.push(chunk); size += chunk.length; } if (size >= maxBytes) response.destroy(); });
140
+ const finish = () => resolve(new Response(Buffer.concat(chunks).subarray(0, maxBytes), { status: response.statusCode ?? 500, headers: response.headers as HeadersInit }));
141
+ response.once("end", finish); response.once("close", finish); response.once("error", finish);
142
+ });
143
+ req.once("error", reject);
144
+ req.end();
145
+ });
146
+ }
147
+
69
148
  function isPublicAddress(address: string): boolean {
70
149
  if (isIP(address) === 4) {
71
150
  const [a, b] = address.split(".").map(Number);
@@ -17,7 +17,7 @@ interface DiscoveryOptions {
17
17
  registryPath?: string;
18
18
  }
19
19
 
20
- interface FeedEntry { sourceUrl: string; companyName?: string; companyDomain?: string; reference?: string; channel?: DiscoveryChannel; domainEvidence?: "authoritative_dataset" | "company_registry" | "company_redirect" }
20
+ interface FeedEntry { sourceUrl: string; companyName?: string; companyDomain?: string; reference?: string; channel?: DiscoveryChannel; domainEvidence?: "authoritative_dataset" | "company_registry" | "company_redirect" | "company_page_link" }
21
21
  interface DiscoveryIssue { sourceUrl: string; reason: string; detail: string; companyName?: string; reference?: string }
22
22
 
23
23
  export interface SourceDiscoveryReport extends ReportMeta {
@@ -106,7 +106,7 @@ async function discoverEntries(feed: FeedEntry[], candidatesPath: string, report
106
106
  rejected.push({ index, issue: issue(entry, "invalid_channel", `Unsupported discovery channel: ${entry.channel}`) });
107
107
  continue;
108
108
  }
109
- if (entry.domainEvidence !== undefined && !["authoritative_dataset", "company_registry", "company_redirect"].includes(entry.domainEvidence)) {
109
+ if (entry.domainEvidence !== undefined && !["authoritative_dataset", "company_registry", "company_redirect", "company_page_link"].includes(entry.domainEvidence)) {
110
110
  rejected.push({ index, issue: issue(entry, "invalid_domain_evidence", `Unsupported domain evidence: ${entry.domainEvidence}`) });
111
111
  continue;
112
112
  }
@@ -114,8 +114,8 @@ async function discoverEntries(feed: FeedEntry[], candidatesPath: string, report
114
114
  const companyName = typeof entry.companyName === "string" ? entry.companyName.trim() : "";
115
115
  const reference = entry.reference?.trim() || feedReference;
116
116
  const match = entry.companyDomain && companyName ? [{ companyName, companyDomain: entry.companyDomain.toLowerCase(), method: "normalized_token" as const, reference }] : [];
117
- const redirectTrusted = entry.domainEvidence === "company_redirect" && entry.companyDomain && referenceBelongsToDomain(reference, entry.companyDomain);
118
- const datasetTrusted = trustDomainEvidence && entry.domainEvidence && entry.domainEvidence !== "company_redirect";
117
+ const redirectTrusted = (entry.domainEvidence === "company_redirect" || entry.domainEvidence === "company_page_link") && entry.companyDomain && referenceBelongsToDomain(reference, entry.companyDomain);
118
+ const datasetTrusted = trustDomainEvidence && entry.domainEvidence && entry.domainEvidence !== "company_redirect" && entry.domainEvidence !== "company_page_link";
119
119
  const evidence: IdentityEvidence[] = (redirectTrusted || datasetTrusted) && entry.companyDomain && companyName ? [{ companyName, companyDomain: entry.companyDomain.toLowerCase(), kind: entry.domainEvidence!, reference, observedAt: new Date().toISOString() }] : [];
120
120
  registryRows.push({ sourceKey: key, sourceUrl: source.canonicalSourceUrl, ats: source.ats, token: source.token,
121
121
  discoveredFrom: [{ channel: entry.channel ?? "dataset", reference }], companyMatches: match, identityEvidence: evidence, attempts: [] });
@@ -54,8 +54,8 @@ export async function enrichSourcesFromCompanies(registryPath: string, companies
54
54
  }
55
55
 
56
56
  function winningIdentity(lead: EnrichmentLead): IdentityEvidence | undefined {
57
- const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect"].includes(evidence.kind));
58
- const rank = { provider_structured_domain: 4, company_redirect: 3, company_registry: 2, authoritative_dataset: 1 } as const;
57
+ const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect", "company_page_link"].includes(evidence.kind));
58
+ const rank = { provider_structured_domain: 4, company_redirect: 3, company_page_link: 3, company_registry: 2, authoritative_dataset: 1 } as const;
59
59
  return [...qualifying].sort((left, right) => rank[right.kind] - rank[left.kind] || left.companyDomain.localeCompare(right.companyDomain))[0];
60
60
  }
61
61
  function countStates(leads: EnrichmentLead[]): Record<string, number> { const counts: Record<string, number> = {}; for (const lead of leads) { const state = deriveLeadState(lead); counts[state] = (counts[state] ?? 0) + 1; } return counts; }
@@ -84,7 +84,7 @@ function isVerifiedCatalogRecord(company: unknown): boolean {
84
84
  && verification.canonicalSourceUrl === source.canonicalSourceUrl
85
85
  && typeof verification.checkedAt === "string" && Number.isFinite(Date.parse(verification.checkedAt))
86
86
  && typeof verification.observedCompanyName === "string" && verification.observedCompanyName.length > 0
87
- && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect"].includes(String(verification.identityEvidence))
87
+ && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect", "company_page_link"].includes(String(verification.identityEvidence))
88
88
  && typeof verification.contentType === "string" && typeof verification.payloadVersion === "string" && verification.payloadVersion.length > 0
89
89
  && Number.isInteger(verification.jobCount) && Number(verification.jobCount) > 0;
90
90
  }
@@ -195,6 +195,8 @@ async function recordRegistryOutcomes(registryPath: string, result: Awaited<Retu
195
195
  ? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "provider_structured_domain", reference: company.sourceUrl, observedAt: now.toISOString() }]
196
196
  : company.verification.identityEvidence === "company_redirect" && company.domainEvidence?.kind === "company_redirect"
197
197
  ? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "company_redirect", reference: company.domainEvidence.reference, observedAt: now.toISOString() }]
198
+ : company.verification.identityEvidence === "company_page_link" && company.domainEvidence?.kind === "company_page_link"
199
+ ? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "company_page_link", reference: company.domainEvidence.reference, observedAt: now.toISOString() }]
198
200
  : [];
199
201
  additions.push({ ...lead, identityEvidence: evidence, attempts: [attempt], promotedAt: now.toISOString() });
200
202
  }
@@ -1,5 +1,5 @@
1
1
  import type { Ats, RejectedSource, SourceCandidate, SourceRejectionReason, SourceVerificationResult, VerifiedCompany } from "./types.ts";
2
- import { fetchSafeHead, type HeadTransport, type ResolveHost } from "./safe-head.ts";
2
+ import { extractLinks, fetchSafeHead, fetchSafePage, robotsAllows, type HeadTransport, type PageTransport, type ResolveHost } from "./safe-head.ts";
3
3
  import { abortableDelay, fetchSourceJobs, isTransientStatus, retryDelayMs } from "./catalog.ts";
4
4
  import { isEligibleForCountry } from "./locations.ts";
5
5
  import { providerSpec, resolveProviderSource } from "./providers.ts";
@@ -14,6 +14,7 @@ interface VerificationOptions {
14
14
  timeoutMs?: number;
15
15
  resolveHost?: ResolveHost;
16
16
  headTransport?: HeadTransport;
17
+ pageTransport?: PageTransport;
17
18
  requireCountry?: string;
18
19
  countryGateTimeoutMs?: number;
19
20
  providerConcurrency?: Partial<Record<Ats, number>>;
@@ -56,8 +57,9 @@ export async function verifyCandidates(candidates: SourceCandidate[], options: V
56
57
  const release = await providerLimits.get(source.ats)!.acquire();
57
58
  const providerFetch = providerCooldowns.get(source.ats)!.wrap(fetcher);
58
59
  try {
59
- const evidence = await probe(candidate, source, providerFetch, timeoutMs, options.resolveHost, options.headTransport);
60
- if (!identityMatches(candidate.companyName, candidate.companyDomain, evidence.observedCompanyName, source.token)) {
60
+ const evidence = await probe(candidate, source, providerFetch, timeoutMs, options.resolveHost, options.headTransport, options.pageTransport);
61
+ const replayed = evidence.identityEvidence === "company_redirect" || evidence.identityEvidence === "company_page_link";
62
+ if (!replayed && !identityMatches(candidate.companyName, candidate.companyDomain, evidence.observedCompanyName, source.token)) {
61
63
  rejected.push({ index, value: rejection(candidate, "identity_mismatch", `Expected ${candidate.companyName}; observed ${evidence.observedCompanyName}`) });
62
64
  continue;
63
65
  }
@@ -149,7 +151,7 @@ class ProviderCooldown {
149
151
  }
150
152
  }
151
153
 
152
- async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher: Fetch, timeoutMs: number, resolveHost?: ResolveHost, headTransport?: HeadTransport) {
154
+ async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher: Fetch, timeoutMs: number, resolveHost?: ResolveHost, headTransport?: HeadTransport, pageTransport?: PageTransport) {
153
155
  const controller = new AbortController();
154
156
  const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${timeoutMs}ms`)), timeoutMs);
155
157
  const workday = source.ats === "workday" ? parseWorkdayToken(source.token) : undefined;
@@ -169,14 +171,16 @@ async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher
169
171
  if (!jobs) throw new VerificationError("invalid_payload", "Payload does not contain the expected jobs array");
170
172
  if (jobs.length === 0) throw new VerificationError("empty_board", "Source has no jobs, so identity cannot be verified");
171
173
  const providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
172
- const namedProvider = source.ats === "greenhouse" || source.ats === "workday" || Boolean(spec && providerName);
174
+ const tenantMatches = source.ats === "workday" && identityMatches(candidate.companyName, candidate.companyDomain, workday!.tenant, source.token);
175
+ const namedProvider = source.ats === "greenhouse" || tenantMatches || Boolean(spec && providerName);
173
176
  const hasDomainLink = !namedProvider && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
174
177
  const hasRedirectEvidence = !namedProvider && await verifiedCompanyRedirect(candidate, source, resolveHost, headTransport, timeoutMs);
175
- if (!namedProvider && !hasDomainLink && !hasRedirectEvidence) throw new VerificationError("identity_mismatch", `Neither structured identity fields nor a verified company redirect link to ${candidate.companyDomain}`);
176
- const observedCompanyName = providerName || candidate.companyName;
178
+ const hasPageLink = !namedProvider && !hasDomainLink && !hasRedirectEvidence && await verifiedCompanyPageLink(candidate, source, resolveHost, pageTransport, timeoutMs);
179
+ if (!namedProvider && !hasDomainLink && !hasRedirectEvidence && !hasPageLink) throw new VerificationError("identity_mismatch", `Neither structured identity fields, a verified company redirect, nor a company careers-page link point to ${candidate.companyDomain}`);
180
+ const observedCompanyName = namedProvider && providerName ? providerName : candidate.companyName;
177
181
  return {
178
182
  observedCompanyName,
179
- identityEvidence: source.ats === "workday" ? "provider_tenant" as const : (providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : "company_redirect" as const),
183
+ identityEvidence: tenantMatches ? "provider_tenant" as const : (namedProvider && providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : hasRedirectEvidence ? "company_redirect" as const : "company_page_link" as const),
180
184
  contentType,
181
185
  payloadVersion: spec ? spec.payloadVersion(body) : source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
182
186
  jobCount: source.ats === "workday" && isRecord(body) && typeof body.total === "number" ? body.total : jobs.length,
@@ -252,7 +256,7 @@ function validateCandidate(candidate: SourceCandidate): string | null {
252
256
  if (!isRecord(candidate.discoveredFrom) || typeof candidate.discoveredFrom.channel !== "string" || !channels.has(candidate.discoveredFrom.channel) || typeof candidate.discoveredFrom.reference !== "string" || !candidate.discoveredFrom.reference.trim()) return "discoveredFrom must contain a supported channel and reference";
253
257
  if (candidate.cohorts !== undefined && (!Array.isArray(candidate.cohorts) || !candidate.cohorts.every((code) => typeof code === "string"))) return "cohorts must be an array of country codes";
254
258
  if (candidate.domainEvidence !== undefined) {
255
- if (!isRecord(candidate.domainEvidence) || !["authoritative_dataset", "company_registry", "company_redirect"].includes(String(candidate.domainEvidence.kind)) || typeof candidate.domainEvidence.reference !== "string" || !candidate.domainEvidence.reference) return "domainEvidence must contain a supported kind and reference";
259
+ if (!isRecord(candidate.domainEvidence) || !["authoritative_dataset", "company_registry", "company_redirect", "company_page_link"].includes(String(candidate.domainEvidence.kind)) || typeof candidate.domainEvidence.reference !== "string" || !candidate.domainEvidence.reference) return "domainEvidence must contain a supported kind and reference";
256
260
  }
257
261
  return null;
258
262
  }
@@ -300,4 +304,18 @@ async function verifiedCompanyRedirect(candidate: SourceCandidate, expected: Res
300
304
  }
301
305
 
302
306
 
307
+ /** Replays company_page_link evidence: the company's own careers page, fetched once within robots rules, still links to the exact board. */
308
+ async function verifiedCompanyPageLink(candidate: SourceCandidate, expected: ResolvedSource, resolveHost?: ResolveHost, pageTransport?: PageTransport, timeoutMs?: number): Promise<boolean> {
309
+ if (candidate.domainEvidence?.kind !== "company_page_link") return false;
310
+ try {
311
+ const host = new URL(candidate.domainEvidence.reference).hostname.toLowerCase().replace(/^www\./, "");
312
+ const domain = candidate.companyDomain.toLowerCase().replace(/^www\./, "");
313
+ if (host !== domain && !host.endsWith(`.${domain}`)) return false;
314
+ if (!(await robotsAllows(candidate.domainEvidence.reference, { resolveHost, transport: pageTransport, timeoutMs }))) return false;
315
+ const page = await fetchSafePage(candidate.domainEvidence.reference, { resolveHost, transport: pageTransport, timeoutMs });
316
+ if (page.status !== 200) return false;
317
+ return extractLinks(page.html, page.finalUrl).some((link) => { const observed = resolveSource(link); return observed?.ats === expected.ats && observed.token.toLowerCase() === expected.token.toLowerCase(); });
318
+ } catch { return false; }
319
+ }
320
+
303
321
  class VerificationError extends Error { constructor(readonly reason: SourceRejectionReason, message: string) { super(message); } }
package/src/types.ts CHANGED
@@ -2,7 +2,7 @@ export const ALL_PROVIDERS = ["greenhouse", "lever", "ashby", "workday", "recrui
2
2
  export type Ats = (typeof ALL_PROVIDERS)[number];
3
3
 
4
4
  export interface DomainEvidence {
5
- kind: "authoritative_dataset" | "company_registry" | "company_redirect";
5
+ kind: "authoritative_dataset" | "company_registry" | "company_redirect" | "company_page_link";
6
6
  reference: string;
7
7
  }
8
8
 
@@ -40,7 +40,7 @@ export interface SourceVerification {
40
40
  checkedAt: string;
41
41
  canonicalSourceUrl: string;
42
42
  observedCompanyName: string;
43
- identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect" | "provider_board";
43
+ identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect" | "company_page_link" | "provider_board";
44
44
  contentType: string;
45
45
  payloadVersion: string;
46
46
  jobCount: number;
package/src/version.ts CHANGED
@@ -1 +1 @@
1
- export const VERSION = "0.1.12";
1
+ export const VERSION = "0.1.13";