openings 0.1.12 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +1 -1
- package/data/companies.json +258 -231
- package/package.json +1 -1
- package/src/career-tracing.ts +23 -5
- package/src/enrichment-registry.ts +3 -2
- package/src/providers.ts +2 -1
- package/src/safe-head.ts +79 -0
- package/src/source-discovery.ts +4 -4
- package/src/source-enrichment.ts +3 -3
- package/src/source-pipeline.ts +2 -0
- package/src/source-verification.ts +27 -9
- package/src/types.ts +2 -2
- package/src/version.ts +1 -1
package/package.json
CHANGED
package/src/career-tracing.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { atomicJson } from "./atomic-file.ts";
|
|
|
3
3
|
import { assertCompatibleReport, stampReport, type ReportMeta } from "./report-meta.ts";
|
|
4
4
|
import { mergeSourceCandidates } from "./source-discovery.ts";
|
|
5
5
|
import { resolveSource } from "./source-verification.ts";
|
|
6
|
-
import { fetchSafeHead, type HeadTransport, type ResolveHost } from "./safe-head.ts";
|
|
6
|
+
import { extractLinks, fetchSafeHead, fetchSafePage, robotsAllows, type HeadTransport, type PageTransport, type ResolveHost } from "./safe-head.ts";
|
|
7
7
|
import type { SourceCandidate } from "./types.ts";
|
|
8
8
|
import { deriveLeadState, mergeEnrichmentLeads, type EnrichmentLead, type IdentityEvidence } from "./enrichment-registry.ts";
|
|
9
9
|
|
|
@@ -11,7 +11,7 @@ type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
|
|
|
11
11
|
|
|
12
12
|
interface CompanySeed { companyName: string; companyDomain: string; careerUrl?: string }
|
|
13
13
|
interface TraceIssue { companyName?: string; companyDomain?: string; careerUrls?: string[]; reason: string; detail: string }
|
|
14
|
-
interface TraceOptions { country?: string; fetch?: Fetch; concurrency?: number; timeoutMs?: number; searchKey?: string; commonCrawlReportPath?: string; resolveHost?: ResolveHost; headTransport?: HeadTransport; registryPath?: string }
|
|
14
|
+
interface TraceOptions { country?: string; fetch?: Fetch; concurrency?: number; timeoutMs?: number; searchKey?: string; commonCrawlReportPath?: string; resolveHost?: ResolveHost; headTransport?: HeadTransport; pageTransport?: PageTransport; pageLinks?: boolean; registryPath?: string }
|
|
15
15
|
|
|
16
16
|
export interface CareerTraceReport extends ReportMeta {
|
|
17
17
|
companiesChecked: number;
|
|
@@ -64,6 +64,8 @@ export async function traceCareerSources(inputPath: string, candidatesPath: stri
|
|
|
64
64
|
failures.push({ index, issue: { ...identity(seed), reason: "search_failed", detail: error instanceof Error ? error.message : String(error) } });
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
|
+
let evidenceKind: "company_redirect" | "company_page_link" = "company_redirect";
|
|
68
|
+
const reachablePages: string[] = [];
|
|
67
69
|
if (!found) {
|
|
68
70
|
for (const careerUrl of careerUrls) {
|
|
69
71
|
try {
|
|
@@ -71,23 +73,39 @@ export async function traceCareerSources(inputPath: string, candidatesPath: stri
|
|
|
71
73
|
if (!result.response.ok) continue;
|
|
72
74
|
const source = resolveSource(result.finalUrl);
|
|
73
75
|
if (source) { found = source; reference = careerUrl; break; }
|
|
76
|
+
reachablePages.push(result.finalUrl);
|
|
74
77
|
} catch (error) {
|
|
75
78
|
failures.push({ index, issue: { ...identity(seed), careerUrls: [careerUrl], reason: "career_request_failed", detail: error instanceof Error ? error.message : String(error) } });
|
|
76
79
|
}
|
|
77
80
|
}
|
|
78
81
|
}
|
|
82
|
+
// No redirect: read the company's own careers page once, within robots rules, for a link to a supported board. Only hrefs are inspected.
|
|
83
|
+
if (!found && options.pageLinks !== false) {
|
|
84
|
+
for (const pageUrl of reachablePages) {
|
|
85
|
+
try {
|
|
86
|
+
if (!(await robotsAllows(pageUrl, { resolveHost: options.resolveHost, transport: options.pageTransport, timeoutMs }))) continue;
|
|
87
|
+
const page = await fetchSafePage(pageUrl, { resolveHost: options.resolveHost, transport: options.pageTransport, timeoutMs });
|
|
88
|
+
if (page.status !== 200) continue;
|
|
89
|
+
const boards = extractLinks(page.html, page.finalUrl).map((link) => resolveSource(link)).filter((source): source is NonNullable<typeof source> => Boolean(source));
|
|
90
|
+
const best = boards.find((source) => sourceMatchesCompany(source.token, seed)) ?? boards[0];
|
|
91
|
+
if (best) { found = best; reference = page.finalUrl; evidenceKind = "company_page_link"; break; }
|
|
92
|
+
} catch (error) {
|
|
93
|
+
failures.push({ index, issue: { ...identity(seed), careerUrls: [pageUrl], reason: "career_page_failed", detail: error instanceof Error ? error.message : String(error) } });
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
79
97
|
if (!found) {
|
|
80
98
|
const lead = commonCrawlLeads.find((item) => sourceMatchesCompany(item.source.token, seed));
|
|
81
99
|
if (lead) { found = lead.source; reference = lead.reference; channel = "dataset"; }
|
|
82
100
|
}
|
|
83
101
|
if (!found) {
|
|
84
|
-
unresolved.push({ index, issue: { ...identity(seed), careerUrls, reason: "ats_not_resolved", detail: "Career URLs
|
|
102
|
+
unresolved.push({ index, issue: { ...identity(seed), careerUrls, reason: "ats_not_resolved", detail: "Career URLs neither redirected to nor linked a supported structured job source" } });
|
|
85
103
|
continue;
|
|
86
104
|
}
|
|
87
105
|
const companyName = seed.companyName.trim();
|
|
88
106
|
const companyDomain = normalizeDomain(seed.companyDomain);
|
|
89
107
|
const companyMatch = { companyName, companyDomain, method: "normalized_token" as const, reference };
|
|
90
|
-
const identityEvidence: IdentityEvidence[] = channel === "career_page" ? [{ companyName, companyDomain, kind:
|
|
108
|
+
const identityEvidence: IdentityEvidence[] = channel === "career_page" ? [{ companyName, companyDomain, kind: evidenceKind, reference, observedAt: new Date().toISOString() }] : [];
|
|
91
109
|
const lead: EnrichmentLead = { sourceKey: `${found.ats}:${found.token.toLowerCase()}`, sourceUrl: found.canonicalSourceUrl, ats: found.ats, token: found.token,
|
|
92
110
|
discoveredFrom: [{ channel, reference }], companyMatches: [companyMatch], identityEvidence, attempts: [] };
|
|
93
111
|
registryRows.push(lead);
|
|
@@ -96,7 +114,7 @@ export async function traceCareerSources(inputPath: string, candidatesPath: stri
|
|
|
96
114
|
companyName, companyDomain, sourceUrl: found.canonicalSourceUrl,
|
|
97
115
|
cohorts: country ? [country] : undefined,
|
|
98
116
|
discoveredFrom: { channel, reference },
|
|
99
|
-
domainEvidence: channel === "career_page" ? { kind:
|
|
117
|
+
domainEvidence: channel === "career_page" ? { kind: evidenceKind, reference } : undefined,
|
|
100
118
|
} });
|
|
101
119
|
}
|
|
102
120
|
}
|
|
@@ -24,6 +24,7 @@ export interface EnrichmentRegistry { version: 1; updatedAt: string; leads: Enri
|
|
|
24
24
|
const evidenceRank: Record<IdentityEvidence["kind"], number> = {
|
|
25
25
|
provider_structured_domain: 4,
|
|
26
26
|
company_redirect: 3,
|
|
27
|
+
company_page_link: 3,
|
|
27
28
|
company_registry: 2,
|
|
28
29
|
authoritative_dataset: 1,
|
|
29
30
|
};
|
|
@@ -32,7 +33,7 @@ export function deriveLeadState(lead: EnrichmentLead): EnrichmentState {
|
|
|
32
33
|
const latestAttempt = lead.attempts.at(-1);
|
|
33
34
|
if (latestAttempt?.outcome === "permanent_failure" && !supersededFailure(lead, latestAttempt)) return "rejected";
|
|
34
35
|
if (!lead.companyMatches.length) return lead.promotedAt && latestAttempt?.outcome === "success" ? "verified" : "unresolved";
|
|
35
|
-
const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect"].includes(evidence.kind));
|
|
36
|
+
const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect", "company_page_link"].includes(evidence.kind));
|
|
36
37
|
if (!qualifying.length) return lead.promotedAt && latestAttempt?.outcome === "success" ? "verified" : "matched";
|
|
37
38
|
const winningRank = Math.max(...qualifying.map((evidence) => evidenceRank[evidence.kind]));
|
|
38
39
|
const winningDomains = new Set(qualifying.filter((evidence) => evidenceRank[evidence.kind] === winningRank).map((evidence) => normalizeDomain(evidence.companyDomain)));
|
|
@@ -125,7 +126,7 @@ function isRegistry(value: unknown): value is EnrichmentRegistry {
|
|
|
125
126
|
function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }
|
|
126
127
|
function validProvenance(value: unknown): boolean { return isRecord(value) && ["search", "career_page", "provider_directory", "community", "dataset", "legacy"].includes(String(value.channel)) && typeof value.reference === "string"; }
|
|
127
128
|
function validMatch(value: unknown): boolean { return isRecord(value) && typeof value.companyName === "string" && typeof value.companyDomain === "string" && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(value.companyDomain) && ["normalized_token", "normalized_name", "normalized_domain", "search_result"].includes(String(value.method)) && typeof value.reference === "string"; }
|
|
128
|
-
function validEvidence(value: unknown): boolean { return isRecord(value) && typeof value.companyName === "string" && typeof value.companyDomain === "string" && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(value.companyDomain) && ["provider_structured_domain", "company_redirect", "company_registry", "authoritative_dataset"].includes(String(value.kind)) && typeof value.reference === "string" && typeof value.observedAt === "string" && Number.isFinite(Date.parse(value.observedAt)); }
|
|
129
|
+
function validEvidence(value: unknown): boolean { return isRecord(value) && typeof value.companyName === "string" && typeof value.companyDomain === "string" && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(value.companyDomain) && ["provider_structured_domain", "company_redirect", "company_page_link", "company_registry", "authoritative_dataset"].includes(String(value.kind)) && typeof value.reference === "string" && typeof value.observedAt === "string" && Number.isFinite(Date.parse(value.observedAt)); }
|
|
129
130
|
function validAttempt(value: unknown): boolean { return isRecord(value) && typeof value.attemptedAt === "string" && Number.isFinite(Date.parse(value.attemptedAt)) && ["success", "transient_failure", "permanent_failure"].includes(String(value.outcome)) && (value.nextEligibleAt === undefined || typeof value.nextEligibleAt === "string" && Number.isFinite(Date.parse(value.nextEligibleAt))) && (value.category === undefined || typeof value.category === "string") && (value.detail === undefined || typeof value.detail === "string") && (value.evidenceRank === undefined || typeof value.evidenceRank === "number" && Number.isFinite(value.evidenceRank)); }
|
|
130
131
|
function sourceIdentityMatches(value: Record<string, unknown>): boolean {
|
|
131
132
|
const source = resolveSource(String(value.sourceUrl));
|
package/src/providers.ts
CHANGED
|
@@ -204,8 +204,9 @@ export function countryLabel(value: string): string {
|
|
|
204
204
|
try { const name = regionNames.of(code); return name && name !== code ? name : value.trim(); } catch { return value.trim(); }
|
|
205
205
|
}
|
|
206
206
|
|
|
207
|
+
const RESERVED_TOKENS = new Set(["my-applications", "sign-in", "signin", "login", "register", "privacy", "cookie-policy", "job-alerts", "search"]);
|
|
207
208
|
function validToken(value: string | undefined): string | null {
|
|
208
|
-
return value && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) ? value : null;
|
|
209
|
+
return value && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) && !RESERVED_TOKENS.has(value.toLowerCase()) ? value : null;
|
|
209
210
|
}
|
|
210
211
|
|
|
211
212
|
export function plainText(value: string): string {
|
package/src/safe-head.ts
CHANGED
|
@@ -6,6 +6,63 @@ import { connect as connectTls } from "node:tls";
|
|
|
6
6
|
export type ResolveHost = (hostname: string) => Promise<string[]>;
|
|
7
7
|
export type HeadTransport = (url: URL, address: string, signal: AbortSignal) => Promise<Response>;
|
|
8
8
|
export interface SafeHeadResult { response: Response; finalUrl: string }
|
|
9
|
+
export type PageTransport = (url: URL, address: string, signal: AbortSignal, maxBytes: number) => Promise<Response>;
|
|
10
|
+
export interface SafePageResult { status: number; finalUrl: string; html: string }
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* One bounded GET of a company-owned page, with the same DNS pinning and HTTPS-only redirect rules as fetchSafeHead.
|
|
14
|
+
* Reads at most `maxBytes` and only ever hands back the body for link extraction; nothing else is retained.
|
|
15
|
+
*/
|
|
16
|
+
export async function fetchSafePage(url: string, options: { resolveHost?: ResolveHost; transport?: PageTransport; timeoutMs?: number; maxBytes?: number } = {}): Promise<SafePageResult> {
|
|
17
|
+
const maxBytes = options.maxBytes ?? 1_000_000;
|
|
18
|
+
const controller = new AbortController();
|
|
19
|
+
const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${options.timeoutMs ?? 15_000}ms`)), options.timeoutMs ?? 15_000);
|
|
20
|
+
try {
|
|
21
|
+
let current = new URL(url);
|
|
22
|
+
for (let redirects = 0; redirects <= 5; redirects += 1) {
|
|
23
|
+
const address = await withAbort(publicAddress(current, options.resolveHost ?? resolveAddresses), controller.signal);
|
|
24
|
+
const response = await withAbort((options.transport ?? pinnedGet)(current, address, controller.signal, maxBytes), controller.signal);
|
|
25
|
+
if ([301, 302, 303, 307, 308].includes(response.status) && response.headers.get("location")) {
|
|
26
|
+
current = new URL(response.headers.get("location")!, current);
|
|
27
|
+
if (current.protocol !== "https:") throw new Error("Career redirect must use HTTPS");
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
const html = (await response.text()).slice(0, maxBytes);
|
|
31
|
+
return { status: response.status, finalUrl: current.href, html };
|
|
32
|
+
}
|
|
33
|
+
throw new Error("Career redirect limit exceeded");
|
|
34
|
+
} finally { clearTimeout(timer); }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** True unless robots.txt for the page's origin disallows the path for everyone or for this crawler. Fetch failures count as allowed, an absent file does. */
|
|
38
|
+
export async function robotsAllows(pageUrl: string, options: { resolveHost?: ResolveHost; transport?: PageTransport; timeoutMs?: number } = {}): Promise<boolean> {
|
|
39
|
+
const url = new URL(pageUrl);
|
|
40
|
+
let text = "";
|
|
41
|
+
try {
|
|
42
|
+
const robots = await fetchSafePage(`${url.origin}/robots.txt`, { ...options, maxBytes: 64_000 });
|
|
43
|
+
if (robots.status !== 200) return true;
|
|
44
|
+
text = robots.html;
|
|
45
|
+
} catch { return true; }
|
|
46
|
+
let applies = false;
|
|
47
|
+
const disallowed: string[] = [];
|
|
48
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
49
|
+
const line = raw.replace(/#.*$/, "").trim();
|
|
50
|
+
const agent = /^user-agent:\s*(.+)$/i.exec(line);
|
|
51
|
+
if (agent) { const name = agent[1]!.trim().toLowerCase(); applies = name === "*" || name.includes("openings"); continue; }
|
|
52
|
+
const rule = /^disallow:\s*(.*)$/i.exec(line);
|
|
53
|
+
if (rule && applies && rule[1]!.trim()) disallowed.push(rule[1]!.trim());
|
|
54
|
+
}
|
|
55
|
+
return !disallowed.some((prefix) => url.pathname.startsWith(prefix));
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Absolute https links found in a page's anchors. Only hrefs are read; the page text is never inspected. */
|
|
59
|
+
export function extractLinks(html: string, base: string): string[] {
|
|
60
|
+
const links = new Set<string>();
|
|
61
|
+
for (const match of html.matchAll(/href\s*=\s*["']([^"'#\s]+)["']/gi)) {
|
|
62
|
+
try { const url = new URL(match[1]!, base); if (url.protocol === "https:") links.add(url.href); } catch { /* not a url */ }
|
|
63
|
+
}
|
|
64
|
+
return [...links];
|
|
65
|
+
}
|
|
9
66
|
|
|
10
67
|
export async function fetchSafeHead(url: string, options: { resolveHost?: ResolveHost; transport?: HeadTransport; timeoutMs?: number } = {}): Promise<SafeHeadResult> {
|
|
11
68
|
const controller = new AbortController();
|
|
@@ -66,6 +123,28 @@ function pinnedHead(url: URL, address: string, signal: AbortSignal): Promise<Res
|
|
|
66
123
|
});
|
|
67
124
|
}
|
|
68
125
|
|
|
126
|
+
function pinnedGet(url: URL, address: string, signal: AbortSignal, maxBytes: number): Promise<Response> {
|
|
127
|
+
return new Promise((resolve, reject) => {
|
|
128
|
+
const req = request(url, {
|
|
129
|
+
method: "GET", signal, servername: url.hostname, agent: false, headers: { "user-agent": "openings-discovery/0.1 (+https://avagama.co/openings/)", accept: "text/html" },
|
|
130
|
+
createConnection: () => {
|
|
131
|
+
const socket = connectTls({ host: address, port: Number(url.port || 443), servername: url.hostname });
|
|
132
|
+
const destroy = () => socket.destroy(signal.reason instanceof Error ? signal.reason : undefined);
|
|
133
|
+
if (signal.aborted) destroy();
|
|
134
|
+
else signal.addEventListener("abort", destroy, { once: true });
|
|
135
|
+
return socket;
|
|
136
|
+
},
|
|
137
|
+
}, (response) => {
|
|
138
|
+
const chunks: Buffer[] = []; let size = 0;
|
|
139
|
+
response.on("data", (chunk: Buffer) => { if (size < maxBytes) { chunks.push(chunk); size += chunk.length; } if (size >= maxBytes) response.destroy(); });
|
|
140
|
+
const finish = () => resolve(new Response(Buffer.concat(chunks).subarray(0, maxBytes), { status: response.statusCode ?? 500, headers: response.headers as HeadersInit }));
|
|
141
|
+
response.once("end", finish); response.once("close", finish); response.once("error", finish);
|
|
142
|
+
});
|
|
143
|
+
req.once("error", reject);
|
|
144
|
+
req.end();
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
69
148
|
function isPublicAddress(address: string): boolean {
|
|
70
149
|
if (isIP(address) === 4) {
|
|
71
150
|
const [a, b] = address.split(".").map(Number);
|
package/src/source-discovery.ts
CHANGED
|
@@ -17,7 +17,7 @@ interface DiscoveryOptions {
|
|
|
17
17
|
registryPath?: string;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
interface FeedEntry { sourceUrl: string; companyName?: string; companyDomain?: string; reference?: string; channel?: DiscoveryChannel; domainEvidence?: "authoritative_dataset" | "company_registry" | "company_redirect" }
|
|
20
|
+
interface FeedEntry { sourceUrl: string; companyName?: string; companyDomain?: string; reference?: string; channel?: DiscoveryChannel; domainEvidence?: "authoritative_dataset" | "company_registry" | "company_redirect" | "company_page_link" }
|
|
21
21
|
interface DiscoveryIssue { sourceUrl: string; reason: string; detail: string; companyName?: string; reference?: string }
|
|
22
22
|
|
|
23
23
|
export interface SourceDiscoveryReport extends ReportMeta {
|
|
@@ -106,7 +106,7 @@ async function discoverEntries(feed: FeedEntry[], candidatesPath: string, report
|
|
|
106
106
|
rejected.push({ index, issue: issue(entry, "invalid_channel", `Unsupported discovery channel: ${entry.channel}`) });
|
|
107
107
|
continue;
|
|
108
108
|
}
|
|
109
|
-
if (entry.domainEvidence !== undefined && !["authoritative_dataset", "company_registry", "company_redirect"].includes(entry.domainEvidence)) {
|
|
109
|
+
if (entry.domainEvidence !== undefined && !["authoritative_dataset", "company_registry", "company_redirect", "company_page_link"].includes(entry.domainEvidence)) {
|
|
110
110
|
rejected.push({ index, issue: issue(entry, "invalid_domain_evidence", `Unsupported domain evidence: ${entry.domainEvidence}`) });
|
|
111
111
|
continue;
|
|
112
112
|
}
|
|
@@ -114,8 +114,8 @@ async function discoverEntries(feed: FeedEntry[], candidatesPath: string, report
|
|
|
114
114
|
const companyName = typeof entry.companyName === "string" ? entry.companyName.trim() : "";
|
|
115
115
|
const reference = entry.reference?.trim() || feedReference;
|
|
116
116
|
const match = entry.companyDomain && companyName ? [{ companyName, companyDomain: entry.companyDomain.toLowerCase(), method: "normalized_token" as const, reference }] : [];
|
|
117
|
-
const redirectTrusted = entry.domainEvidence === "company_redirect" && entry.companyDomain && referenceBelongsToDomain(reference, entry.companyDomain);
|
|
118
|
-
const datasetTrusted = trustDomainEvidence && entry.domainEvidence && entry.domainEvidence !== "company_redirect";
|
|
117
|
+
const redirectTrusted = (entry.domainEvidence === "company_redirect" || entry.domainEvidence === "company_page_link") && entry.companyDomain && referenceBelongsToDomain(reference, entry.companyDomain);
|
|
118
|
+
const datasetTrusted = trustDomainEvidence && entry.domainEvidence && entry.domainEvidence !== "company_redirect" && entry.domainEvidence !== "company_page_link";
|
|
119
119
|
const evidence: IdentityEvidence[] = (redirectTrusted || datasetTrusted) && entry.companyDomain && companyName ? [{ companyName, companyDomain: entry.companyDomain.toLowerCase(), kind: entry.domainEvidence!, reference, observedAt: new Date().toISOString() }] : [];
|
|
120
120
|
registryRows.push({ sourceKey: key, sourceUrl: source.canonicalSourceUrl, ats: source.ats, token: source.token,
|
|
121
121
|
discoveredFrom: [{ channel: entry.channel ?? "dataset", reference }], companyMatches: match, identityEvidence: evidence, attempts: [] });
|
package/src/source-enrichment.ts
CHANGED
|
@@ -54,8 +54,8 @@ export async function enrichSourcesFromCompanies(registryPath: string, companies
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
function winningIdentity(lead: EnrichmentLead): IdentityEvidence | undefined {
|
|
57
|
-
const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect"].includes(evidence.kind));
|
|
58
|
-
const rank = { provider_structured_domain: 4, company_redirect: 3, company_registry: 2, authoritative_dataset: 1 } as const;
|
|
57
|
+
const qualifying = lead.identityEvidence.filter((evidence) => !["lever", "ashby"].includes(lead.ats) || ["provider_structured_domain", "company_redirect", "company_page_link"].includes(evidence.kind));
|
|
58
|
+
const rank = { provider_structured_domain: 4, company_redirect: 3, company_page_link: 3, company_registry: 2, authoritative_dataset: 1 } as const;
|
|
59
59
|
return [...qualifying].sort((left, right) => rank[right.kind] - rank[left.kind] || left.companyDomain.localeCompare(right.companyDomain))[0];
|
|
60
60
|
}
|
|
61
61
|
function countStates(leads: EnrichmentLead[]): Record<string, number> { const counts: Record<string, number> = {}; for (const lead of leads) { const state = deriveLeadState(lead); counts[state] = (counts[state] ?? 0) + 1; } return counts; }
|
|
@@ -84,7 +84,7 @@ function isVerifiedCatalogRecord(company: unknown): boolean {
|
|
|
84
84
|
&& verification.canonicalSourceUrl === source.canonicalSourceUrl
|
|
85
85
|
&& typeof verification.checkedAt === "string" && Number.isFinite(Date.parse(verification.checkedAt))
|
|
86
86
|
&& typeof verification.observedCompanyName === "string" && verification.observedCompanyName.length > 0
|
|
87
|
-
&& ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect"].includes(String(verification.identityEvidence))
|
|
87
|
+
&& ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect", "company_page_link"].includes(String(verification.identityEvidence))
|
|
88
88
|
&& typeof verification.contentType === "string" && typeof verification.payloadVersion === "string" && verification.payloadVersion.length > 0
|
|
89
89
|
&& Number.isInteger(verification.jobCount) && Number(verification.jobCount) > 0;
|
|
90
90
|
}
|
package/src/source-pipeline.ts
CHANGED
|
@@ -195,6 +195,8 @@ async function recordRegistryOutcomes(registryPath: string, result: Awaited<Retu
|
|
|
195
195
|
? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "provider_structured_domain", reference: company.sourceUrl, observedAt: now.toISOString() }]
|
|
196
196
|
: company.verification.identityEvidence === "company_redirect" && company.domainEvidence?.kind === "company_redirect"
|
|
197
197
|
? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "company_redirect", reference: company.domainEvidence.reference, observedAt: now.toISOString() }]
|
|
198
|
+
: company.verification.identityEvidence === "company_page_link" && company.domainEvidence?.kind === "company_page_link"
|
|
199
|
+
? [{ companyName: company.name, companyDomain: company.companyDomain, kind: "company_page_link", reference: company.domainEvidence.reference, observedAt: now.toISOString() }]
|
|
198
200
|
: [];
|
|
199
201
|
additions.push({ ...lead, identityEvidence: evidence, attempts: [attempt], promotedAt: now.toISOString() });
|
|
200
202
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Ats, RejectedSource, SourceCandidate, SourceRejectionReason, SourceVerificationResult, VerifiedCompany } from "./types.ts";
|
|
2
|
-
import { fetchSafeHead, type HeadTransport, type ResolveHost } from "./safe-head.ts";
|
|
2
|
+
import { extractLinks, fetchSafeHead, fetchSafePage, robotsAllows, type HeadTransport, type PageTransport, type ResolveHost } from "./safe-head.ts";
|
|
3
3
|
import { abortableDelay, fetchSourceJobs, isTransientStatus, retryDelayMs } from "./catalog.ts";
|
|
4
4
|
import { isEligibleForCountry } from "./locations.ts";
|
|
5
5
|
import { providerSpec, resolveProviderSource } from "./providers.ts";
|
|
@@ -14,6 +14,7 @@ interface VerificationOptions {
|
|
|
14
14
|
timeoutMs?: number;
|
|
15
15
|
resolveHost?: ResolveHost;
|
|
16
16
|
headTransport?: HeadTransport;
|
|
17
|
+
pageTransport?: PageTransport;
|
|
17
18
|
requireCountry?: string;
|
|
18
19
|
countryGateTimeoutMs?: number;
|
|
19
20
|
providerConcurrency?: Partial<Record<Ats, number>>;
|
|
@@ -56,8 +57,9 @@ export async function verifyCandidates(candidates: SourceCandidate[], options: V
|
|
|
56
57
|
const release = await providerLimits.get(source.ats)!.acquire();
|
|
57
58
|
const providerFetch = providerCooldowns.get(source.ats)!.wrap(fetcher);
|
|
58
59
|
try {
|
|
59
|
-
const evidence = await probe(candidate, source, providerFetch, timeoutMs, options.resolveHost, options.headTransport);
|
|
60
|
-
|
|
60
|
+
const evidence = await probe(candidate, source, providerFetch, timeoutMs, options.resolveHost, options.headTransport, options.pageTransport);
|
|
61
|
+
const replayed = evidence.identityEvidence === "company_redirect" || evidence.identityEvidence === "company_page_link";
|
|
62
|
+
if (!replayed && !identityMatches(candidate.companyName, candidate.companyDomain, evidence.observedCompanyName, source.token)) {
|
|
61
63
|
rejected.push({ index, value: rejection(candidate, "identity_mismatch", `Expected ${candidate.companyName}; observed ${evidence.observedCompanyName}`) });
|
|
62
64
|
continue;
|
|
63
65
|
}
|
|
@@ -149,7 +151,7 @@ class ProviderCooldown {
|
|
|
149
151
|
}
|
|
150
152
|
}
|
|
151
153
|
|
|
152
|
-
async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher: Fetch, timeoutMs: number, resolveHost?: ResolveHost, headTransport?: HeadTransport) {
|
|
154
|
+
async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher: Fetch, timeoutMs: number, resolveHost?: ResolveHost, headTransport?: HeadTransport, pageTransport?: PageTransport) {
|
|
153
155
|
const controller = new AbortController();
|
|
154
156
|
const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
155
157
|
const workday = source.ats === "workday" ? parseWorkdayToken(source.token) : undefined;
|
|
@@ -169,14 +171,16 @@ async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher
|
|
|
169
171
|
if (!jobs) throw new VerificationError("invalid_payload", "Payload does not contain the expected jobs array");
|
|
170
172
|
if (jobs.length === 0) throw new VerificationError("empty_board", "Source has no jobs, so identity cannot be verified");
|
|
171
173
|
const providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
|
|
172
|
-
const
|
|
174
|
+
const tenantMatches = source.ats === "workday" && identityMatches(candidate.companyName, candidate.companyDomain, workday!.tenant, source.token);
|
|
175
|
+
const namedProvider = source.ats === "greenhouse" || tenantMatches || Boolean(spec && providerName);
|
|
173
176
|
const hasDomainLink = !namedProvider && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
|
|
174
177
|
const hasRedirectEvidence = !namedProvider && await verifiedCompanyRedirect(candidate, source, resolveHost, headTransport, timeoutMs);
|
|
175
|
-
|
|
176
|
-
|
|
178
|
+
const hasPageLink = !namedProvider && !hasDomainLink && !hasRedirectEvidence && await verifiedCompanyPageLink(candidate, source, resolveHost, pageTransport, timeoutMs);
|
|
179
|
+
if (!namedProvider && !hasDomainLink && !hasRedirectEvidence && !hasPageLink) throw new VerificationError("identity_mismatch", `Neither structured identity fields, a verified company redirect, nor a company careers-page link point to ${candidate.companyDomain}`);
|
|
180
|
+
const observedCompanyName = namedProvider && providerName ? providerName : candidate.companyName;
|
|
177
181
|
return {
|
|
178
182
|
observedCompanyName,
|
|
179
|
-
identityEvidence:
|
|
183
|
+
identityEvidence: tenantMatches ? "provider_tenant" as const : (namedProvider && providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : hasRedirectEvidence ? "company_redirect" as const : "company_page_link" as const),
|
|
180
184
|
contentType,
|
|
181
185
|
payloadVersion: spec ? spec.payloadVersion(body) : source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
|
|
182
186
|
jobCount: source.ats === "workday" && isRecord(body) && typeof body.total === "number" ? body.total : jobs.length,
|
|
@@ -252,7 +256,7 @@ function validateCandidate(candidate: SourceCandidate): string | null {
|
|
|
252
256
|
if (!isRecord(candidate.discoveredFrom) || typeof candidate.discoveredFrom.channel !== "string" || !channels.has(candidate.discoveredFrom.channel) || typeof candidate.discoveredFrom.reference !== "string" || !candidate.discoveredFrom.reference.trim()) return "discoveredFrom must contain a supported channel and reference";
|
|
253
257
|
if (candidate.cohorts !== undefined && (!Array.isArray(candidate.cohorts) || !candidate.cohorts.every((code) => typeof code === "string"))) return "cohorts must be an array of country codes";
|
|
254
258
|
if (candidate.domainEvidence !== undefined) {
|
|
255
|
-
if (!isRecord(candidate.domainEvidence) || !["authoritative_dataset", "company_registry", "company_redirect"].includes(String(candidate.domainEvidence.kind)) || typeof candidate.domainEvidence.reference !== "string" || !candidate.domainEvidence.reference) return "domainEvidence must contain a supported kind and reference";
|
|
259
|
+
if (!isRecord(candidate.domainEvidence) || !["authoritative_dataset", "company_registry", "company_redirect", "company_page_link"].includes(String(candidate.domainEvidence.kind)) || typeof candidate.domainEvidence.reference !== "string" || !candidate.domainEvidence.reference) return "domainEvidence must contain a supported kind and reference";
|
|
256
260
|
}
|
|
257
261
|
return null;
|
|
258
262
|
}
|
|
@@ -300,4 +304,18 @@ async function verifiedCompanyRedirect(candidate: SourceCandidate, expected: Res
|
|
|
300
304
|
}
|
|
301
305
|
|
|
302
306
|
|
|
307
|
+
/** Replays company_page_link evidence: the company's own careers page, fetched once within robots rules, still links to the exact board. */
|
|
308
|
+
async function verifiedCompanyPageLink(candidate: SourceCandidate, expected: ResolvedSource, resolveHost?: ResolveHost, pageTransport?: PageTransport, timeoutMs?: number): Promise<boolean> {
|
|
309
|
+
if (candidate.domainEvidence?.kind !== "company_page_link") return false;
|
|
310
|
+
try {
|
|
311
|
+
const host = new URL(candidate.domainEvidence.reference).hostname.toLowerCase().replace(/^www\./, "");
|
|
312
|
+
const domain = candidate.companyDomain.toLowerCase().replace(/^www\./, "");
|
|
313
|
+
if (host !== domain && !host.endsWith(`.${domain}`)) return false;
|
|
314
|
+
if (!(await robotsAllows(candidate.domainEvidence.reference, { resolveHost, transport: pageTransport, timeoutMs }))) return false;
|
|
315
|
+
const page = await fetchSafePage(candidate.domainEvidence.reference, { resolveHost, transport: pageTransport, timeoutMs });
|
|
316
|
+
if (page.status !== 200) return false;
|
|
317
|
+
return extractLinks(page.html, page.finalUrl).some((link) => { const observed = resolveSource(link); return observed?.ats === expected.ats && observed.token.toLowerCase() === expected.token.toLowerCase(); });
|
|
318
|
+
} catch { return false; }
|
|
319
|
+
}
|
|
320
|
+
|
|
303
321
|
class VerificationError extends Error { constructor(readonly reason: SourceRejectionReason, message: string) { super(message); } }
|
package/src/types.ts
CHANGED
|
@@ -2,7 +2,7 @@ export const ALL_PROVIDERS = ["greenhouse", "lever", "ashby", "workday", "recrui
|
|
|
2
2
|
export type Ats = (typeof ALL_PROVIDERS)[number];
|
|
3
3
|
|
|
4
4
|
export interface DomainEvidence {
|
|
5
|
-
kind: "authoritative_dataset" | "company_registry" | "company_redirect";
|
|
5
|
+
kind: "authoritative_dataset" | "company_registry" | "company_redirect" | "company_page_link";
|
|
6
6
|
reference: string;
|
|
7
7
|
}
|
|
8
8
|
|
|
@@ -40,7 +40,7 @@ export interface SourceVerification {
|
|
|
40
40
|
checkedAt: string;
|
|
41
41
|
canonicalSourceUrl: string;
|
|
42
42
|
observedCompanyName: string;
|
|
43
|
-
identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect" | "provider_board";
|
|
43
|
+
identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect" | "company_page_link" | "provider_board";
|
|
44
44
|
contentType: string;
|
|
45
45
|
payloadVersion: string;
|
|
46
46
|
jobCount: number;
|
package/src/version.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const VERSION = "0.1.
|
|
1
|
+
export const VERSION = "0.1.13";
|