openings 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +2 -2
- package/README.md +1 -1
- package/data/companies.json +33544 -978
- package/package.json +2 -2
- package/src/board-verification.ts +6 -4
- package/src/catalog.ts +19 -0
- package/src/cli.ts +32 -3
- package/src/common-crawl-discovery.ts +5 -3
- package/src/country-coverage.ts +2 -1
- package/src/enrichment-registry.ts +3 -2
- package/src/mcp.ts +1 -1
- package/src/providers.ts +183 -0
- package/src/source-verification.ts +13 -7
- package/src/subdomain-discovery.ts +85 -0
- package/src/types.ts +2 -1
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "openings",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "A free, candidate-safe job-search substrate for AI agents",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },
|
|
7
7
|
"homepage": "https://github.com/abhay-avagama/hiring-agent#readme",
|
|
8
8
|
"bugs": { "url": "https://github.com/abhay-avagama/hiring-agent/issues" },
|
|
9
|
-
"keywords": ["jobs", "resume", "matching", "mcp", "greenhouse", "lever", "ashby", "workday", "recruitee"],
|
|
9
|
+
"keywords": ["jobs", "resume", "matching", "mcp", "greenhouse", "lever", "ashby", "workday", "recruitee", "smartrecruiters", "workable", "breezy"],
|
|
10
10
|
"type": "module",
|
|
11
11
|
"bin": { "openings-mcp": "./src/package-mcp.ts" },
|
|
12
12
|
"exports": "./src/index.ts",
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { atomicJson } from "./atomic-file.ts";
|
|
2
2
|
import { readEnrichmentRegistry, mergeEnrichmentLeads, type EnrichmentLead, type LeadAttempt } from "./enrichment-registry.ts";
|
|
3
3
|
import { withFileLock } from "./file-lock.ts";
|
|
4
|
+
import { providerSpec } from "./providers.ts";
|
|
4
5
|
import { resolveSource } from "./source-verification.ts";
|
|
5
6
|
import type { Ats, Company, SourceVerification } from "./types.ts";
|
|
6
7
|
|
|
7
8
|
type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
|
|
8
9
|
|
|
9
10
|
/** Providers whose public board is accepted as identity on its own. Workday stays on the company-identity path because its crawls are expensive. */
|
|
10
|
-
export const BOARD_TIER_PROVIDERS: ReadonlySet<Ats> = new Set(["greenhouse", "lever", "ashby", "recruitee"]);
|
|
11
|
+
export const BOARD_TIER_PROVIDERS: ReadonlySet<Ats> = new Set(["greenhouse", "lever", "ashby", "recruitee", "smartrecruiters", "workable", "breezy"]);
|
|
11
12
|
|
|
12
13
|
export interface BoardVerificationOptions {
|
|
13
14
|
fetch?: Fetch;
|
|
@@ -127,11 +128,12 @@ async function probeBoard(lead: EnrichmentLead, fetcher: Fetch, timeoutMs: numbe
|
|
|
127
128
|
if (!response.ok) { await response.body?.cancel().catch(() => undefined); throw new BoardError(response.status === 404 || response.status === 410 ? "invalid_payload" : "unreachable", `HTTP ${response.status}`); }
|
|
128
129
|
let body: unknown;
|
|
129
130
|
try { body = await response.json(); } catch { throw new BoardError("invalid_payload", "Endpoint did not return JSON"); }
|
|
130
|
-
const
|
|
131
|
+
const spec = providerSpec(source.ats);
|
|
132
|
+
const jobs = spec ? spec.jobsFromBody(body) : source.ats === "lever" ? asArray(body) : asArray(isRecord(body) ? body[source.ats === "recruitee" ? "offers" : "jobs"] : undefined);
|
|
131
133
|
if (!jobs) throw new BoardError("invalid_payload", "Payload does not contain the expected jobs array");
|
|
132
134
|
if (jobs.length === 0) throw new BoardError("empty_board", "Board has no jobs");
|
|
133
|
-
const providerName = source.ats === "greenhouse" ? majority(jobs.map((job) => typeof job.company_name === "string" ? job.company_name.trim() : "").filter(Boolean)) : "";
|
|
134
|
-
const payloadVersion = source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`;
|
|
135
|
+
const providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => typeof job.company_name === "string" ? job.company_name.trim() : "").filter(Boolean)) : "";
|
|
136
|
+
const payloadVersion = spec ? spec.payloadVersion(body) : source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`;
|
|
135
137
|
return { providerName, contentType: response.headers.get("content-type") ?? "unknown", payloadVersion, jobCount: jobs.length };
|
|
136
138
|
} finally { clearTimeout(timer); }
|
|
137
139
|
}
|
package/src/catalog.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Company, Job, JobSummary, SearchQuery } from "./types.ts";
|
|
2
|
+
import { providerSpec, type JsonGet } from "./providers.ts";
|
|
2
3
|
import { classifyJob, isEligibleForCountry, normalizeLocation } from "./locations.ts";
|
|
3
4
|
|
|
4
5
|
type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
|
|
@@ -100,6 +101,8 @@ export function createCatalog(options: CatalogOptions): Catalog {
|
|
|
100
101
|
const description = await fetchWorkdayDescription(company, job.url, fetcher);
|
|
101
102
|
job = { ...job, description };
|
|
102
103
|
}
|
|
104
|
+
const spec = job && !job.description.trim() ? providerSpec(company.ats) : undefined;
|
|
105
|
+
if (job && spec?.detail) job = { ...job, description: await spec.detail(company, job, jsonGetter(fetcher, company.name)) };
|
|
103
106
|
if (job && !job.description.trim()) throw new Error(`Full description unavailable for job: ${id}`);
|
|
104
107
|
return job;
|
|
105
108
|
},
|
|
@@ -118,6 +121,13 @@ export interface FetchJobsObserver {
|
|
|
118
121
|
}
|
|
119
122
|
export async function fetchSourceJobs(company: Company, fetcher: Fetch = globalThis.fetch, signal?: AbortSignal, observer?: FetchJobsObserver): Promise<Job[]> {
|
|
120
123
|
if (company.ats === "workday") return fetchWorkdayJobs(company, fetcher, signal, observer);
|
|
124
|
+
const spec = providerSpec(company.ats);
|
|
125
|
+
if (spec) {
|
|
126
|
+
const get = jsonGetter(fetcher, company.name, signal, observer);
|
|
127
|
+
const records = spec.fetchAll ? await spec.fetchAll(company.token, get) : spec.jobsFromBody(await get(spec.endpoint(company.token)));
|
|
128
|
+
if (!records) throw new Error(`${company.name} job board returned an invalid payload`);
|
|
129
|
+
return records.map((record) => spec.normalize(company, record));
|
|
130
|
+
}
|
|
121
131
|
const url = company.ats === "greenhouse"
|
|
122
132
|
? `https://boards-api.greenhouse.io/v1/boards/${encodeURIComponent(company.token)}/jobs?content=true`
|
|
123
133
|
: company.ats === "lever"
|
|
@@ -196,6 +206,15 @@ async function fetchWorkdayJobs(company: Company, fetcher: Fetch, signal?: Abort
|
|
|
196
206
|
return jobs.map((job) => normalizeWorkday(company, source, job));
|
|
197
207
|
}
|
|
198
208
|
|
|
209
|
+
/** JSON fetch with the catalog's retry and backoff, shaped for the table-driven providers. */
|
|
210
|
+
function jsonGetter(fetcher: Fetch, companyName: string, signal?: AbortSignal, observer?: FetchJobsObserver): JsonGet {
|
|
211
|
+
return async (url) => {
|
|
212
|
+
const response = await fetchWithRetry(fetcher, url, signal ? { signal } : undefined, companyName, observer);
|
|
213
|
+
if (!response.ok) { await response.body?.cancel().catch(() => undefined); throw new Error(`${companyName} job board returned HTTP ${response.status}`); }
|
|
214
|
+
return response.json();
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
|
|
199
218
|
async function fetchWithRetry(fetcher: Fetch, input: string | URL, init: RequestInit | undefined, companyName: string, observer?: FetchJobsObserver): Promise<Response> {
|
|
200
219
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
201
220
|
const response = await fetcher(input, init);
|
package/src/cli.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
|
+
import { ALL_PROVIDERS, type Ats } from "./types.ts";
|
|
2
3
|
import { readFile } from "node:fs/promises";
|
|
3
4
|
import { relative, resolve } from "node:path";
|
|
4
5
|
import { createRuntime } from "./runtime.ts";
|
|
5
6
|
import { atomicJson } from "./atomic-file.ts";
|
|
6
7
|
import { verifyBoards } from "./board-verification.ts";
|
|
8
|
+
import { discoverCareerSubdomains } from "./subdomain-discovery.ts";
|
|
7
9
|
import { runSourceVerification } from "./source-pipeline.ts";
|
|
8
10
|
import { runSourceDiscovery, runYcSourceDiscovery } from "./source-discovery.ts";
|
|
9
11
|
import { discoverAndPromote } from "./source-discovery-pipeline.ts";
|
|
@@ -23,12 +25,13 @@ Usage:
|
|
|
23
25
|
openings crawl [--country CODE | --companies FILE] [--concurrency N] [--source-cache-hours N] [--source-limit N] [--delay-ms N] [--workday-page-delay-ms N] [--data-dir PATH]
|
|
24
26
|
openings snapshot export [--input FILE] [--output-dir PATH]
|
|
25
27
|
openings coverage report --country CODE [--snapshot FILE] [--catalog FILE] [--candidates FILE] [--registry FILE] [--output FILE] [--as-of ISO]
|
|
28
|
+
openings sources discover-subdomains SEEDS.json [--output FILE] [--limit N] [--delay-ms N] [--report FILE]
|
|
26
29
|
openings sources verify-boards REGISTRY.json [--output FILE] [--limit N] [--concurrency N] [--report FILE]
|
|
27
30
|
openings sources verify CANDIDATES.json [--output FILE] [--state-file FILE] [--concurrency N] [--workday-concurrency N] [--limit N] [--require-country CODE] [--registry FILE] [--retry-deferred]
|
|
28
31
|
openings sources discover FEED.json [--country CODE] [--registry FILE] [--output FILE] [--catalog FILE] [--report FILE]
|
|
29
32
|
openings sources discover-yc --country CODE [--registry FILE] [--output FILE] [--catalog FILE] [--report FILE]
|
|
30
33
|
openings sources seed-companies-yc --country CODE [--output FILE]
|
|
31
|
-
openings sources discover-common-crawl [--country CODE] [--provider
|
|
34
|
+
openings sources discover-common-crawl [--country CODE] [--provider NAME] [--index-record-limit N] [--sample-token-limit N] [--exclude-token TOKEN] [--report-only] [--registry FILE] [--output FILE] [--report FILE] [--index-url URL]
|
|
32
35
|
openings sources enrich COMPANIES.json [--companies FILE]... [--evidence-kind authoritative_dataset|company_registry] [--registry FILE] [--output FILE] [--report FILE]
|
|
33
36
|
openings sources trace-careers COMPANIES.json [--country CODE] [--registry FILE] [--common-crawl-report FILE] [--search-key-env NAME] [--output FILE] [--catalog FILE] [--report FILE]
|
|
34
37
|
openings sources probe-jobposting COMPANIES.md [--catalog FILE] [--report FILE] [--company-limit 10|20]
|
|
@@ -159,6 +162,14 @@ export async function run(args: string[]): Promise<number> {
|
|
|
159
162
|
)), null, 2));
|
|
160
163
|
return 0;
|
|
161
164
|
}
|
|
165
|
+
if (rest[0] === "discover-subdomains") {
|
|
166
|
+
const parsed = parseDiscoverSubdomains(rest.slice(1));
|
|
167
|
+
if (typeof parsed === "string") return fail(parsed);
|
|
168
|
+
const report = await discoverCareerSubdomains(parsed.seeds, parsed.output, { limit: parsed.limit, delayMs: parsed.delayMs });
|
|
169
|
+
if (parsed.report) await atomicJson(parsed.report, report);
|
|
170
|
+
console.log(JSON.stringify({ ...report, seeds: report.seeds.length }, null, 2));
|
|
171
|
+
return 0;
|
|
172
|
+
}
|
|
162
173
|
if (rest[0] === "verify-boards") {
|
|
163
174
|
const parsed = parseVerifyBoards(rest.slice(1));
|
|
164
175
|
if (typeof parsed === "string") return fail(parsed);
|
|
@@ -318,7 +329,7 @@ function parseCommonCrawlDiscovery(args: string[]) {
|
|
|
318
329
|
let country: string | undefined;
|
|
319
330
|
let indexUrl: string | undefined;
|
|
320
331
|
let registryPath: string | undefined = "data/enrichment-leads.json";
|
|
321
|
-
let provider:
|
|
332
|
+
let provider: Ats | undefined;
|
|
322
333
|
let indexRecordLimit: number | undefined;
|
|
323
334
|
let sampleTokenLimit: number | undefined;
|
|
324
335
|
const excludeTokens: string[] = [];
|
|
@@ -331,7 +342,7 @@ function parseCommonCrawlDiscovery(args: string[]) {
|
|
|
331
342
|
else if (arg === "--index-url") { indexUrl = args[++index]; if (!indexUrl) return "--index-url requires a URL"; try { new URL(indexUrl); } catch { return "--index-url requires a valid URL"; } }
|
|
332
343
|
else if (arg === "--registry") { if (reportOnly) return "Use either --registry or --report-only, not both"; registryPath = args[++index] ?? ""; if (!registryPath) return "--registry requires a file"; }
|
|
333
344
|
else if (arg === "--report-only") { if (registryPath !== "data/enrichment-leads.json") return "Use either --registry or --report-only, not both"; reportOnly = true; registryPath = undefined; }
|
|
334
|
-
else if (arg === "--provider") { const value = args[++index]; if (value
|
|
345
|
+
else if (arg === "--provider") { const value = args[++index]; if (!value || !(ALL_PROVIDERS as readonly string[]).includes(value)) return `--provider must be one of ${ALL_PROVIDERS.join(", ")}`; provider = value as Ats; }
|
|
335
346
|
else if (arg === "--index-record-limit") { indexRecordLimit = Number(args[++index]); if (!Number.isInteger(indexRecordLimit) || indexRecordLimit < 1 || indexRecordLimit > 2_000) return "--index-record-limit must be an integer from 1 to 2000"; }
|
|
336
347
|
else if (arg === "--sample-token-limit") { sampleTokenLimit = Number(args[++index]); if (!Number.isInteger(sampleTokenLimit) || sampleTokenLimit < 1 || sampleTokenLimit > 60) return "--sample-token-limit must be an integer from 1 to 60"; }
|
|
337
348
|
else if (arg === "--exclude-token") { const value = args[++index]?.trim(); if (!value || !/^[a-z0-9-]+$/i.test(value)) return "--exclude-token requires an ATS token"; excludeTokens.push(value.toLowerCase()); }
|
|
@@ -544,6 +555,24 @@ async function readCompanyFile(path: string): Promise<string[]> {
|
|
|
544
555
|
return (await readFile(path, "utf8")).split(/\r?\n/).map((line) => line.trim()).filter((line) => line && !line.startsWith("#"));
|
|
545
556
|
}
|
|
546
557
|
|
|
558
|
+
function parseDiscoverSubdomains(args: string[]): { seeds: string; output: string; limit?: number; delayMs?: number; report?: string } | string {
|
|
559
|
+
const seeds = args[0];
|
|
560
|
+
if (!seeds || seeds.startsWith("--")) return "sources discover-subdomains requires a seed JSON file of { companyName, companyDomain }";
|
|
561
|
+
let output = ".openings/career-seeds-subdomains.json";
|
|
562
|
+
let limit: number | undefined;
|
|
563
|
+
let delayMs: number | undefined;
|
|
564
|
+
let report: string | undefined;
|
|
565
|
+
for (let index = 1; index < args.length; index += 1) {
|
|
566
|
+
const arg = args[index];
|
|
567
|
+
if (arg === "--output") output = args[++index] ?? output;
|
|
568
|
+
else if (arg === "--report") report = args[++index];
|
|
569
|
+
else if (arg === "--limit") { limit = Number(args[++index]); if (!Number.isInteger(limit) || limit < 1) return "--limit must be a positive integer"; }
|
|
570
|
+
else if (arg === "--delay-ms") { delayMs = Number(args[++index]); if (!Number.isInteger(delayMs) || delayMs < 0) return "--delay-ms must be a non-negative integer"; }
|
|
571
|
+
else return `Unknown option: ${arg}`;
|
|
572
|
+
}
|
|
573
|
+
return { seeds, output, limit, delayMs, report };
|
|
574
|
+
}
|
|
575
|
+
|
|
547
576
|
function parseVerifyBoards(args: string[]): { registry: string; output: string; limit?: number; concurrency?: number; report?: string } | string {
|
|
548
577
|
const registry = args[0];
|
|
549
578
|
if (!registry || registry.startsWith("--")) return "sources verify-boards requires a registry JSON file";
|
|
@@ -2,6 +2,7 @@ import { readFile } from "node:fs/promises";
|
|
|
2
2
|
import { atomicJson } from "./atomic-file.ts";
|
|
3
3
|
import { stampReport, type ReportMeta } from "./report-meta.ts";
|
|
4
4
|
import { mergeEnrichmentLeads, type EnrichmentLead } from "./enrichment-registry.ts";
|
|
5
|
+
import { PROVIDERS } from "./providers.ts";
|
|
5
6
|
import { resolveSource } from "./source-verification.ts";
|
|
6
7
|
import type { Ats } from "./types.ts";
|
|
7
8
|
import { assertArtifactFile } from "./artifact-path.ts";
|
|
@@ -36,11 +37,12 @@ export interface CommonCrawlDiscoveryReport extends ReportMeta {
|
|
|
36
37
|
rejections: Array<{ value: string; reason: string }>;
|
|
37
38
|
}
|
|
38
39
|
|
|
39
|
-
const patterns = ["job-boards.greenhouse.io/*", "boards.greenhouse.io/*", "jobs.lever.co/*", "jobs.ashbyhq.com/*", "*.myworkdayjobs.com/*", "*.recruitee.com/*"];
|
|
40
|
-
const recordsPerPattern = 10_000;
|
|
41
40
|
const providerPatterns: Record<Ats, string[]> = {
|
|
42
|
-
greenhouse:
|
|
41
|
+
greenhouse: ["job-boards.greenhouse.io/*", "boards.greenhouse.io/*"], lever: ["jobs.lever.co/*"], ashby: ["jobs.ashbyhq.com/*"], workday: ["*.myworkdayjobs.com/*"], recruitee: ["*.recruitee.com/*"],
|
|
42
|
+
...Object.fromEntries(PROVIDERS.map((spec) => [spec.ats, spec.crawlPatterns])) as Record<"smartrecruiters" | "workable" | "breezy", string[]>,
|
|
43
43
|
};
|
|
44
|
+
const patterns = Object.values(providerPatterns).flat();
|
|
45
|
+
const recordsPerPattern = 10_000;
|
|
44
46
|
|
|
45
47
|
export async function discoverCommonCrawlSources(candidatesPath: string, reportPath: string, options: Options = {}): Promise<CommonCrawlDiscoveryReport> {
|
|
46
48
|
if (options.provider === "recruitee") await assertArtifactFile(reportPath);
|
package/src/country-coverage.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { deriveLeadState, readEnrichmentRegistry, type EnrichmentState } from ".
|
|
|
4
4
|
import { isEligibleForCountry } from "./locations.ts";
|
|
5
5
|
import { stampReport, type ReportMeta } from "./report-meta.ts";
|
|
6
6
|
import { resolveSource } from "./source-verification.ts";
|
|
7
|
+
import { ALL_PROVIDERS } from "./types.ts";
|
|
7
8
|
import type { Ats, EligibilityConfidence, JobSnapshot, SourceCandidate, VerifiedCompany } from "./types.ts";
|
|
8
9
|
import { projectJobCoverage } from "./job-coverage.ts";
|
|
9
10
|
|
|
@@ -110,7 +111,7 @@ async function readCatalog(path: string): Promise<Record<string, Omit<VerifiedCo
|
|
|
110
111
|
const value: unknown = JSON.parse(await readFile(path, "utf8"));
|
|
111
112
|
if (!isRecord(value)) throw new Error(`Invalid verified catalog: ${path}`);
|
|
112
113
|
for (const [slug, company] of Object.entries(value)) {
|
|
113
|
-
if (!isRecord(company) || !
|
|
114
|
+
if (!isRecord(company) || !(ALL_PROVIDERS as readonly string[]).includes(String(company.ats)) || typeof company.token !== "string"
|
|
114
115
|
|| (typeof company.companyDomain !== "string" && !(isRecord(company.verification) && company.verification.identityEvidence === "provider_board")) || typeof company.sourceUrl !== "string"
|
|
115
116
|
|| company.cohorts !== undefined && (!Array.isArray(company.cohorts) || !company.cohorts.every(validCountryCode))
|
|
116
117
|
|| !validVerification(company.verification)) throw new Error(`Invalid verified catalog source: ${slug}`);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
2
|
import { atomicJson } from "./atomic-file.ts";
|
|
3
3
|
import { withFileLock } from "./file-lock.ts";
|
|
4
|
+
import { ALL_PROVIDERS } from "./types.ts";
|
|
4
5
|
import type { Ats, DiscoveryProvenance, DomainEvidence } from "./types.ts";
|
|
5
6
|
import { resolveSource } from "./source-verification.ts";
|
|
6
7
|
|
|
@@ -110,9 +111,9 @@ function consecutiveTransientFailures(attempts: LeadAttempt[]): number { let cou
|
|
|
110
111
|
function isRegistry(value: unknown): value is EnrichmentRegistry {
|
|
111
112
|
if (!isRecord(value) || value.version !== 1 || typeof value.updatedAt !== "string" || !Array.isArray(value.leads)) return false;
|
|
112
113
|
return value.leads.every((lead) => isRecord(lead)
|
|
113
|
-
&& typeof lead.sourceKey === "string" &&
|
|
114
|
+
&& typeof lead.sourceKey === "string" && new RegExp(`^(${ALL_PROVIDERS.join("|")}):.+`).test(lead.sourceKey)
|
|
114
115
|
&& typeof lead.sourceUrl === "string" && typeof lead.token === "string"
|
|
115
|
-
&&
|
|
116
|
+
&& (ALL_PROVIDERS as readonly string[]).includes(String(lead.ats))
|
|
116
117
|
&& Array.isArray(lead.discoveredFrom) && lead.discoveredFrom.every(validProvenance)
|
|
117
118
|
&& Array.isArray(lead.companyMatches) && lead.companyMatches.every(validMatch)
|
|
118
119
|
&& Array.isArray(lead.identityEvidence) && lead.identityEvidence.every(validEvidence)
|
package/src/mcp.ts
CHANGED
|
@@ -22,7 +22,7 @@ export function createMcpHandler(tools: ToolHandler) {
|
|
|
22
22
|
const base = { jsonrpc: "2.0" as const, id: request.id ?? null };
|
|
23
23
|
try {
|
|
24
24
|
if (request.method === "initialize") {
|
|
25
|
-
return { ...base, result: { protocolVersion: "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: "openings", version: "0.1.
|
|
25
|
+
return { ...base, result: { protocolVersion: "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: "openings", version: "0.1.4" } } };
|
|
26
26
|
}
|
|
27
27
|
if (request.method === "ping") return { ...base, result: {} };
|
|
28
28
|
if (request.method === "tools/list") return { ...base, result: { tools: tools.list() } };
|
package/src/providers.ts
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { classifyJob } from "./locations.ts";
|
|
2
|
+
import type { Ats, Company, Job } from "./types.ts";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Table-driven adapters for providers added after the original five. Each entry knows how to recognise a board URL,
|
|
6
|
+
* where its public structured endpoint lives, how to read the payload, and how to turn one record into a Job.
|
|
7
|
+
* The original providers keep their hand-written paths; new ones only need a row here.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
type Rec = Record<string, unknown>;
|
|
11
|
+
/** Fetches a URL and returns its parsed JSON body; the caller supplies retry and pacing. */
|
|
12
|
+
export type JsonGet = (url: string) => Promise<unknown>;
|
|
13
|
+
|
|
14
|
+
export interface ProviderSpec {
|
|
15
|
+
ats: Ats;
|
|
16
|
+
label: string;
|
|
17
|
+
/** Registrable host suffixes on which this provider serves boards and job pages. */
|
|
18
|
+
hosts: string[];
|
|
19
|
+
/** Common Crawl URL patterns that surface this provider's boards. */
|
|
20
|
+
crawlPatterns: string[];
|
|
21
|
+
resolve(url: URL): string | null;
|
|
22
|
+
canonicalUrl(token: string): string;
|
|
23
|
+
endpoint(token: string): string;
|
|
24
|
+
jobsFromBody(body: unknown): Rec[] | null;
|
|
25
|
+
providerName(jobs: Rec[], body: unknown): string;
|
|
26
|
+
payloadVersion(body: unknown): string;
|
|
27
|
+
normalize(company: Company, job: Rec): Job;
|
|
28
|
+
/** Fetches every record when the endpoint paginates. Defaults to one request to `endpoint`. */
|
|
29
|
+
fetchAll?(token: string, get: JsonGet): Promise<Rec[]>;
|
|
30
|
+
/** Fetches the full description for one job when the listing omits it. */
|
|
31
|
+
detail?(company: Company, job: Job, get: JsonGet): Promise<string>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const PAGE = 100;
|
|
35
|
+
const MAX_RECORDS = 2000;
|
|
36
|
+
|
|
37
|
+
const smartrecruiters: ProviderSpec = {
|
|
38
|
+
ats: "smartrecruiters",
|
|
39
|
+
label: "SmartRecruiters",
|
|
40
|
+
hosts: ["smartrecruiters.com"],
|
|
41
|
+
crawlPatterns: ["jobs.smartrecruiters.com/*", "careers.smartrecruiters.com/*"],
|
|
42
|
+
resolve(url) {
|
|
43
|
+
const parts = url.pathname.split("/").filter(Boolean);
|
|
44
|
+
if (url.hostname === "jobs.smartrecruiters.com" || url.hostname === "careers.smartrecruiters.com") return validToken(parts[0]);
|
|
45
|
+
if (url.hostname === "api.smartrecruiters.com" && parts[0] === "v1" && parts[1] === "companies") return validToken(parts[2]);
|
|
46
|
+
return null;
|
|
47
|
+
},
|
|
48
|
+
canonicalUrl: (token) => `https://jobs.smartrecruiters.com/${token}`,
|
|
49
|
+
endpoint: (token) => `https://api.smartrecruiters.com/v1/companies/${encodeURIComponent(token)}/postings?limit=${PAGE}&offset=0`,
|
|
50
|
+
jobsFromBody: (body) => (isRecord(body) ? asRecords(body.content) : null),
|
|
51
|
+
providerName: (jobs) => majority(jobs.map((job) => (isRecord(job.company) ? str(job.company.name) : ""))),
|
|
52
|
+
payloadVersion: () => "smartrecruiters-postings:v1",
|
|
53
|
+
async fetchAll(token, get) {
|
|
54
|
+
const records: Rec[] = [];
|
|
55
|
+
for (let offset = 0; offset < MAX_RECORDS; offset += PAGE) {
|
|
56
|
+
const body = await get(`https://api.smartrecruiters.com/v1/companies/${encodeURIComponent(token)}/postings?limit=${PAGE}&offset=${offset}`);
|
|
57
|
+
const page = smartrecruiters.jobsFromBody(body);
|
|
58
|
+
if (!page) throw new Error("SmartRecruiters returned an invalid postings payload");
|
|
59
|
+
records.push(...page);
|
|
60
|
+
const total = isRecord(body) && typeof body.totalFound === "number" ? body.totalFound : records.length;
|
|
61
|
+
if (page.length < PAGE || records.length >= total) break;
|
|
62
|
+
}
|
|
63
|
+
return records;
|
|
64
|
+
},
|
|
65
|
+
normalize(company, job) {
|
|
66
|
+
const loc = isRecord(job.location) ? job.location : {};
|
|
67
|
+
const country = str(loc.country);
|
|
68
|
+
const location = str(loc.fullLocation) || [str(loc.city), str(loc.region), country.length === 2 ? country.toUpperCase() : country].filter(Boolean).join(", ") || "Unspecified";
|
|
69
|
+
const remote = loc.remote === true;
|
|
70
|
+
return classifyJob({
|
|
71
|
+
id: `smartrecruiters:${company.slug}:${str(job.id)}`, company: company.name, title: str(job.name), location,
|
|
72
|
+
remote, workMode: remote ? "remote" : loc.hybrid === true ? "hybrid" : "unknown",
|
|
73
|
+
eligibleCountries: [], excludedCountries: [], eligibleRegions: [], eligibilityConfidence: "unknown",
|
|
74
|
+
url: `https://jobs.smartrecruiters.com/${company.token}/${encodeURIComponent(str(job.id))}`, updatedAt: str(job.releasedDate) || undefined, description: "",
|
|
75
|
+
});
|
|
76
|
+
},
|
|
77
|
+
async detail(company, job, get) {
|
|
78
|
+
const id = job.id.split(":")[2] ?? "";
|
|
79
|
+
const body = await get(`https://api.smartrecruiters.com/v1/companies/${encodeURIComponent(company.token)}/postings/${encodeURIComponent(id)}`);
|
|
80
|
+
const sections = isRecord(body) && isRecord(body.jobAd) && isRecord(body.jobAd.sections) ? body.jobAd.sections : {};
|
|
81
|
+
return ["jobDescription", "qualifications", "additionalInformation"].map((key) => { const section = sections[key]; return isRecord(section) ? plainText(str(section.text)) : ""; }).filter(Boolean).join("\n\n");
|
|
82
|
+
},
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
const workable: ProviderSpec = {
|
|
86
|
+
ats: "workable",
|
|
87
|
+
label: "Workable",
|
|
88
|
+
hosts: ["workable.com"],
|
|
89
|
+
crawlPatterns: ["apply.workable.com/*"],
|
|
90
|
+
resolve(url) {
|
|
91
|
+
const parts = url.pathname.split("/").filter(Boolean);
|
|
92
|
+
if (url.hostname === "apply.workable.com") return parts[0] && !["j", "api", "jobs", "embed", "assets"].includes(parts[0]) ? validToken(parts[0]) : null;
|
|
93
|
+
const match = /^([a-z0-9-]+)\.workable\.com$/i.exec(url.hostname);
|
|
94
|
+
if (match && !["www", "apply", "help", "resources", "jobs", "careers", "api", "app", "assets", "cdn", "status", "developers", "blog", "partners"].includes(match[1]!.toLowerCase())) return match[1]!.toLowerCase();
|
|
95
|
+
return null;
|
|
96
|
+
},
|
|
97
|
+
canonicalUrl: (token) => `https://apply.workable.com/${token}/`,
|
|
98
|
+
endpoint: (token) => `https://apply.workable.com/api/v1/widget/accounts/${encodeURIComponent(token)}`,
|
|
99
|
+
jobsFromBody: (body) => (isRecord(body) ? asRecords(body.jobs) : null),
|
|
100
|
+
providerName: (_jobs, body) => (isRecord(body) ? str(body.name) : ""),
|
|
101
|
+
payloadVersion: () => "workable-widget:v1",
|
|
102
|
+
normalize(company, job) {
|
|
103
|
+
const first = asRecords(job.locations)?.[0];
|
|
104
|
+
const location = [str(job.city), str(job.state), str(job.country) || (first ? str(first.country) : "")].filter(Boolean).join(", ") || "Unspecified";
|
|
105
|
+
const remote = job.telecommuting === true;
|
|
106
|
+
return classifyJob({
|
|
107
|
+
id: `workable:${company.slug}:${str(job.shortcode)}`, company: company.name, title: str(job.title), location,
|
|
108
|
+
remote, workMode: remote ? "remote" : "unknown",
|
|
109
|
+
eligibleCountries: [], excludedCountries: [], eligibleRegions: [], eligibilityConfidence: "unknown",
|
|
110
|
+
url: str(job.url) || `https://apply.workable.com/j/${encodeURIComponent(str(job.shortcode))}`, updatedAt: str(job.published_on) || undefined, description: "",
|
|
111
|
+
});
|
|
112
|
+
},
|
|
113
|
+
async detail(company, job, get) {
|
|
114
|
+
const shortcode = job.id.split(":")[2] ?? "";
|
|
115
|
+
const body = await get(`https://apply.workable.com/api/v2/accounts/${encodeURIComponent(company.token)}/jobs/${encodeURIComponent(shortcode)}`);
|
|
116
|
+
if (!isRecord(body)) return "";
|
|
117
|
+
return ["description", "requirements", "benefits"].map((key) => plainText(str(body[key]))).filter(Boolean).join("\n\n");
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
const breezy: ProviderSpec = {
|
|
122
|
+
ats: "breezy",
|
|
123
|
+
label: "Breezy",
|
|
124
|
+
hosts: ["breezy.hr"],
|
|
125
|
+
crawlPatterns: ["*.breezy.hr/*"],
|
|
126
|
+
resolve(url) {
|
|
127
|
+
const match = /^([a-z0-9-]+)\.breezy\.hr$/i.exec(url.hostname);
|
|
128
|
+
return match && !["www", "app", "api", "help", "blog"].includes(match[1]!.toLowerCase()) ? match[1]!.toLowerCase() : null;
|
|
129
|
+
},
|
|
130
|
+
canonicalUrl: (token) => `https://${token}.breezy.hr`,
|
|
131
|
+
endpoint: (token) => `https://${token}.breezy.hr/json?verbose=true`,
|
|
132
|
+
jobsFromBody: (body) => asRecords(body),
|
|
133
|
+
providerName: (jobs) => majority(jobs.map((job) => (isRecord(job.company) ? str(job.company.name) : ""))),
|
|
134
|
+
payloadVersion: () => "breezy-positions:v1",
|
|
135
|
+
normalize(company, job) {
|
|
136
|
+
const loc = isRecord(job.location) ? job.location : {};
|
|
137
|
+
const location = str(loc.name) || [str(loc.city), isRecord(loc.state) ? str(loc.state.name) : "", isRecord(loc.country) ? str(loc.country.name) : ""].filter(Boolean).join(", ") || "Unspecified";
|
|
138
|
+
const remote = loc.is_remote === true;
|
|
139
|
+
return classifyJob({
|
|
140
|
+
id: `breezy:${company.slug}:${str(job.id)}`, company: company.name, title: str(job.name), location,
|
|
141
|
+
remote, workMode: remote ? "remote" : "unknown",
|
|
142
|
+
eligibleCountries: [], excludedCountries: [], eligibleRegions: [], eligibilityConfidence: "unknown",
|
|
143
|
+
url: str(job.url) || `https://${company.token}.breezy.hr/p/${encodeURIComponent(str(job.friendly_id) || str(job.id))}`, updatedAt: str(job.published_date) || undefined, description: plainText(str(job.description)),
|
|
144
|
+
});
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
export const PROVIDERS: ReadonlyArray<ProviderSpec> = [smartrecruiters, workable, breezy];
|
|
149
|
+
|
|
150
|
+
export function providerSpec(ats: string): ProviderSpec | undefined {
|
|
151
|
+
return PROVIDERS.find((spec) => spec.ats === ats);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Resolves a URL against the table-driven providers; returns null when none claims it. */
|
|
155
|
+
export function resolveProviderSource(value: string): { ats: Ats; token: string; canonicalSourceUrl: string; structuredEndpoint: string } | null {
|
|
156
|
+
let url: URL;
|
|
157
|
+
try { url = new URL(value); } catch { return null; }
|
|
158
|
+
for (const spec of PROVIDERS) {
|
|
159
|
+
const token = spec.resolve(url);
|
|
160
|
+
if (token) return { ats: spec.ats, token, canonicalSourceUrl: spec.canonicalUrl(token), structuredEndpoint: spec.endpoint(token) };
|
|
161
|
+
}
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function validToken(value: string | undefined): string | null {
|
|
166
|
+
return value && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) ? value : null;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export function plainText(value: string): string {
|
|
170
|
+
return value
|
|
171
|
+
.replace(/<br\s*\/?\s*>/gi, "\n").replace(/<\/(p|li|div|h[1-6])>/gi, "\n").replace(/<[^>]+>/g, "")
|
|
172
|
+
.replace(/ /g, " ").replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'")
|
|
173
|
+
.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function majority(values: string[]): string {
|
|
177
|
+
const counts = new Map<string, number>();
|
|
178
|
+
for (const value of values) if (value) counts.set(value, (counts.get(value) ?? 0) + 1);
|
|
179
|
+
return [...counts].sort((a, b) => b[1] - a[1])[0]?.[0] ?? "";
|
|
180
|
+
}
|
|
181
|
+
function str(value: unknown): string { return typeof value === "string" ? value.trim() : typeof value === "number" ? String(value) : ""; }
|
|
182
|
+
function asRecords(value: unknown): Rec[] | null { return Array.isArray(value) && value.every(isRecord) ? value : null; }
|
|
183
|
+
function isRecord(value: unknown): value is Rec { return typeof value === "object" && value !== null && !Array.isArray(value); }
|
|
@@ -2,6 +2,8 @@ import type { Ats, RejectedSource, SourceCandidate, SourceRejectionReason, Sourc
|
|
|
2
2
|
import { fetchSafeHead, type HeadTransport, type ResolveHost } from "./safe-head.ts";
|
|
3
3
|
import { abortableDelay, fetchSourceJobs, isTransientStatus, retryDelayMs } from "./catalog.ts";
|
|
4
4
|
import { isEligibleForCountry } from "./locations.ts";
|
|
5
|
+
import { providerSpec, resolveProviderSource } from "./providers.ts";
|
|
6
|
+
import { ALL_PROVIDERS } from "./types.ts";
|
|
5
7
|
|
|
6
8
|
type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
|
|
7
9
|
|
|
@@ -29,7 +31,7 @@ export async function verifyCandidates(candidates: SourceCandidate[], options: V
|
|
|
29
31
|
const rejected: Array<{ index: number; value: RejectedSource }> = [];
|
|
30
32
|
const providerLimits = new Map<Ats, Semaphore>();
|
|
31
33
|
const providerCooldowns = new Map<Ats, ProviderCooldown>();
|
|
32
|
-
for (const ats of
|
|
34
|
+
for (const ats of ALL_PROVIDERS) {
|
|
33
35
|
const fallback = ats === "workday" ? 2 : concurrency;
|
|
34
36
|
providerLimits.set(ats, new Semaphore(Math.max(1, Math.trunc(options.providerConcurrency?.[ats] ?? fallback))));
|
|
35
37
|
providerCooldowns.set(ats, new ProviderCooldown());
|
|
@@ -162,19 +164,21 @@ async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher
|
|
|
162
164
|
const contentType = response.headers.get("content-type") ?? "unknown";
|
|
163
165
|
let body: unknown;
|
|
164
166
|
try { body = await response.json(); } catch { throw new VerificationError("invalid_payload", "Endpoint did not return JSON"); }
|
|
165
|
-
const
|
|
167
|
+
const spec = providerSpec(source.ats);
|
|
168
|
+
const jobs = spec ? spec.jobsFromBody(body) : source.ats === "greenhouse" ? recordArray(body, "jobs") : source.ats === "lever" ? array(body) : source.ats === "workday" ? recordArray(body, "jobPostings") : source.ats === "recruitee" ? recordArray(body, "offers") : recordArray(body, "jobs");
|
|
166
169
|
if (!jobs) throw new VerificationError("invalid_payload", "Payload does not contain the expected jobs array");
|
|
167
170
|
if (jobs.length === 0) throw new VerificationError("empty_board", "Source has no jobs, so identity cannot be verified");
|
|
168
|
-
const providerName = source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
|
|
169
|
-
const
|
|
170
|
-
const
|
|
171
|
-
|
|
171
|
+
const providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
|
|
172
|
+
const namedProvider = source.ats === "greenhouse" || source.ats === "workday" || Boolean(spec && providerName);
|
|
173
|
+
const hasDomainLink = !namedProvider && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
|
|
174
|
+
const hasRedirectEvidence = !namedProvider && await verifiedCompanyRedirect(candidate, source, resolveHost, headTransport, timeoutMs);
|
|
175
|
+
if (!namedProvider && !hasDomainLink && !hasRedirectEvidence) throw new VerificationError("identity_mismatch", `Neither structured identity fields nor a verified company redirect link to ${candidate.companyDomain}`);
|
|
172
176
|
const observedCompanyName = providerName || candidate.companyName;
|
|
173
177
|
return {
|
|
174
178
|
observedCompanyName,
|
|
175
179
|
identityEvidence: source.ats === "workday" ? "provider_tenant" as const : (providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : "company_redirect" as const),
|
|
176
180
|
contentType,
|
|
177
|
-
payloadVersion: source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
|
|
181
|
+
payloadVersion: spec ? spec.payloadVersion(body) : source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
|
|
178
182
|
jobCount: source.ats === "workday" && isRecord(body) && typeof body.total === "number" ? body.total : jobs.length,
|
|
179
183
|
};
|
|
180
184
|
} finally { clearTimeout(timer); }
|
|
@@ -192,6 +196,8 @@ async function fetchProbeWithRetry(fetcher: Fetch, endpoint: string, init: Reque
|
|
|
192
196
|
}
|
|
193
197
|
|
|
194
198
|
export function resolveSource(value: string): ResolvedSource | null {
|
|
199
|
+
const tableDriven = resolveProviderSource(value);
|
|
200
|
+
if (tableDriven) return tableDriven;
|
|
195
201
|
let url: URL;
|
|
196
202
|
try { url = new URL(value); } catch { return null; }
|
|
197
203
|
const parts = url.pathname.split("/").filter(Boolean);
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { promises as dns } from "node:dns";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import { atomicJson } from "./atomic-file.ts";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Passive careers-subdomain discovery. For each company domain it reads certificate transparency logs (no requests to the
|
|
7
|
+
* company) plus a short conventional list, keeps names that look like a careers host, confirms each resolves in DNS, and
|
|
8
|
+
* emits tracer seeds. The career tracer then makes the single safe HEAD request that turns a subdomain into a board.
|
|
9
|
+
* Never brute-forces names and never fetches a page.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
|
|
13
|
+
type Resolve = (hostname: string) => Promise<string[]>;
|
|
14
|
+
|
|
15
|
+
export interface SubdomainSeed { companyName: string; companyDomain: string }
|
|
16
|
+
export interface SubdomainDiscoveryReport {
|
|
17
|
+
generatedAt: string;
|
|
18
|
+
companies: number;
|
|
19
|
+
certificateNames: number;
|
|
20
|
+
candidates: number;
|
|
21
|
+
resolved: number;
|
|
22
|
+
seeds: Array<SubdomainSeed & { careerUrl: string; via: "certificate_transparency" | "conventional_name" }>;
|
|
23
|
+
failures: Array<{ companyDomain: string; reason: "ct_request_failed" | "invalid_domain"; detail: string }>;
|
|
24
|
+
outputPath: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const CAREER_WORDS = new Set(["careers", "career", "jobs", "job", "apply", "hiring", "talent", "talents", "recruit", "recruiting", "recruitment", "join", "joinus", "workwithus", "opportunities", "vacancies", "openings"]);
|
|
28
|
+
const CONVENTIONAL = ["careers", "jobs", "apply", "hiring", "talent", "recruit", "join"];
|
|
29
|
+
|
|
30
|
+
/** True when the subdomain part (everything before the company domain) reads like a careers host. */
|
|
31
|
+
export function looksLikeCareersHost(subdomain: string): boolean {
|
|
32
|
+
const words = subdomain.toLowerCase().split(/[.-]+/).filter(Boolean);
|
|
33
|
+
return words.some((word) => CAREER_WORDS.has(word)) || (words.includes("work") && words.includes("us"));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export async function discoverCareerSubdomains(seedsPath: string, outputPath: string, options: { fetch?: Fetch; resolve?: Resolve; now?: () => Date; limit?: number; delayMs?: number } = {}): Promise<SubdomainDiscoveryReport> {
|
|
37
|
+
const fetcher = options.fetch ?? globalThis.fetch;
|
|
38
|
+
const resolve = options.resolve ?? (async (hostname: string) => { try { return await dns.resolve(hostname, "A"); } catch { try { return await dns.resolve(hostname, "CNAME"); } catch { return []; } } });
|
|
39
|
+
const now = options.now ?? (() => new Date());
|
|
40
|
+
const delayMs = Math.max(0, options.delayMs ?? 1_000);
|
|
41
|
+
const seeds = (await readSeeds(seedsPath)).slice(0, options.limit ?? Number.POSITIVE_INFINITY);
|
|
42
|
+
const report: SubdomainDiscoveryReport = { generatedAt: now().toISOString(), companies: seeds.length, certificateNames: 0, candidates: 0, resolved: 0, seeds: [], failures: [], outputPath };
|
|
43
|
+
|
|
44
|
+
for (const [index, seed] of seeds.entries()) {
|
|
45
|
+
const domain = seed.companyDomain.toLowerCase().replace(/^www\./, "");
|
|
46
|
+
if (!/^[a-z0-9.-]+\.[a-z]{2,}$/.test(domain)) { report.failures.push({ companyDomain: seed.companyDomain, reason: "invalid_domain", detail: "companyDomain must be a hostname" }); continue; }
|
|
47
|
+
const candidates = new Map<string, "certificate_transparency" | "conventional_name">();
|
|
48
|
+
for (const name of CONVENTIONAL) candidates.set(`${name}.${domain}`, "conventional_name");
|
|
49
|
+
try {
|
|
50
|
+
if (index > 0 && delayMs) await new Promise((done) => setTimeout(done, delayMs));
|
|
51
|
+
const names = await certificateNames(domain, fetcher);
|
|
52
|
+
report.certificateNames += names.length;
|
|
53
|
+
for (const name of names) if (looksLikeCareersHost(name.slice(0, -domain.length - 1))) candidates.set(name, candidates.get(name) ?? "certificate_transparency");
|
|
54
|
+
} catch (error) {
|
|
55
|
+
report.failures.push({ companyDomain: domain, reason: "ct_request_failed", detail: error instanceof Error ? error.message : String(error) });
|
|
56
|
+
}
|
|
57
|
+
report.candidates += candidates.size;
|
|
58
|
+
for (const [host, via] of candidates) {
|
|
59
|
+
if ((await resolve(host)).length === 0) continue;
|
|
60
|
+
report.resolved += 1;
|
|
61
|
+
report.seeds.push({ companyName: seed.companyName, companyDomain: domain, careerUrl: `https://${host}/`, via });
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
await atomicJson(outputPath, report.seeds.map(({ companyName, companyDomain, careerUrl }) => ({ companyName, companyDomain, careerUrl })));
|
|
65
|
+
return report;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Distinct hostnames under `domain` that appear in certificate transparency logs. One request per company. */
|
|
69
|
+
export async function certificateNames(domain: string, fetcher: Fetch): Promise<string[]> {
|
|
70
|
+
const response = await fetcher(`https://crt.sh/?q=${encodeURIComponent(`%.${domain}`)}&output=json`, { headers: { "user-agent": "openings-discovery/0.1 (+https://avagama.co/openings/)" }, signal: AbortSignal.timeout(60_000) });
|
|
71
|
+
if (!response.ok) throw new Error(`crt.sh returned HTTP ${response.status}`);
|
|
72
|
+
const rows = await response.json() as Array<{ name_value?: string }>;
|
|
73
|
+
const names = new Set<string>();
|
|
74
|
+
for (const row of rows) for (const raw of String(row.name_value ?? "").split("\n")) {
|
|
75
|
+
const name = raw.trim().toLowerCase().replace(/^\*\./, "");
|
|
76
|
+
if (name.endsWith(`.${domain}`) && /^[a-z0-9.-]+$/.test(name)) names.add(name);
|
|
77
|
+
}
|
|
78
|
+
return [...names].sort();
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function readSeeds(path: string): Promise<SubdomainSeed[]> {
|
|
82
|
+
const value = JSON.parse(await readFile(path, "utf8")) as unknown;
|
|
83
|
+
if (!Array.isArray(value) || !value.every((row) => typeof row === "object" && row !== null && typeof (row as SubdomainSeed).companyName === "string" && typeof (row as SubdomainSeed).companyDomain === "string")) throw new Error("Seed file must be a JSON array of { companyName, companyDomain }");
|
|
84
|
+
return value as SubdomainSeed[];
|
|
85
|
+
}
|
package/src/types.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
export
|
|
1
|
+
export const ALL_PROVIDERS = ["greenhouse", "lever", "ashby", "workday", "recruitee", "smartrecruiters", "workable", "breezy"] as const;
|
|
2
|
+
export type Ats = (typeof ALL_PROVIDERS)[number];
|
|
2
3
|
|
|
3
4
|
export interface DomainEvidence {
|
|
4
5
|
kind: "authoritative_dataset" | "company_registry" | "company_redirect";
|