openings 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "openings",
3
- "version": "0.1.3",
3
+ "version": "0.1.4",
4
4
  "description": "A free, candidate-safe job-search substrate for AI agents",
5
5
  "license": "MIT",
6
6
  "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },
7
7
  "homepage": "https://github.com/abhay-avagama/hiring-agent#readme",
8
8
  "bugs": { "url": "https://github.com/abhay-avagama/hiring-agent/issues" },
9
- "keywords": ["jobs", "resume", "matching", "mcp", "greenhouse", "lever", "ashby", "workday", "recruitee"],
9
+ "keywords": ["jobs", "resume", "matching", "mcp", "greenhouse", "lever", "ashby", "workday", "recruitee", "smartrecruiters", "workable", "breezy"],
10
10
  "type": "module",
11
11
  "bin": { "openings-mcp": "./src/package-mcp.ts" },
12
12
  "exports": "./src/index.ts",
@@ -1,13 +1,14 @@
1
1
  import { atomicJson } from "./atomic-file.ts";
2
2
  import { readEnrichmentRegistry, mergeEnrichmentLeads, type EnrichmentLead, type LeadAttempt } from "./enrichment-registry.ts";
3
3
  import { withFileLock } from "./file-lock.ts";
4
+ import { providerSpec } from "./providers.ts";
4
5
  import { resolveSource } from "./source-verification.ts";
5
6
  import type { Ats, Company, SourceVerification } from "./types.ts";
6
7
 
7
8
  type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
8
9
 
9
10
  /** Providers whose public board is accepted as identity on its own. Workday stays on the company-identity path because its crawls are expensive. */
10
- export const BOARD_TIER_PROVIDERS: ReadonlySet<Ats> = new Set(["greenhouse", "lever", "ashby", "recruitee"]);
11
+ export const BOARD_TIER_PROVIDERS: ReadonlySet<Ats> = new Set(["greenhouse", "lever", "ashby", "recruitee", "smartrecruiters", "workable", "breezy"]);
11
12
 
12
13
  export interface BoardVerificationOptions {
13
14
  fetch?: Fetch;
@@ -127,11 +128,12 @@ async function probeBoard(lead: EnrichmentLead, fetcher: Fetch, timeoutMs: numbe
127
128
  if (!response.ok) { await response.body?.cancel().catch(() => undefined); throw new BoardError(response.status === 404 || response.status === 410 ? "invalid_payload" : "unreachable", `HTTP ${response.status}`); }
128
129
  let body: unknown;
129
130
  try { body = await response.json(); } catch { throw new BoardError("invalid_payload", "Endpoint did not return JSON"); }
130
- const jobs = source.ats === "lever" ? asArray(body) : asArray(isRecord(body) ? body[source.ats === "recruitee" ? "offers" : "jobs"] : undefined);
131
+ const spec = providerSpec(source.ats);
132
+ const jobs = spec ? spec.jobsFromBody(body) : source.ats === "lever" ? asArray(body) : asArray(isRecord(body) ? body[source.ats === "recruitee" ? "offers" : "jobs"] : undefined);
131
133
  if (!jobs) throw new BoardError("invalid_payload", "Payload does not contain the expected jobs array");
132
134
  if (jobs.length === 0) throw new BoardError("empty_board", "Board has no jobs");
133
- const providerName = source.ats === "greenhouse" ? majority(jobs.map((job) => typeof job.company_name === "string" ? job.company_name.trim() : "").filter(Boolean)) : "";
134
- const payloadVersion = source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`;
135
+ const providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => typeof job.company_name === "string" ? job.company_name.trim() : "").filter(Boolean)) : "";
136
+ const payloadVersion = spec ? spec.payloadVersion(body) : source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`;
135
137
  return { providerName, contentType: response.headers.get("content-type") ?? "unknown", payloadVersion, jobCount: jobs.length };
136
138
  } finally { clearTimeout(timer); }
137
139
  }
package/src/catalog.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import type { Company, Job, JobSummary, SearchQuery } from "./types.ts";
2
+ import { providerSpec, type JsonGet } from "./providers.ts";
2
3
  import { classifyJob, isEligibleForCountry, normalizeLocation } from "./locations.ts";
3
4
 
4
5
  type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
@@ -100,6 +101,8 @@ export function createCatalog(options: CatalogOptions): Catalog {
100
101
  const description = await fetchWorkdayDescription(company, job.url, fetcher);
101
102
  job = { ...job, description };
102
103
  }
104
+ const spec = job && !job.description.trim() ? providerSpec(company.ats) : undefined;
105
+ if (job && spec?.detail) job = { ...job, description: await spec.detail(company, job, jsonGetter(fetcher, company.name)) };
103
106
  if (job && !job.description.trim()) throw new Error(`Full description unavailable for job: ${id}`);
104
107
  return job;
105
108
  },
@@ -118,6 +121,13 @@ export interface FetchJobsObserver {
118
121
  }
119
122
  export async function fetchSourceJobs(company: Company, fetcher: Fetch = globalThis.fetch, signal?: AbortSignal, observer?: FetchJobsObserver): Promise<Job[]> {
120
123
  if (company.ats === "workday") return fetchWorkdayJobs(company, fetcher, signal, observer);
124
+ const spec = providerSpec(company.ats);
125
+ if (spec) {
126
+ const get = jsonGetter(fetcher, company.name, signal, observer);
127
+ const records = spec.fetchAll ? await spec.fetchAll(company.token, get) : spec.jobsFromBody(await get(spec.endpoint(company.token)));
128
+ if (!records) throw new Error(`${company.name} job board returned an invalid payload`);
129
+ return records.map((record) => spec.normalize(company, record));
130
+ }
121
131
  const url = company.ats === "greenhouse"
122
132
  ? `https://boards-api.greenhouse.io/v1/boards/${encodeURIComponent(company.token)}/jobs?content=true`
123
133
  : company.ats === "lever"
@@ -196,6 +206,15 @@ async function fetchWorkdayJobs(company: Company, fetcher: Fetch, signal?: Abort
196
206
  return jobs.map((job) => normalizeWorkday(company, source, job));
197
207
  }
198
208
 
209
+ /** JSON fetch with the catalog's retry and backoff, shaped for the table-driven providers. */
210
+ function jsonGetter(fetcher: Fetch, companyName: string, signal?: AbortSignal, observer?: FetchJobsObserver): JsonGet {
211
+ return async (url) => {
212
+ const response = await fetchWithRetry(fetcher, url, signal ? { signal } : undefined, companyName, observer);
213
+ if (!response.ok) { await response.body?.cancel().catch(() => undefined); throw new Error(`${companyName} job board returned HTTP ${response.status}`); }
214
+ return response.json();
215
+ };
216
+ }
217
+
199
218
  async function fetchWithRetry(fetcher: Fetch, input: string | URL, init: RequestInit | undefined, companyName: string, observer?: FetchJobsObserver): Promise<Response> {
200
219
  for (let attempt = 0; attempt < 3; attempt += 1) {
201
220
  const response = await fetcher(input, init);
package/src/cli.ts CHANGED
@@ -1,9 +1,11 @@
1
1
  #!/usr/bin/env bun
2
+ import { ALL_PROVIDERS, type Ats } from "./types.ts";
2
3
  import { readFile } from "node:fs/promises";
3
4
  import { relative, resolve } from "node:path";
4
5
  import { createRuntime } from "./runtime.ts";
5
6
  import { atomicJson } from "./atomic-file.ts";
6
7
  import { verifyBoards } from "./board-verification.ts";
8
+ import { discoverCareerSubdomains } from "./subdomain-discovery.ts";
7
9
  import { runSourceVerification } from "./source-pipeline.ts";
8
10
  import { runSourceDiscovery, runYcSourceDiscovery } from "./source-discovery.ts";
9
11
  import { discoverAndPromote } from "./source-discovery-pipeline.ts";
@@ -23,12 +25,13 @@ Usage:
23
25
  openings crawl [--country CODE | --companies FILE] [--concurrency N] [--source-cache-hours N] [--source-limit N] [--delay-ms N] [--workday-page-delay-ms N] [--data-dir PATH]
24
26
  openings snapshot export [--input FILE] [--output-dir PATH]
25
27
  openings coverage report --country CODE [--snapshot FILE] [--catalog FILE] [--candidates FILE] [--registry FILE] [--output FILE] [--as-of ISO]
28
+ openings sources discover-subdomains SEEDS.json [--output FILE] [--limit N] [--delay-ms N] [--report FILE]
26
29
  openings sources verify-boards REGISTRY.json [--output FILE] [--limit N] [--concurrency N] [--report FILE]
27
30
  openings sources verify CANDIDATES.json [--output FILE] [--state-file FILE] [--concurrency N] [--workday-concurrency N] [--limit N] [--require-country CODE] [--registry FILE] [--retry-deferred]
28
31
  openings sources discover FEED.json [--country CODE] [--registry FILE] [--output FILE] [--catalog FILE] [--report FILE]
29
32
  openings sources discover-yc --country CODE [--registry FILE] [--output FILE] [--catalog FILE] [--report FILE]
30
33
  openings sources seed-companies-yc --country CODE [--output FILE]
31
- openings sources discover-common-crawl [--country CODE] [--provider recruitee] [--index-record-limit N] [--sample-token-limit N] [--exclude-token TOKEN] [--report-only] [--registry FILE] [--output FILE] [--report FILE] [--index-url URL]
34
+ openings sources discover-common-crawl [--country CODE] [--provider NAME] [--index-record-limit N] [--sample-token-limit N] [--exclude-token TOKEN] [--report-only] [--registry FILE] [--output FILE] [--report FILE] [--index-url URL]
32
35
  openings sources enrich COMPANIES.json [--companies FILE]... [--evidence-kind authoritative_dataset|company_registry] [--registry FILE] [--output FILE] [--report FILE]
33
36
  openings sources trace-careers COMPANIES.json [--country CODE] [--registry FILE] [--common-crawl-report FILE] [--search-key-env NAME] [--output FILE] [--catalog FILE] [--report FILE]
34
37
  openings sources probe-jobposting COMPANIES.md [--catalog FILE] [--report FILE] [--company-limit 10|20]
@@ -159,6 +162,14 @@ export async function run(args: string[]): Promise<number> {
159
162
  )), null, 2));
160
163
  return 0;
161
164
  }
165
+ if (rest[0] === "discover-subdomains") {
166
+ const parsed = parseDiscoverSubdomains(rest.slice(1));
167
+ if (typeof parsed === "string") return fail(parsed);
168
+ const report = await discoverCareerSubdomains(parsed.seeds, parsed.output, { limit: parsed.limit, delayMs: parsed.delayMs });
169
+ if (parsed.report) await atomicJson(parsed.report, report);
170
+ console.log(JSON.stringify({ ...report, seeds: report.seeds.length }, null, 2));
171
+ return 0;
172
+ }
162
173
  if (rest[0] === "verify-boards") {
163
174
  const parsed = parseVerifyBoards(rest.slice(1));
164
175
  if (typeof parsed === "string") return fail(parsed);
@@ -318,7 +329,7 @@ function parseCommonCrawlDiscovery(args: string[]) {
318
329
  let country: string | undefined;
319
330
  let indexUrl: string | undefined;
320
331
  let registryPath: string | undefined = "data/enrichment-leads.json";
321
- let provider: "recruitee" | undefined;
332
+ let provider: Ats | undefined;
322
333
  let indexRecordLimit: number | undefined;
323
334
  let sampleTokenLimit: number | undefined;
324
335
  const excludeTokens: string[] = [];
@@ -331,7 +342,7 @@ function parseCommonCrawlDiscovery(args: string[]) {
331
342
  else if (arg === "--index-url") { indexUrl = args[++index]; if (!indexUrl) return "--index-url requires a URL"; try { new URL(indexUrl); } catch { return "--index-url requires a valid URL"; } }
332
343
  else if (arg === "--registry") { if (reportOnly) return "Use either --registry or --report-only, not both"; registryPath = args[++index] ?? ""; if (!registryPath) return "--registry requires a file"; }
333
344
  else if (arg === "--report-only") { if (registryPath !== "data/enrichment-leads.json") return "Use either --registry or --report-only, not both"; reportOnly = true; registryPath = undefined; }
334
- else if (arg === "--provider") { const value = args[++index]; if (value !== "recruitee") return "--provider currently supports only recruitee"; provider = value; }
345
+ else if (arg === "--provider") { const value = args[++index]; if (!value || !(ALL_PROVIDERS as readonly string[]).includes(value)) return `--provider must be one of ${ALL_PROVIDERS.join(", ")}`; provider = value as Ats; }
335
346
  else if (arg === "--index-record-limit") { indexRecordLimit = Number(args[++index]); if (!Number.isInteger(indexRecordLimit) || indexRecordLimit < 1 || indexRecordLimit > 2_000) return "--index-record-limit must be an integer from 1 to 2000"; }
336
347
  else if (arg === "--sample-token-limit") { sampleTokenLimit = Number(args[++index]); if (!Number.isInteger(sampleTokenLimit) || sampleTokenLimit < 1 || sampleTokenLimit > 60) return "--sample-token-limit must be an integer from 1 to 60"; }
337
348
  else if (arg === "--exclude-token") { const value = args[++index]?.trim(); if (!value || !/^[a-z0-9-]+$/i.test(value)) return "--exclude-token requires an ATS token"; excludeTokens.push(value.toLowerCase()); }
@@ -544,6 +555,24 @@ async function readCompanyFile(path: string): Promise<string[]> {
544
555
  return (await readFile(path, "utf8")).split(/\r?\n/).map((line) => line.trim()).filter((line) => line && !line.startsWith("#"));
545
556
  }
546
557
 
558
+ function parseDiscoverSubdomains(args: string[]): { seeds: string; output: string; limit?: number; delayMs?: number; report?: string } | string {
559
+ const seeds = args[0];
560
+ if (!seeds || seeds.startsWith("--")) return "sources discover-subdomains requires a seed JSON file of { companyName, companyDomain }";
561
+ let output = ".openings/career-seeds-subdomains.json";
562
+ let limit: number | undefined;
563
+ let delayMs: number | undefined;
564
+ let report: string | undefined;
565
+ for (let index = 1; index < args.length; index += 1) {
566
+ const arg = args[index];
567
+ if (arg === "--output") output = args[++index] ?? output;
568
+ else if (arg === "--report") report = args[++index];
569
+ else if (arg === "--limit") { limit = Number(args[++index]); if (!Number.isInteger(limit) || limit < 1) return "--limit must be a positive integer"; }
570
+ else if (arg === "--delay-ms") { delayMs = Number(args[++index]); if (!Number.isInteger(delayMs) || delayMs < 0) return "--delay-ms must be a non-negative integer"; }
571
+ else return `Unknown option: ${arg}`;
572
+ }
573
+ return { seeds, output, limit, delayMs, report };
574
+ }
575
+
547
576
  function parseVerifyBoards(args: string[]): { registry: string; output: string; limit?: number; concurrency?: number; report?: string } | string {
548
577
  const registry = args[0];
549
578
  if (!registry || registry.startsWith("--")) return "sources verify-boards requires a registry JSON file";
@@ -2,6 +2,7 @@ import { readFile } from "node:fs/promises";
2
2
  import { atomicJson } from "./atomic-file.ts";
3
3
  import { stampReport, type ReportMeta } from "./report-meta.ts";
4
4
  import { mergeEnrichmentLeads, type EnrichmentLead } from "./enrichment-registry.ts";
5
+ import { PROVIDERS } from "./providers.ts";
5
6
  import { resolveSource } from "./source-verification.ts";
6
7
  import type { Ats } from "./types.ts";
7
8
  import { assertArtifactFile } from "./artifact-path.ts";
@@ -36,11 +37,12 @@ export interface CommonCrawlDiscoveryReport extends ReportMeta {
36
37
  rejections: Array<{ value: string; reason: string }>;
37
38
  }
38
39
 
39
- const patterns = ["job-boards.greenhouse.io/*", "boards.greenhouse.io/*", "jobs.lever.co/*", "jobs.ashbyhq.com/*", "*.myworkdayjobs.com/*", "*.recruitee.com/*"];
40
- const recordsPerPattern = 10_000;
41
40
  const providerPatterns: Record<Ats, string[]> = {
42
- greenhouse: patterns.slice(0, 2), lever: [patterns[2]!], ashby: [patterns[3]!], workday: [patterns[4]!], recruitee: [patterns[5]!],
41
+ greenhouse: ["job-boards.greenhouse.io/*", "boards.greenhouse.io/*"], lever: ["jobs.lever.co/*"], ashby: ["jobs.ashbyhq.com/*"], workday: ["*.myworkdayjobs.com/*"], recruitee: ["*.recruitee.com/*"],
42
+ ...Object.fromEntries(PROVIDERS.map((spec) => [spec.ats, spec.crawlPatterns])) as Record<"smartrecruiters" | "workable" | "breezy", string[]>,
43
43
  };
44
+ const patterns = Object.values(providerPatterns).flat();
45
+ const recordsPerPattern = 10_000;
44
46
 
45
47
  export async function discoverCommonCrawlSources(candidatesPath: string, reportPath: string, options: Options = {}): Promise<CommonCrawlDiscoveryReport> {
46
48
  if (options.provider === "recruitee") await assertArtifactFile(reportPath);
@@ -4,6 +4,7 @@ import { deriveLeadState, readEnrichmentRegistry, type EnrichmentState } from ".
4
4
  import { isEligibleForCountry } from "./locations.ts";
5
5
  import { stampReport, type ReportMeta } from "./report-meta.ts";
6
6
  import { resolveSource } from "./source-verification.ts";
7
+ import { ALL_PROVIDERS } from "./types.ts";
7
8
  import type { Ats, EligibilityConfidence, JobSnapshot, SourceCandidate, VerifiedCompany } from "./types.ts";
8
9
  import { projectJobCoverage } from "./job-coverage.ts";
9
10
 
@@ -110,7 +111,7 @@ async function readCatalog(path: string): Promise<Record<string, Omit<VerifiedCo
110
111
  const value: unknown = JSON.parse(await readFile(path, "utf8"));
111
112
  if (!isRecord(value)) throw new Error(`Invalid verified catalog: ${path}`);
112
113
  for (const [slug, company] of Object.entries(value)) {
113
- if (!isRecord(company) || !["greenhouse", "lever", "ashby", "workday", "recruitee"].includes(String(company.ats)) || typeof company.token !== "string"
114
+ if (!isRecord(company) || !(ALL_PROVIDERS as readonly string[]).includes(String(company.ats)) || typeof company.token !== "string"
114
115
  || (typeof company.companyDomain !== "string" && !(isRecord(company.verification) && company.verification.identityEvidence === "provider_board")) || typeof company.sourceUrl !== "string"
115
116
  || company.cohorts !== undefined && (!Array.isArray(company.cohorts) || !company.cohorts.every(validCountryCode))
116
117
  || !validVerification(company.verification)) throw new Error(`Invalid verified catalog source: ${slug}`);
@@ -1,6 +1,7 @@
1
1
  import { readFile } from "node:fs/promises";
2
2
  import { atomicJson } from "./atomic-file.ts";
3
3
  import { withFileLock } from "./file-lock.ts";
4
+ import { ALL_PROVIDERS } from "./types.ts";
4
5
  import type { Ats, DiscoveryProvenance, DomainEvidence } from "./types.ts";
5
6
  import { resolveSource } from "./source-verification.ts";
6
7
 
@@ -110,9 +111,9 @@ function consecutiveTransientFailures(attempts: LeadAttempt[]): number { let cou
110
111
  function isRegistry(value: unknown): value is EnrichmentRegistry {
111
112
  if (!isRecord(value) || value.version !== 1 || typeof value.updatedAt !== "string" || !Array.isArray(value.leads)) return false;
112
113
  return value.leads.every((lead) => isRecord(lead)
113
- && typeof lead.sourceKey === "string" && /^(greenhouse|lever|ashby|workday|recruitee):.+/.test(lead.sourceKey)
114
+ && typeof lead.sourceKey === "string" && new RegExp(`^(${ALL_PROVIDERS.join("|")}):.+`).test(lead.sourceKey)
114
115
  && typeof lead.sourceUrl === "string" && typeof lead.token === "string"
115
- && ["greenhouse", "lever", "ashby", "workday", "recruitee"].includes(String(lead.ats))
116
+ && (ALL_PROVIDERS as readonly string[]).includes(String(lead.ats))
116
117
  && Array.isArray(lead.discoveredFrom) && lead.discoveredFrom.every(validProvenance)
117
118
  && Array.isArray(lead.companyMatches) && lead.companyMatches.every(validMatch)
118
119
  && Array.isArray(lead.identityEvidence) && lead.identityEvidence.every(validEvidence)
package/src/mcp.ts CHANGED
@@ -22,7 +22,7 @@ export function createMcpHandler(tools: ToolHandler) {
22
22
  const base = { jsonrpc: "2.0" as const, id: request.id ?? null };
23
23
  try {
24
24
  if (request.method === "initialize") {
25
- return { ...base, result: { protocolVersion: "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: "openings", version: "0.1.3" } } };
25
+ return { ...base, result: { protocolVersion: "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: "openings", version: "0.1.4" } } };
26
26
  }
27
27
  if (request.method === "ping") return { ...base, result: {} };
28
28
  if (request.method === "tools/list") return { ...base, result: { tools: tools.list() } };
@@ -0,0 +1,183 @@
1
+ import { classifyJob } from "./locations.ts";
2
+ import type { Ats, Company, Job } from "./types.ts";
3
+
4
+ /**
5
+ * Table-driven adapters for providers added after the original five. Each entry knows how to recognise a board URL,
6
+ * where its public structured endpoint lives, how to read the payload, and how to turn one record into a Job.
7
+ * The original providers keep their hand-written paths; new ones only need a row here.
8
+ */
9
+
10
+ type Rec = Record<string, unknown>;
11
+ /** Fetches a URL and returns its parsed JSON body; the caller supplies retry and pacing. */
12
+ export type JsonGet = (url: string) => Promise<unknown>;
13
+
14
+ export interface ProviderSpec {
15
+ ats: Ats;
16
+ label: string;
17
+ /** Registrable host suffixes on which this provider serves boards and job pages. */
18
+ hosts: string[];
19
+ /** Common Crawl URL patterns that surface this provider's boards. */
20
+ crawlPatterns: string[];
21
+ resolve(url: URL): string | null;
22
+ canonicalUrl(token: string): string;
23
+ endpoint(token: string): string;
24
+ jobsFromBody(body: unknown): Rec[] | null;
25
+ providerName(jobs: Rec[], body: unknown): string;
26
+ payloadVersion(body: unknown): string;
27
+ normalize(company: Company, job: Rec): Job;
28
+ /** Fetches every record when the endpoint paginates. Defaults to one request to `endpoint`. */
29
+ fetchAll?(token: string, get: JsonGet): Promise<Rec[]>;
30
+ /** Fetches the full description for one job when the listing omits it. */
31
+ detail?(company: Company, job: Job, get: JsonGet): Promise<string>;
32
+ }
33
+
34
+ const PAGE = 100;
35
+ const MAX_RECORDS = 2000;
36
+
37
+ const smartrecruiters: ProviderSpec = {
38
+ ats: "smartrecruiters",
39
+ label: "SmartRecruiters",
40
+ hosts: ["smartrecruiters.com"],
41
+ crawlPatterns: ["jobs.smartrecruiters.com/*", "careers.smartrecruiters.com/*"],
42
+ resolve(url) {
43
+ const parts = url.pathname.split("/").filter(Boolean);
44
+ if (url.hostname === "jobs.smartrecruiters.com" || url.hostname === "careers.smartrecruiters.com") return validToken(parts[0]);
45
+ if (url.hostname === "api.smartrecruiters.com" && parts[0] === "v1" && parts[1] === "companies") return validToken(parts[2]);
46
+ return null;
47
+ },
48
+ canonicalUrl: (token) => `https://jobs.smartrecruiters.com/${token}`,
49
+ endpoint: (token) => `https://api.smartrecruiters.com/v1/companies/${encodeURIComponent(token)}/postings?limit=${PAGE}&offset=0`,
50
+ jobsFromBody: (body) => (isRecord(body) ? asRecords(body.content) : null),
51
+ providerName: (jobs) => majority(jobs.map((job) => (isRecord(job.company) ? str(job.company.name) : ""))),
52
+ payloadVersion: () => "smartrecruiters-postings:v1",
53
+ async fetchAll(token, get) {
54
+ const records: Rec[] = [];
55
+ for (let offset = 0; offset < MAX_RECORDS; offset += PAGE) {
56
+ const body = await get(`https://api.smartrecruiters.com/v1/companies/${encodeURIComponent(token)}/postings?limit=${PAGE}&offset=${offset}`);
57
+ const page = smartrecruiters.jobsFromBody(body);
58
+ if (!page) throw new Error("SmartRecruiters returned an invalid postings payload");
59
+ records.push(...page);
60
+ const total = isRecord(body) && typeof body.totalFound === "number" ? body.totalFound : records.length;
61
+ if (page.length < PAGE || records.length >= total) break;
62
+ }
63
+ return records;
64
+ },
65
+ normalize(company, job) {
66
+ const loc = isRecord(job.location) ? job.location : {};
67
+ const country = str(loc.country);
68
+ const location = str(loc.fullLocation) || [str(loc.city), str(loc.region), country.length === 2 ? country.toUpperCase() : country].filter(Boolean).join(", ") || "Unspecified";
69
+ const remote = loc.remote === true;
70
+ return classifyJob({
71
+ id: `smartrecruiters:${company.slug}:${str(job.id)}`, company: company.name, title: str(job.name), location,
72
+ remote, workMode: remote ? "remote" : loc.hybrid === true ? "hybrid" : "unknown",
73
+ eligibleCountries: [], excludedCountries: [], eligibleRegions: [], eligibilityConfidence: "unknown",
74
+ url: `https://jobs.smartrecruiters.com/${company.token}/${encodeURIComponent(str(job.id))}`, updatedAt: str(job.releasedDate) || undefined, description: "",
75
+ });
76
+ },
77
+ async detail(company, job, get) {
78
+ const id = job.id.split(":")[2] ?? "";
79
+ const body = await get(`https://api.smartrecruiters.com/v1/companies/${encodeURIComponent(company.token)}/postings/${encodeURIComponent(id)}`);
80
+ const sections = isRecord(body) && isRecord(body.jobAd) && isRecord(body.jobAd.sections) ? body.jobAd.sections : {};
81
+ return ["jobDescription", "qualifications", "additionalInformation"].map((key) => { const section = sections[key]; return isRecord(section) ? plainText(str(section.text)) : ""; }).filter(Boolean).join("\n\n");
82
+ },
83
+ };
84
+
85
+ const workable: ProviderSpec = {
86
+ ats: "workable",
87
+ label: "Workable",
88
+ hosts: ["workable.com"],
89
+ crawlPatterns: ["apply.workable.com/*"],
90
+ resolve(url) {
91
+ const parts = url.pathname.split("/").filter(Boolean);
92
+ if (url.hostname === "apply.workable.com") return parts[0] && !["j", "api", "jobs", "embed", "assets"].includes(parts[0]) ? validToken(parts[0]) : null;
93
+ const match = /^([a-z0-9-]+)\.workable\.com$/i.exec(url.hostname);
94
+ if (match && !["www", "apply", "help", "resources", "jobs", "careers", "api", "app", "assets", "cdn", "status", "developers", "blog", "partners"].includes(match[1]!.toLowerCase())) return match[1]!.toLowerCase();
95
+ return null;
96
+ },
97
+ canonicalUrl: (token) => `https://apply.workable.com/${token}/`,
98
+ endpoint: (token) => `https://apply.workable.com/api/v1/widget/accounts/${encodeURIComponent(token)}`,
99
+ jobsFromBody: (body) => (isRecord(body) ? asRecords(body.jobs) : null),
100
+ providerName: (_jobs, body) => (isRecord(body) ? str(body.name) : ""),
101
+ payloadVersion: () => "workable-widget:v1",
102
+ normalize(company, job) {
103
+ const first = asRecords(job.locations)?.[0];
104
+ const location = [str(job.city), str(job.state), str(job.country) || (first ? str(first.country) : "")].filter(Boolean).join(", ") || "Unspecified";
105
+ const remote = job.telecommuting === true;
106
+ return classifyJob({
107
+ id: `workable:${company.slug}:${str(job.shortcode)}`, company: company.name, title: str(job.title), location,
108
+ remote, workMode: remote ? "remote" : "unknown",
109
+ eligibleCountries: [], excludedCountries: [], eligibleRegions: [], eligibilityConfidence: "unknown",
110
+ url: str(job.url) || `https://apply.workable.com/j/${encodeURIComponent(str(job.shortcode))}`, updatedAt: str(job.published_on) || undefined, description: "",
111
+ });
112
+ },
113
+ async detail(company, job, get) {
114
+ const shortcode = job.id.split(":")[2] ?? "";
115
+ const body = await get(`https://apply.workable.com/api/v2/accounts/${encodeURIComponent(company.token)}/jobs/${encodeURIComponent(shortcode)}`);
116
+ if (!isRecord(body)) return "";
117
+ return ["description", "requirements", "benefits"].map((key) => plainText(str(body[key]))).filter(Boolean).join("\n\n");
118
+ },
119
+ };
120
+
121
+ const breezy: ProviderSpec = {
122
+ ats: "breezy",
123
+ label: "Breezy",
124
+ hosts: ["breezy.hr"],
125
+ crawlPatterns: ["*.breezy.hr/*"],
126
+ resolve(url) {
127
+ const match = /^([a-z0-9-]+)\.breezy\.hr$/i.exec(url.hostname);
128
+ return match && !["www", "app", "api", "help", "blog"].includes(match[1]!.toLowerCase()) ? match[1]!.toLowerCase() : null;
129
+ },
130
+ canonicalUrl: (token) => `https://${token}.breezy.hr`,
131
+ endpoint: (token) => `https://${token}.breezy.hr/json?verbose=true`,
132
+ jobsFromBody: (body) => asRecords(body),
133
+ providerName: (jobs) => majority(jobs.map((job) => (isRecord(job.company) ? str(job.company.name) : ""))),
134
+ payloadVersion: () => "breezy-positions:v1",
135
+ normalize(company, job) {
136
+ const loc = isRecord(job.location) ? job.location : {};
137
+ const location = str(loc.name) || [str(loc.city), isRecord(loc.state) ? str(loc.state.name) : "", isRecord(loc.country) ? str(loc.country.name) : ""].filter(Boolean).join(", ") || "Unspecified";
138
+ const remote = loc.is_remote === true;
139
+ return classifyJob({
140
+ id: `breezy:${company.slug}:${str(job.id)}`, company: company.name, title: str(job.name), location,
141
+ remote, workMode: remote ? "remote" : "unknown",
142
+ eligibleCountries: [], excludedCountries: [], eligibleRegions: [], eligibilityConfidence: "unknown",
143
+ url: str(job.url) || `https://${company.token}.breezy.hr/p/${encodeURIComponent(str(job.friendly_id) || str(job.id))}`, updatedAt: str(job.published_date) || undefined, description: plainText(str(job.description)),
144
+ });
145
+ },
146
+ };
147
+
148
+ export const PROVIDERS: ReadonlyArray<ProviderSpec> = [smartrecruiters, workable, breezy];
149
+
150
+ export function providerSpec(ats: string): ProviderSpec | undefined {
151
+ return PROVIDERS.find((spec) => spec.ats === ats);
152
+ }
153
+
154
+ /** Resolves a URL against the table-driven providers; returns null when none claims it. */
155
+ export function resolveProviderSource(value: string): { ats: Ats; token: string; canonicalSourceUrl: string; structuredEndpoint: string } | null {
156
+ let url: URL;
157
+ try { url = new URL(value); } catch { return null; }
158
+ for (const spec of PROVIDERS) {
159
+ const token = spec.resolve(url);
160
+ if (token) return { ats: spec.ats, token, canonicalSourceUrl: spec.canonicalUrl(token), structuredEndpoint: spec.endpoint(token) };
161
+ }
162
+ return null;
163
+ }
164
+
165
+ function validToken(value: string | undefined): string | null {
166
+ return value && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) ? value : null;
167
+ }
168
+
169
+ export function plainText(value: string): string {
170
+ return value
171
+ .replace(/<br\s*\/?\s*>/gi, "\n").replace(/<\/(p|li|div|h[1-6])>/gi, "\n").replace(/<[^>]+>/g, "")
172
+ .replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'")
173
+ .replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
174
+ }
175
+
176
+ function majority(values: string[]): string {
177
+ const counts = new Map<string, number>();
178
+ for (const value of values) if (value) counts.set(value, (counts.get(value) ?? 0) + 1);
179
+ return [...counts].sort((a, b) => b[1] - a[1])[0]?.[0] ?? "";
180
+ }
181
+ function str(value: unknown): string { return typeof value === "string" ? value.trim() : typeof value === "number" ? String(value) : ""; }
182
+ function asRecords(value: unknown): Rec[] | null { return Array.isArray(value) && value.every(isRecord) ? value : null; }
183
+ function isRecord(value: unknown): value is Rec { return typeof value === "object" && value !== null && !Array.isArray(value); }
@@ -2,6 +2,8 @@ import type { Ats, RejectedSource, SourceCandidate, SourceRejectionReason, Sourc
2
2
  import { fetchSafeHead, type HeadTransport, type ResolveHost } from "./safe-head.ts";
3
3
  import { abortableDelay, fetchSourceJobs, isTransientStatus, retryDelayMs } from "./catalog.ts";
4
4
  import { isEligibleForCountry } from "./locations.ts";
5
+ import { providerSpec, resolveProviderSource } from "./providers.ts";
6
+ import { ALL_PROVIDERS } from "./types.ts";
5
7
 
6
8
  type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
7
9
 
@@ -29,7 +31,7 @@ export async function verifyCandidates(candidates: SourceCandidate[], options: V
29
31
  const rejected: Array<{ index: number; value: RejectedSource }> = [];
30
32
  const providerLimits = new Map<Ats, Semaphore>();
31
33
  const providerCooldowns = new Map<Ats, ProviderCooldown>();
32
- for (const ats of ["greenhouse", "lever", "ashby", "workday", "recruitee"] as const) {
34
+ for (const ats of ALL_PROVIDERS) {
33
35
  const fallback = ats === "workday" ? 2 : concurrency;
34
36
  providerLimits.set(ats, new Semaphore(Math.max(1, Math.trunc(options.providerConcurrency?.[ats] ?? fallback))));
35
37
  providerCooldowns.set(ats, new ProviderCooldown());
@@ -162,19 +164,21 @@ async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher
162
164
  const contentType = response.headers.get("content-type") ?? "unknown";
163
165
  let body: unknown;
164
166
  try { body = await response.json(); } catch { throw new VerificationError("invalid_payload", "Endpoint did not return JSON"); }
165
- const jobs = source.ats === "greenhouse" ? recordArray(body, "jobs") : source.ats === "lever" ? array(body) : source.ats === "workday" ? recordArray(body, "jobPostings") : source.ats === "recruitee" ? recordArray(body, "offers") : recordArray(body, "jobs");
167
+ const spec = providerSpec(source.ats);
168
+ const jobs = spec ? spec.jobsFromBody(body) : source.ats === "greenhouse" ? recordArray(body, "jobs") : source.ats === "lever" ? array(body) : source.ats === "workday" ? recordArray(body, "jobPostings") : source.ats === "recruitee" ? recordArray(body, "offers") : recordArray(body, "jobs");
166
169
  if (!jobs) throw new VerificationError("invalid_payload", "Payload does not contain the expected jobs array");
167
170
  if (jobs.length === 0) throw new VerificationError("empty_board", "Source has no jobs, so identity cannot be verified");
168
- const providerName = source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
169
- const hasDomainLink = !["greenhouse", "workday"].includes(source.ats) && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
170
- const hasRedirectEvidence = !["greenhouse", "workday"].includes(source.ats) && await verifiedCompanyRedirect(candidate, source, resolveHost, headTransport, timeoutMs);
171
- if (!["greenhouse", "workday"].includes(source.ats) && !hasDomainLink && !hasRedirectEvidence) throw new VerificationError("identity_mismatch", `Neither structured identity fields nor a verified company redirect link to ${candidate.companyDomain}`);
171
+ const providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
172
+ const namedProvider = source.ats === "greenhouse" || source.ats === "workday" || Boolean(spec && providerName);
173
+ const hasDomainLink = !namedProvider && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
174
+ const hasRedirectEvidence = !namedProvider && await verifiedCompanyRedirect(candidate, source, resolveHost, headTransport, timeoutMs);
175
+ if (!namedProvider && !hasDomainLink && !hasRedirectEvidence) throw new VerificationError("identity_mismatch", `Neither structured identity fields nor a verified company redirect link to ${candidate.companyDomain}`);
172
176
  const observedCompanyName = providerName || candidate.companyName;
173
177
  return {
174
178
  observedCompanyName,
175
179
  identityEvidence: source.ats === "workday" ? "provider_tenant" as const : (providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : "company_redirect" as const),
176
180
  contentType,
177
- payloadVersion: source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
181
+ payloadVersion: spec ? spec.payloadVersion(body) : source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
178
182
  jobCount: source.ats === "workday" && isRecord(body) && typeof body.total === "number" ? body.total : jobs.length,
179
183
  };
180
184
  } finally { clearTimeout(timer); }
@@ -192,6 +196,8 @@ async function fetchProbeWithRetry(fetcher: Fetch, endpoint: string, init: Reque
192
196
  }
193
197
 
194
198
  export function resolveSource(value: string): ResolvedSource | null {
199
+ const tableDriven = resolveProviderSource(value);
200
+ if (tableDriven) return tableDriven;
195
201
  let url: URL;
196
202
  try { url = new URL(value); } catch { return null; }
197
203
  const parts = url.pathname.split("/").filter(Boolean);
@@ -0,0 +1,85 @@
1
+ import { promises as dns } from "node:dns";
2
+ import { readFile } from "node:fs/promises";
3
+ import { atomicJson } from "./atomic-file.ts";
4
+
5
+ /**
6
+ * Passive careers-subdomain discovery. For each company domain it reads certificate transparency logs (no requests to the
7
+ * company) plus a short conventional list, keeps names that look like a careers host, confirms each resolves in DNS, and
8
+ * emits tracer seeds. The career tracer then makes the single safe HEAD request that turns a subdomain into a board.
9
+ * Never brute-forces names and never fetches a page.
10
+ */
11
+
12
+ type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
13
+ type Resolve = (hostname: string) => Promise<string[]>;
14
+
15
+ export interface SubdomainSeed { companyName: string; companyDomain: string }
16
+ export interface SubdomainDiscoveryReport {
17
+ generatedAt: string;
18
+ companies: number;
19
+ certificateNames: number;
20
+ candidates: number;
21
+ resolved: number;
22
+ seeds: Array<SubdomainSeed & { careerUrl: string; via: "certificate_transparency" | "conventional_name" }>;
23
+ failures: Array<{ companyDomain: string; reason: "ct_request_failed" | "invalid_domain"; detail: string }>;
24
+ outputPath: string;
25
+ }
26
+
27
+ const CAREER_WORDS = new Set(["careers", "career", "jobs", "job", "apply", "hiring", "talent", "talents", "recruit", "recruiting", "recruitment", "join", "joinus", "workwithus", "opportunities", "vacancies", "openings"]);
28
+ const CONVENTIONAL = ["careers", "jobs", "apply", "hiring", "talent", "recruit", "join"];
29
+
30
+ /** True when the subdomain part (everything before the company domain) reads like a careers host. */
31
+ export function looksLikeCareersHost(subdomain: string): boolean {
32
+ const words = subdomain.toLowerCase().split(/[.-]+/).filter(Boolean);
33
+ return words.some((word) => CAREER_WORDS.has(word)) || (words.includes("work") && words.includes("us"));
34
+ }
35
+
36
+ export async function discoverCareerSubdomains(seedsPath: string, outputPath: string, options: { fetch?: Fetch; resolve?: Resolve; now?: () => Date; limit?: number; delayMs?: number } = {}): Promise<SubdomainDiscoveryReport> {
37
+ const fetcher = options.fetch ?? globalThis.fetch;
38
+ const resolve = options.resolve ?? (async (hostname: string) => { try { return await dns.resolve(hostname, "A"); } catch { try { return await dns.resolve(hostname, "CNAME"); } catch { return []; } } });
39
+ const now = options.now ?? (() => new Date());
40
+ const delayMs = Math.max(0, options.delayMs ?? 1_000);
41
+ const seeds = (await readSeeds(seedsPath)).slice(0, options.limit ?? Number.POSITIVE_INFINITY);
42
+ const report: SubdomainDiscoveryReport = { generatedAt: now().toISOString(), companies: seeds.length, certificateNames: 0, candidates: 0, resolved: 0, seeds: [], failures: [], outputPath };
43
+
44
+ for (const [index, seed] of seeds.entries()) {
45
+ const domain = seed.companyDomain.toLowerCase().replace(/^www\./, "");
46
+ if (!/^[a-z0-9.-]+\.[a-z]{2,}$/.test(domain)) { report.failures.push({ companyDomain: seed.companyDomain, reason: "invalid_domain", detail: "companyDomain must be a hostname" }); continue; }
47
+ const candidates = new Map<string, "certificate_transparency" | "conventional_name">();
48
+ for (const name of CONVENTIONAL) candidates.set(`${name}.${domain}`, "conventional_name");
49
+ try {
50
+ if (index > 0 && delayMs) await new Promise((done) => setTimeout(done, delayMs));
51
+ const names = await certificateNames(domain, fetcher);
52
+ report.certificateNames += names.length;
53
+ for (const name of names) if (looksLikeCareersHost(name.slice(0, -domain.length - 1))) candidates.set(name, candidates.get(name) ?? "certificate_transparency");
54
+ } catch (error) {
55
+ report.failures.push({ companyDomain: domain, reason: "ct_request_failed", detail: error instanceof Error ? error.message : String(error) });
56
+ }
57
+ report.candidates += candidates.size;
58
+ for (const [host, via] of candidates) {
59
+ if ((await resolve(host)).length === 0) continue;
60
+ report.resolved += 1;
61
+ report.seeds.push({ companyName: seed.companyName, companyDomain: domain, careerUrl: `https://${host}/`, via });
62
+ }
63
+ }
64
+ await atomicJson(outputPath, report.seeds.map(({ companyName, companyDomain, careerUrl }) => ({ companyName, companyDomain, careerUrl })));
65
+ return report;
66
+ }
67
+
68
+ /** Distinct hostnames under `domain` that appear in certificate transparency logs. One request per company. */
69
+ export async function certificateNames(domain: string, fetcher: Fetch): Promise<string[]> {
70
+ const response = await fetcher(`https://crt.sh/?q=${encodeURIComponent(`%.${domain}`)}&output=json`, { headers: { "user-agent": "openings-discovery/0.1 (+https://avagama.co/openings/)" }, signal: AbortSignal.timeout(60_000) });
71
+ if (!response.ok) throw new Error(`crt.sh returned HTTP ${response.status}`);
72
+ const rows = await response.json() as Array<{ name_value?: string }>;
73
+ const names = new Set<string>();
74
+ for (const row of rows) for (const raw of String(row.name_value ?? "").split("\n")) {
75
+ const name = raw.trim().toLowerCase().replace(/^\*\./, "");
76
+ if (name.endsWith(`.${domain}`) && /^[a-z0-9.-]+$/.test(name)) names.add(name);
77
+ }
78
+ return [...names].sort();
79
+ }
80
+
81
+ async function readSeeds(path: string): Promise<SubdomainSeed[]> {
82
+ const value = JSON.parse(await readFile(path, "utf8")) as unknown;
83
+ if (!Array.isArray(value) || !value.every((row) => typeof row === "object" && row !== null && typeof (row as SubdomainSeed).companyName === "string" && typeof (row as SubdomainSeed).companyDomain === "string")) throw new Error("Seed file must be a JSON array of { companyName, companyDomain }");
84
+ return value as SubdomainSeed[];
85
+ }
package/src/types.ts CHANGED
@@ -1,4 +1,5 @@
1
- export type Ats = "greenhouse" | "lever" | "ashby" | "workday" | "recruitee";
1
+ export const ALL_PROVIDERS = ["greenhouse", "lever", "ashby", "workday", "recruitee", "smartrecruiters", "workable", "breezy"] as const;
2
+ export type Ats = (typeof ALL_PROVIDERS)[number];
2
3
 
3
4
  export interface DomainEvidence {
4
5
  kind: "authoritative_dataset" | "company_registry" | "company_redirect";