openings 0.1.2 → 0.1.4

package/src/source-verification.ts

@@ -2,6 +2,8 @@ import type { Ats, RejectedSource, SourceCandidate, SourceRejectionReason, Sourc
 import { fetchSafeHead, type HeadTransport, type ResolveHost } from "./safe-head.ts";
 import { abortableDelay, fetchSourceJobs, isTransientStatus, retryDelayMs } from "./catalog.ts";
 import { isEligibleForCountry } from "./locations.ts";
+import { providerSpec, resolveProviderSource } from "./providers.ts";
+import { ALL_PROVIDERS } from "./types.ts";
 
 type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
 
@@ -29,7 +31,7 @@ export async function verifyCandidates(candidates: SourceCandidate[], options: V
   const rejected: Array<{ index: number; value: RejectedSource }> = [];
   const providerLimits = new Map<Ats, Semaphore>();
   const providerCooldowns = new Map<Ats, ProviderCooldown>();
-  for (const ats of ["greenhouse", "lever", "ashby", "workday", "recruitee"] as const) {
+  for (const ats of ALL_PROVIDERS) {
     const fallback = ats === "workday" ? 2 : concurrency;
     providerLimits.set(ats, new Semaphore(Math.max(1, Math.trunc(options.providerConcurrency?.[ats] ?? fallback))));
     providerCooldowns.set(ats, new ProviderCooldown());
@@ -162,19 +164,21 @@ async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher
     const contentType = response.headers.get("content-type") ?? "unknown";
     let body: unknown;
     try { body = await response.json(); } catch { throw new VerificationError("invalid_payload", "Endpoint did not return JSON"); }
-    const jobs = source.ats === "greenhouse" ? recordArray(body, "jobs") : source.ats === "lever" ? array(body) : source.ats === "workday" ? recordArray(body, "jobPostings") : source.ats === "recruitee" ? recordArray(body, "offers") : recordArray(body, "jobs");
+    const spec = providerSpec(source.ats);
+    const jobs = spec ? spec.jobsFromBody(body) : source.ats === "greenhouse" ? recordArray(body, "jobs") : source.ats === "lever" ? array(body) : source.ats === "workday" ? recordArray(body, "jobPostings") : source.ats === "recruitee" ? recordArray(body, "offers") : recordArray(body, "jobs");
     if (!jobs) throw new VerificationError("invalid_payload", "Payload does not contain the expected jobs array");
     if (jobs.length === 0) throw new VerificationError("empty_board", "Source has no jobs, so identity cannot be verified");
-    const providerName = source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
-    const hasDomainLink = !["greenhouse", "workday"].includes(source.ats) && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
-    const hasRedirectEvidence = !["greenhouse", "workday"].includes(source.ats) && await verifiedCompanyRedirect(candidate, source, resolveHost, headTransport, timeoutMs);
-    if (!["greenhouse", "workday"].includes(source.ats) && !hasDomainLink && !hasRedirectEvidence) throw new VerificationError("identity_mismatch", `Neither structured identity fields nor a verified company redirect link to ${candidate.companyDomain}`);
+    const providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
+    const namedProvider = source.ats === "greenhouse" || source.ats === "workday" || Boolean(spec && providerName);
+    const hasDomainLink = !namedProvider && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
+    const hasRedirectEvidence = !namedProvider && await verifiedCompanyRedirect(candidate, source, resolveHost, headTransport, timeoutMs);
+    if (!namedProvider && !hasDomainLink && !hasRedirectEvidence) throw new VerificationError("identity_mismatch", `Neither structured identity fields nor a verified company redirect link to ${candidate.companyDomain}`);
     const observedCompanyName = providerName || candidate.companyName;
     return {
       observedCompanyName,
       identityEvidence: source.ats === "workday" ? "provider_tenant" as const : (providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : "company_redirect" as const),
       contentType,
-      payloadVersion: source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
+      payloadVersion: spec ? spec.payloadVersion(body) : source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
       jobCount: source.ats === "workday" && isRecord(body) && typeof body.total === "number" ? body.total : jobs.length,
     };
   } finally { clearTimeout(timer); }
@@ -192,6 +196,8 @@ async function fetchProbeWithRetry(fetcher: Fetch, endpoint: string, init: Reque
 }
 
 export function resolveSource(value: string): ResolvedSource | null {
+  const tableDriven = resolveProviderSource(value);
+  if (tableDriven) return tableDriven;
   let url: URL;
   try { url = new URL(value); } catch { return null; }
   const parts = url.pathname.split("/").filter(Boolean);

package/src/subdomain-discovery.ts

@@ -0,0 +1,85 @@
+import { promises as dns } from "node:dns";
+import { readFile } from "node:fs/promises";
+import { atomicJson } from "./atomic-file.ts";
+
+/**
+ * Passive careers-subdomain discovery. For each company domain it reads certificate transparency logs (no requests to the
+ * company) plus a short conventional list, keeps names that look like a careers host, confirms each resolves in DNS, and
+ * emits tracer seeds. The career tracer then makes the single safe HEAD request that turns a subdomain into a board.
+ * Never brute-forces names and never fetches a page.
+ */
+
+type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
+type Resolve = (hostname: string) => Promise<string[]>;
+
+export interface SubdomainSeed { companyName: string; companyDomain: string }
+export interface SubdomainDiscoveryReport {
+  generatedAt: string;
+  companies: number;
+  certificateNames: number;
+  candidates: number;
+  resolved: number;
+  seeds: Array<SubdomainSeed & { careerUrl: string; via: "certificate_transparency" | "conventional_name" }>;
+  failures: Array<{ companyDomain: string; reason: "ct_request_failed" | "invalid_domain"; detail: string }>;
+  outputPath: string;
+}
+
+const CAREER_WORDS = new Set(["careers", "career", "jobs", "job", "apply", "hiring", "talent", "talents", "recruit", "recruiting", "recruitment", "join", "joinus", "workwithus", "opportunities", "vacancies", "openings"]);
+const CONVENTIONAL = ["careers", "jobs", "apply", "hiring", "talent", "recruit", "join"];
+
+/** True when the subdomain part (everything before the company domain) reads like a careers host. */
+export function looksLikeCareersHost(subdomain: string): boolean {
+  const words = subdomain.toLowerCase().split(/[.-]+/).filter(Boolean);
+  return words.some((word) => CAREER_WORDS.has(word)) || (words.includes("work") && words.includes("us"));
+}
+
+export async function discoverCareerSubdomains(seedsPath: string, outputPath: string, options: { fetch?: Fetch; resolve?: Resolve; now?: () => Date; limit?: number; delayMs?: number } = {}): Promise<SubdomainDiscoveryReport> {
+  const fetcher = options.fetch ?? globalThis.fetch;
+  const resolve = options.resolve ?? (async (hostname: string) => { try { return await dns.resolve(hostname, "A"); } catch { try { return await dns.resolve(hostname, "CNAME"); } catch { return []; } } });
+  const now = options.now ?? (() => new Date());
+  const delayMs = Math.max(0, options.delayMs ?? 1_000);
+  const seeds = (await readSeeds(seedsPath)).slice(0, options.limit ?? Number.POSITIVE_INFINITY);
+  const report: SubdomainDiscoveryReport = { generatedAt: now().toISOString(), companies: seeds.length, certificateNames: 0, candidates: 0, resolved: 0, seeds: [], failures: [], outputPath };
+
+  for (const [index, seed] of seeds.entries()) {
+    const domain = seed.companyDomain.toLowerCase().replace(/^www\./, "");
+    if (!/^[a-z0-9.-]+\.[a-z]{2,}$/.test(domain)) { report.failures.push({ companyDomain: seed.companyDomain, reason: "invalid_domain", detail: "companyDomain must be a hostname" }); continue; }
+    const candidates = new Map<string, "certificate_transparency" | "conventional_name">();
+    for (const name of CONVENTIONAL) candidates.set(`${name}.${domain}`, "conventional_name");
+    try {
+      if (index > 0 && delayMs) await new Promise((done) => setTimeout(done, delayMs));
+      const names = await certificateNames(domain, fetcher);
+      report.certificateNames += names.length;
+      for (const name of names) if (looksLikeCareersHost(name.slice(0, -domain.length - 1))) candidates.set(name, candidates.get(name) ?? "certificate_transparency");
+    } catch (error) {
+      report.failures.push({ companyDomain: domain, reason: "ct_request_failed", detail: error instanceof Error ? error.message : String(error) });
+    }
+    report.candidates += candidates.size;
+    for (const [host, via] of candidates) {
+      if ((await resolve(host)).length === 0) continue;
+      report.resolved += 1;
+      report.seeds.push({ companyName: seed.companyName, companyDomain: domain, careerUrl: `https://${host}/`, via });
+    }
+  }
+  await atomicJson(outputPath, report.seeds.map(({ companyName, companyDomain, careerUrl }) => ({ companyName, companyDomain, careerUrl })));
+  return report;
+}
+
+/** Distinct hostnames under `domain` that appear in certificate transparency logs. One request per company. */
+export async function certificateNames(domain: string, fetcher: Fetch): Promise<string[]> {
+  const response = await fetcher(`https://crt.sh/?q=${encodeURIComponent(`%.${domain}`)}&output=json`, { headers: { "user-agent": "openings-discovery/0.1 (+https://avagama.co/openings/)" }, signal: AbortSignal.timeout(60_000) });
+  if (!response.ok) throw new Error(`crt.sh returned HTTP ${response.status}`);
+  const rows = await response.json() as Array<{ name_value?: string }>;
+  const names = new Set<string>();
+  for (const row of rows) for (const raw of String(row.name_value ?? "").split("\n")) {
+    const name = raw.trim().toLowerCase().replace(/^\*\./, "");
+    if (name.endsWith(`.${domain}`) && /^[a-z0-9.-]+$/.test(name)) names.add(name);
+  }
+  return [...names].sort();
+}
+
+async function readSeeds(path: string): Promise<SubdomainSeed[]> {
+  const value = JSON.parse(await readFile(path, "utf8")) as unknown;
+  if (!Array.isArray(value) || !value.every((row) => typeof row === "object" && row !== null && typeof (row as SubdomainSeed).companyName === "string" && typeof (row as SubdomainSeed).companyDomain === "string")) throw new Error("Seed file must be a JSON array of { companyName, companyDomain }");
+  return value as SubdomainSeed[];
+}

package/src/types.ts

@@ -1,4 +1,5 @@
-export type Ats = "greenhouse" | "lever" | "ashby" | "workday" | "recruitee";
+export const ALL_PROVIDERS = ["greenhouse", "lever", "ashby", "workday", "recruitee", "smartrecruiters", "workable", "breezy"] as const;
+export type Ats = (typeof ALL_PROVIDERS)[number];
 
 export interface DomainEvidence {
   kind: "authoritative_dataset" | "company_registry" | "company_redirect";
@@ -39,7 +40,7 @@ export interface SourceVerification {
   checkedAt: string;
   canonicalSourceUrl: string;
   observedCompanyName: string;
-  identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect";
+  identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect" | "provider_board";
   contentType: string;
   payloadVersion: string;
   jobCount: number;