openings 0.1.18 → 0.1.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.18",
+  "version": "0.1.20",
   "description": "A free, candidate-safe job-search substrate for AI agents",
   "license": "MIT",
   "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },

package/src/board-verification.ts

@@ -8,7 +8,7 @@ import type { Ats, Company, SourceVerification } from "./types.ts";
 type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
 
 /** Providers whose public board is accepted as identity on its own. Workday boards are identified by tenant; their crawls are heavier but they carry the large employers. */
-export const BOARD_TIER_PROVIDERS: ReadonlySet<Ats> = new Set(["greenhouse", "lever", "ashby", "recruitee", "smartrecruiters", "workable", "breezy", "workday", "freshteam"]);
+export const BOARD_TIER_PROVIDERS: ReadonlySet<Ats> = new Set(["greenhouse", "lever", "ashby", "recruitee", "smartrecruiters", "workable", "breezy", "workday", "freshteam", "keka", "zohorecruit"]);
 
 export interface BoardVerificationOptions {
   fetch?: Fetch;
@@ -129,14 +129,15 @@ async function probeBoard(lead: EnrichmentLead, fetcher: Fetch, timeoutMs: numbe
       : { signal: controller.signal };
     const response = await fetcher(source.structuredEndpoint, init);
     if (!response.ok) { await response.body?.cancel().catch(() => undefined); throw new BoardError(response.status === 404 || response.status === 410 ? "invalid_payload" : "unreachable", `HTTP ${response.status}`); }
-    let body: unknown;
-    try { body = await response.json(); } catch { throw new BoardError("invalid_payload", "Endpoint did not return JSON"); }
     const spec = providerSpec(source.ats);
+    let body: unknown;
+    try { body = spec?.bodyFormat === "text" ? await response.text() : await response.json(); } catch { throw new BoardError("invalid_payload", "Endpoint did not return JSON"); }
     const jobs = spec ? spec.jobsFromBody(body) : source.ats === "lever" ? asArray(body) : asArray(isRecord(body) ? body[source.ats === "recruitee" ? "offers" : source.ats === "workday" ? "jobPostings" : "jobs"] : undefined);
     if (!jobs) throw new BoardError("invalid_payload", "Payload does not contain the expected jobs array");
     if (jobs.length === 0) throw new BoardError("empty_board", "Board has no jobs");
-    const providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => typeof job.company_name === "string" ? job.company_name.trim() : "").filter(Boolean)) : source.ats === "workday" ? humanize(source.token.split("/")[1] ?? source.token) : "";
+    let providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => typeof job.company_name === "string" ? job.company_name.trim() : "").filter(Boolean)) : source.ats === "workday" ? humanize(source.token.split("/")[1] ?? source.token) : "";
     const payloadVersion = spec ? spec.payloadVersion(body) : source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "recruitee" ? "recruitee-careers:v1" : source.ats === "workday" ? "workday-cxs:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`;
+    if (spec?.companyInfo && !providerName) providerName = (await spec.companyInfo(source.token, async (url, format = "json") => { const reply = await fetcher(url, { signal: controller.signal }); if (!reply.ok) { await reply.body?.cancel().catch(() => undefined); throw new BoardError("unreachable", `HTTP ${reply.status}`); } return format === "text" ? reply.text() : reply.json(); }).catch(() => ({ name: "" }))).name;
     const jobCount = source.ats === "workday" && isRecord(body) && typeof body.total === "number" ? body.total : jobs.length;
     return { providerName, contentType: response.headers.get("content-type") ?? "unknown", payloadVersion, jobCount };
   } finally { clearTimeout(timer); }

package/src/catalog.ts

@@ -1,5 +1,6 @@
 import type { Company, Job, JobSummary, SearchQuery } from "./types.ts";
 import { providerSpec, type JsonGet } from "./providers.ts";
+import { crawlSite, sitePostingsToJobs } from "./jobposting-site.ts";
 import { classifyJob, isEligibleForCountry, normalizeLocation } from "./locations.ts";
 
 type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
@@ -134,10 +135,15 @@ export interface FetchJobsObserver {
 }
 export async function fetchSourceJobs(company: Company, fetcher: Fetch = globalThis.fetch, signal?: AbortSignal, observer?: FetchJobsObserver): Promise<Job[]> {
   if (company.ats === "workday") return fetchWorkdayJobs(company, fetcher, signal, observer);
+  if (company.ats === "jobposting") {
+    if (!company.companyDomain) throw new Error(`${company.name} company site source has no company domain`);
+    const site = await crawlSite({ companyName: company.name, companyDomain: company.companyDomain, careerUrl: company.token }, { maxPages: 60, delayMs: 300, signal });
+    return sitePostingsToJobs(company, site.postings);
+  }
   const spec = providerSpec(company.ats);
   if (spec) {
     const get = jsonGetter(fetcher, company.name, signal, observer);
-    const records = spec.fetchAll ? await spec.fetchAll(company.token, get) : spec.jobsFromBody(await get(spec.endpoint(company.token)));
+    const records = spec.fetchAll ? await spec.fetchAll(company.token, get) : spec.jobsFromBody(await get(spec.endpoint(company.token), spec.bodyFormat ?? "json"));
     if (!records) throw new Error(`${company.name} job board returned an invalid payload`);
     return records.map((record) => spec.normalize(company, record));
   }
@@ -243,10 +249,10 @@ async function fetchWorkdayJobs(company: Company, fetcher: Fetch, signal?: Abort
 
 /** JSON fetch with the catalog's retry and backoff, shaped for the table-driven providers. */
 function jsonGetter(fetcher: Fetch, companyName: string, signal?: AbortSignal, observer?: FetchJobsObserver): JsonGet {
-  return async (url) => {
+  return async (url, format = "json") => {
     const response = await fetchWithRetry(fetcher, url, signal ? { signal } : undefined, companyName, observer);
     if (!response.ok) { await response.body?.cancel().catch(() => undefined); throw new Error(`${companyName} job board returned HTTP ${response.status}`); }
-    return response.json();
+    return format === "text" ? response.text() : response.json();
   };
 }
 

package/src/cli.ts

@@ -17,6 +17,9 @@ import { enrichSourcesFromCompanies } from "./source-enrichment.ts";
 import { forceReleaseFileLock, inspectFileLock } from "./file-lock.ts";
 import { generateCountryCoverageReport } from "./country-coverage.ts";
 import { probeJobPostingJsonLd } from "./jobposting-probe.ts";
+import { probeSites } from "./site-probe.ts";
+import { admitSites } from "./site-admission.ts";
+import { kekaTenantCandidates } from "./keka-tenants.ts";
 import { mergeAttemptedRoundLeads, prepareRecruiteeRoundArtifacts } from "./recruitee-round.ts";
 
 const HELP = `Openings — search public company job boards
@@ -35,6 +38,9 @@ Usage:
   openings sources enrich COMPANIES.json [--companies FILE]... [--evidence-kind authoritative_dataset|company_registry] [--registry FILE] [--output FILE] [--report FILE]
   openings sources trace-careers COMPANIES.json [--country CODE] [--registry FILE] [--common-crawl-report FILE] [--search-key-env NAME] [--output FILE] [--catalog FILE] [--report FILE]
   openings sources probe-jobposting COMPANIES.md [--catalog FILE] [--report FILE] [--company-limit 10|20]
+  openings sources probe-sites SEEDS.json [--sample N] [--limit N] [--concurrency N] [--max-pages N] [--delay-ms N] [--report FILE]
+  openings sources admit-sites REPORT.json [--catalog FILE] [--min-postings N]
+  openings sources keka-tenants HOSTS.txt [--output FILE] [--registry FILE] [--concurrency N]
   openings sources prepare-recruitee-round IDENTITIES.json [--catalog FILE] [--artifacts DIR]
   openings sources merge-attempted-round-leads ISOLATED_REGISTRY [--registry FILE]
   openings search [words] [--country CODE|--india] [--location PLACE] [--remote|--onsite]
@@ -114,6 +120,36 @@ export async function run(args: string[]): Promise<number> {
       console.log(JSON.stringify(await mergeAttemptedRoundLeads(parsed.isolatedRegistryPath, parsed.registry), null, 2));
       return 0;
     }
+    if (rest[0] === "probe-sites") {
+      const args = rest.slice(1);
+      const inputPath = args[0];
+      if (!inputPath || inputPath.startsWith("--")) return fail("sources probe-sites requires a seeds JSON file");
+      const num = (flag: string) => { const index = args.indexOf(flag); return index >= 0 ? Number(args[index + 1]) : undefined; };
+      const reportIndex = args.indexOf("--report");
+      const report = await probeSites(inputPath, reportIndex >= 0 ? args[reportIndex + 1]! : ".openings/site-probe.json", { sample: num("--sample"), limit: num("--limit"), concurrency: num("--concurrency"), maxPages: num("--max-pages"), delayMs: num("--delay-ms") });
+      const { sites, ...summary } = report;
+      console.log(JSON.stringify({ ...summary, topSites: sites.slice(0, 15) }, null, 2));
+      return 0;
+    }
+    if (rest[0] === "keka-tenants") {
+      const args = rest.slice(1);
+      const hostsPath = args[0];
+      if (!hostsPath || hostsPath.startsWith("--")) return fail("sources keka-tenants requires a hosts file, one tenant host per line");
+      const outputIndex = args.indexOf("--output");
+      const concurrencyIndex = args.indexOf("--concurrency");
+      const registryIndex = args.indexOf("--registry");
+      console.log(JSON.stringify(await kekaTenantCandidates(hostsPath, outputIndex >= 0 ? args[outputIndex + 1]! : "data/source-candidates.json", { concurrency: concurrencyIndex >= 0 ? Number(args[concurrencyIndex + 1]) : undefined, registryPath: registryIndex >= 0 ? args[registryIndex + 1] : undefined }), null, 2));
+      return 0;
+    }
+    if (rest[0] === "admit-sites") {
+      const args = rest.slice(1);
+      const reportPath = args[0];
+      if (!reportPath || reportPath.startsWith("--")) return fail("sources admit-sites requires a probe report JSON file");
+      const catalogIndex = args.indexOf("--catalog");
+      const minIndex = args.indexOf("--min-postings");
+      console.log(JSON.stringify(await admitSites(reportPath, catalogIndex >= 0 ? args[catalogIndex + 1]! : "data/companies.json", { minPostings: minIndex >= 0 ? Number(args[minIndex + 1]) : undefined }), null, 2));
+      return 0;
+    }
     if (rest[0] === "probe-jobposting") {
       const parsed = parseJobPostingProbe(rest.slice(1));
       if (typeof parsed === "string") return fail(parsed);

package/src/common-crawl-discovery.ts

@@ -39,7 +39,8 @@ export interface CommonCrawlDiscoveryReport extends ReportMeta {
 
 const providerPatterns: Record<Ats, string[]> = {
   greenhouse: ["job-boards.greenhouse.io/*", "boards.greenhouse.io/*"], lever: ["jobs.lever.co/*"], ashby: ["jobs.ashbyhq.com/*"], workday: ["*.myworkdayjobs.com/*"], recruitee: ["*.recruitee.com/*"],
-  ...Object.fromEntries(PROVIDERS.map((spec) => [spec.ats, spec.crawlPatterns])) as Record<"smartrecruiters" | "workable" | "breezy" | "freshteam", string[]>,
+  ...Object.fromEntries(PROVIDERS.map((spec) => [spec.ats, spec.crawlPatterns])) as Record<"smartrecruiters" | "workable" | "breezy" | "freshteam" | "keka" | "zohorecruit", string[]>,
+  jobposting: [], // company sites are found by probing seeds, never by URL pattern
 };
 const patterns = Object.values(providerPatterns).flat();
 const recordsPerPattern = 10_000;
@@ -70,7 +71,12 @@ export async function discoverCommonCrawlSources(candidatesPath: string, reportP
     query.searchParams.set("collapse", "urlkey");
     query.searchParams.set("fl", "url");
     query.searchParams.set("limit", String(indexRecordLimit));
-    const response = await fetcher(query);
+    let response = await fetcher(query);
+    for (let attempt = 1; !response.ok && response.status >= 500 && attempt <= 3; attempt += 1) { // the index server sheds load with 502s; back off and retry
+      await response.body?.cancel().catch(() => undefined);
+      await new Promise((resolve) => setTimeout(resolve, attempt * 5_000));
+      response = await fetcher(query);
+    }
     if (!response.ok) throw new Error(`Common Crawl index returned HTTP ${response.status} for ${pattern}`);
     const lines = (await response.text()).split(/\r?\n/).filter(Boolean).slice(0, indexRecordLimit);
     indexRecordsExamined += lines.length;

package/src/country-coverage.ts

@@ -192,7 +192,7 @@ function validLastCrawl(value: Record<string, unknown>): boolean {
 }
 function validVerification(value: unknown): boolean {
   return isRecord(value) && validDate(value.checkedAt) && typeof value.canonicalSourceUrl === "string" && typeof value.observedCompanyName === "string"
-    && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect"].includes(String(value.identityEvidence))
+    && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect", "company_page_link", "provider_board", "company_site"].includes(String(value.identityEvidence))
     && typeof value.contentType === "string" && typeof value.payloadVersion === "string" && nonNegativeInteger(value.jobCount);
 }
 function validDate(value: unknown): value is string { return typeof value === "string" && Number.isFinite(Date.parse(value)); }

package/src/jobposting-site.ts

@@ -0,0 +1,162 @@
+import { classifyJob } from "./locations.ts";
+import { countryLabel } from "./providers.ts";
+import { extractLinks, fetchSafePage, robotsAllows, type PageTransport, type ResolveHost } from "./safe-head.ts";
+import type { Job } from "./types.ts";
+
+/**
+ * Company-owned career sites: the only thing read is the schema.org JobPosting JSON-LD block that employers publish
+ * for search engines. Discovery is bounded (robots.txt honoured, same company domain, a page cap, a delay between
+ * requests) and page prose is never interpreted.
+ */
+export interface SitePosting { title: string; company?: string; location: string; url: string; identifier?: string; datePosted?: string; validThrough?: string; description: string; remote: boolean }
+export interface SiteSeed { companyName: string; companyDomain: string; careerUrl?: string }
+export interface SiteCrawlOptions { resolveHost?: ResolveHost; pageTransport?: PageTransport; timeoutMs?: number; maxPages?: number; delayMs?: number; sleep?: (ms: number) => Promise<void>; signal?: AbortSignal }
+export interface SiteCrawlResult { careerUrl?: string; robotsBlocked: boolean; candidatePages: number; pagesFetched: number; postings: SitePosting[]; issues: string[] }
+
+const JOB_PATH = /job|career|opening|vacanc|position|recruit|apply|hiring/i;
+
+export function ownedBy(url: string, companyDomain: string): boolean {
+  try {
+    const host = new URL(url).hostname.toLowerCase().replace(/^www\./, "");
+    const domain = companyDomain.toLowerCase().replace(/^www\./, "");
+    return host === domain || host.endsWith(`.${domain}`);
+  } catch { return false; }
+}
+
+/** JobPosting objects from every ld+json block on a page, including @graph, arrays, and ItemList wrappers. */
+export function extractJobPostings(html: string, pageUrl: string): SitePosting[] {
+  const postings: SitePosting[] = [];
+  for (const match of html.matchAll(/<script[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
+    let value: unknown;
+    try { value = JSON.parse(match[1]!.trim().replace(/^<!--|-->$/g, "")); } catch { continue; }
+    for (const node of walk(value)) {
+      const type = node["@type"];
+      const types = Array.isArray(type) ? type.map(String) : [String(type ?? "")];
+      if (!types.some((entry) => entry.toLowerCase() === "jobposting")) continue;
+      const posting = normalizePosting(node, pageUrl);
+      if (posting) postings.push(posting);
+    }
+  }
+  return postings;
+}
+
+function* walk(value: unknown, depth = 0): Generator<Record<string, unknown>> {
+  if (depth > 6) return;
+  if (Array.isArray(value)) { for (const item of value) yield* walk(item, depth + 1); return; }
+  if (!isRecord(value)) return;
+  yield value;
+  for (const key of ["@graph", "itemListElement", "item", "mainEntity", "hasPart"]) if (key in value) yield* walk(value[key], depth + 1);
+}
+
+function normalizePosting(node: Record<string, unknown>, pageUrl: string): SitePosting | null {
+  const title = text(node.title) || text(node.name);
+  if (!title) return null;
+  const organization = isRecord(node.hiringOrganization) ? text(node.hiringOrganization.name) : text(node.hiringOrganization);
+  const locations = (Array.isArray(node.jobLocation) ? node.jobLocation : [node.jobLocation]).filter(isRecord).map(placeText).filter(Boolean);
+  const remote = String(node.jobLocationType ?? "").toUpperCase().includes("TELECOMMUTE");
+  const applicant = isRecord(node.applicantLocationRequirements) ? [node.applicantLocationRequirements] : Array.isArray(node.applicantLocationRequirements) ? node.applicantLocationRequirements.filter(isRecord) : [];
+  const applicantText = applicant.map((entry) => text(entry.name)).filter(Boolean).join("; ");
+  const location = locations.join("; ") || (remote ? `Remote${applicantText ? ` (${applicantText})` : ""}` : "Unspecified");
+  const identifier = isRecord(node.identifier) ? text(node.identifier.value) || text(node.identifier.name) : text(node.identifier);
+  let url = text(node.url) || (isRecord(node.mainEntityOfPage) ? text(node.mainEntityOfPage["@id"]) : text(node.mainEntityOfPage)) || pageUrl;
+  try { url = new URL(url, pageUrl).href; } catch { url = pageUrl; }
+  return {
+    title, company: organization || undefined, location, url, identifier: identifier || undefined,
+    datePosted: isoDate(node.datePosted), validThrough: isoDate(node.validThrough),
+    description: stripHtml(text(node.description)).slice(0, 20_000), remote,
+  };
+}
+
+function placeText(place: Record<string, unknown>): string {
+  const address = isRecord(place.address) ? place.address : place;
+  const parts = [text(address.addressLocality), text(address.addressRegion), text(address.addressCountry) ? countryLabel(text(address.addressCountry)) : ""].filter(Boolean);
+  if (parts.length) return parts.join(", ");
+  return text(place.name) || (typeof place.address === "string" ? place.address : "");
+}
+
+/** Find the careers page, gather job-like links from it and from sitemaps, and read the JobPosting blocks on each. */
+export async function crawlSite(seed: SiteSeed, options: SiteCrawlOptions = {}): Promise<SiteCrawlResult> {
+  const maxPages = options.maxPages ?? 25;
+  const sleep = options.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const fetchOptions = { resolveHost: options.resolveHost, transport: options.pageTransport, timeoutMs: options.timeoutMs ?? 15_000 };
+  const result: SiteCrawlResult = { robotsBlocked: false, candidatePages: 0, pagesFetched: 0, postings: [], issues: [] };
+  let requests = 0;
+  const get = async (url: string) => {
+    if (options.signal?.aborted) throw options.signal.reason instanceof Error ? options.signal.reason : new Error("Crawl aborted");
+    if (requests > 0 && options.delayMs) await sleep(options.delayMs);
+    requests += 1;
+    if (!(await robotsAllows(url, fetchOptions))) { result.robotsBlocked = true; return null; }
+    const page = await fetchSafePage(url, fetchOptions);
+    if (page.status !== 200 || !ownedBy(page.finalUrl, seed.companyDomain)) return null;
+    return page;
+  };
+  const candidates = seed.careerUrl ? [seed.careerUrl] : ["careers", "career", "jobs"].map((path) => `https://${seed.companyDomain}/${path}`);
+  let landing: { finalUrl: string; html: string } | null = null;
+  for (const url of candidates) {
+    try { landing = await get(url); if (landing) { result.careerUrl = landing.finalUrl; break; } }
+    catch (error) { result.issues.push(`${url}: ${error instanceof Error ? error.message : String(error)}`); }
+  }
+  const seen = new Set<string>();
+  const pages: string[] = [];
+  const consider = (url: string) => { if (!seen.has(url) && ownedBy(url, seed.companyDomain) && JOB_PATH.test(new URL(url).pathname)) { seen.add(url); pages.push(url); } };
+  if (landing) {
+    result.postings.push(...extractJobPostings(landing.html, landing.finalUrl));
+    result.pagesFetched += 1;
+    for (const link of extractLinks(landing.html, landing.finalUrl)) consider(link);
+  }
+  // Sitemaps list the job-detail pages that carry the markup; read at most three of them.
+  const origin = `https://${seed.companyDomain.replace(/^www\./, "")}`;
+  const sitemaps = await sitemapUrls(origin, fetchOptions);
+  let sitemapReads = 0;
+  for (const sitemap of sitemaps) {
+    if (sitemapReads >= 3 || pages.length >= maxPages * 4) break;
+    try {
+      const page = await get(sitemap);
+      if (!page) continue;
+      sitemapReads += 1;
+      for (const loc of page.html.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)) {
+        const url = loc[1]!;
+        if (/sitemap/i.test(new URL(url).pathname) && JOB_PATH.test(url) && sitemapReads < 3 && !sitemaps.includes(url)) sitemaps.push(url);
+        else consider(url);
+      }
+    } catch (error) { result.issues.push(`${sitemap}: ${error instanceof Error ? error.message : String(error)}`); }
+  }
+  result.candidatePages = pages.length;
+  for (const url of pages.slice(0, maxPages)) {
+    try {
+      const page = await get(url);
+      if (!page) continue;
+      result.pagesFetched += 1;
+      result.postings.push(...extractJobPostings(page.html, page.finalUrl));
+    } catch (error) { result.issues.push(`${url}: ${error instanceof Error ? error.message : String(error)}`); }
+  }
+  const unique = new Map<string, SitePosting>();
+  for (const posting of result.postings) unique.set(posting.identifier ? `id:${posting.identifier}` : `url:${posting.url}:${posting.title}`, posting);
+  result.postings = [...unique.values()];
+  return result;
+}
+
+async function sitemapUrls(origin: string, fetchOptions: Parameters<typeof fetchSafePage>[1]): Promise<string[]> {
+  const urls = new Set<string>([`${origin}/sitemap.xml`]);
+  try {
+    const robots = await fetchSafePage(`${origin}/robots.txt`, { ...fetchOptions, maxBytes: 64_000 });
+    if (robots.status === 200) for (const match of robots.html.matchAll(/^\s*sitemap:\s*(\S+)/gim)) urls.add(match[1]!);
+  } catch { /* no robots: conventional sitemap only */ }
+  return [...urls];
+}
+
+/** Jobs for the catalog from one company site, in the same shape every ATS adapter produces. */
+export function sitePostingsToJobs(company: { slug: string; name: string }, postings: SitePosting[]): Job[] {
+  return postings.map((posting) => classifyJob({
+    id: `jobposting:${company.slug}:${posting.identifier ?? hash(posting.url + posting.title)}`,
+    company: company.name, title: posting.title, location: posting.location, remote: posting.remote, workMode: posting.remote ? "remote" : "unknown",
+    eligibleCountries: [], excludedCountries: [], eligibleRegions: [], eligibilityConfidence: "unknown",
+    url: posting.url, ...(posting.datePosted ? { updatedAt: posting.datePosted } : {}), description: posting.description,
+  }));
+}
+
+function hash(value: string): string { let h = 2166136261; for (const ch of value) { h ^= ch.charCodeAt(0); h = Math.imul(h, 16777619) >>> 0; } return h.toString(16); }
+function text(value: unknown): string { return typeof value === "string" ? value.trim() : typeof value === "number" ? String(value) : ""; }
+function isoDate(value: unknown): string | undefined { const time = Date.parse(text(value)); return Number.isFinite(time) ? new Date(time).toISOString() : undefined; }
+function stripHtml(value: string): string { return value.replace(/<[^>]+>/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/\s+/g, " ").trim(); }
+function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }

package/src/keka-tenants.ts

@@ -0,0 +1,46 @@
+import { readFile } from "node:fs/promises";
+import { atomicJson } from "./atomic-file.ts";
+import { providerSpec } from "./providers.ts";
+import { mergeEnrichmentLeads, type EnrichmentLead } from "./enrichment-registry.ts";
+import type { SourceCandidate } from "./types.ts";
+
+/**
+ * Keka tenants are discoverable by host, but a board token needs the org id that only the careers portal shell carries.
+ * One fetch per tenant turns a host list into verifiable candidates with the company name and, when Keka exposes it,
+ * the company's own website as the domain to verify against.
+ */
+export async function kekaTenantCandidates(hostsPath: string, outputPath: string, options: { fetcher?: typeof fetch; concurrency?: number; timeoutMs?: number; registryPath?: string } = {}): Promise<{ hosts: number; candidates: number; leads: number; failed: number }> {
+  const fetcher = options.fetcher ?? globalThis.fetch;
+  const hosts = [...new Set((await readFile(hostsPath, "utf8")).split(/\r?\n/).map((line) => line.trim().toLowerCase()).filter((line) => /^[a-z0-9-]+\.keka\.com$/.test(line)))];
+  const spec = providerSpec("keka")!;
+  const candidates: SourceCandidate[] = [];
+  const leads: EnrichmentLead[] = [];
+  let failed = 0;
+  let cursor = 0;
+  async function worker() {
+    while (cursor < hosts.length) {
+      const host = hosts[cursor++]!;
+      const tenant = host.split(".")[0]!;
+      try {
+        const get = async (url: string, format: "json" | "text" = "json") => { const reply = await fetcher(url, { signal: AbortSignal.timeout(options.timeoutMs ?? 15_000), headers: { accept: format === "text" ? "text/html" : "application/json" } }); if (!reply.ok) throw new Error(`HTTP ${reply.status}`); return format === "text" ? reply.text() : reply.json(); };
+        const shell = await get(`https://${host}/careers`, "text") as string;
+        const org = /\/ats\/documents\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\//i.exec(shell)?.[1]?.toLowerCase();
+        if (!org) { failed += 1; continue; }
+        const info = await spec.companyInfo!(`${tenant}/${org}`, get);
+        if (!info.name) { failed += 1; continue; }
+        const token = `${tenant}/${org}`;
+        const sourceUrl = spec.endpoint(token);
+        if (info.website) {
+          candidates.push({ companyName: info.name, companyDomain: info.website, sourceUrl, cohorts: ["IN"], discoveredFrom: { channel: "provider_directory", reference: `https://${host}/careers` } });
+        } else {
+          // No company website to verify against: the board tier can still admit it on the provider's own identity.
+          leads.push({ sourceKey: `keka:${token.toLowerCase()}`, sourceUrl, ats: "keka", token, discoveredFrom: [{ channel: "provider_directory", reference: `https://${host}/careers` }], companyMatches: [], identityEvidence: [], attempts: [] });
+        }
+      } catch { failed += 1; }
+    }
+  }
+  await Promise.all(Array.from({ length: Math.max(1, options.concurrency ?? 6) }, worker));
+  await atomicJson(outputPath, candidates);
+  if (options.registryPath && leads.length) await mergeEnrichmentLeads(options.registryPath, leads);
+  return { hosts: hosts.length, candidates: candidates.length, leads: leads.length, failed };
+}

package/src/locations.ts

@@ -123,9 +123,13 @@ function detectCountries(location: string): string[] {
     return [...new Set(["US", ...byCode.filter((code) => !US_STATE_CODES.has(code))])];
   }
   if (byName.includes("US")) return [...new Set([...byName, ...byCode.filter((code) => !US_STATE_CODES.has(code))])];
+  // "Hyderabad, TG, India": a bare code next to a named India is an Indian state (TG Telangana, KA Karnataka), not Togo or Kazakhstan.
+  if (byName.includes("IN") || INDIA_PATTERN.test(location)) return [...new Set([...byName, ...byCode.filter((code) => !INDIA_STATE_CODES.has(code))])];
   return [...new Set([...byName, ...byCode])];
 }
 
+const INDIA_STATE_CODES = new Set(["AP", "AR", "AS", "BR", "CG", "CT", "GA", "GJ", "HR", "HP", "JH", "KA", "KL", "MP", "MH", "MN", "ML", "MZ", "NL", "OD", "OR", "PB", "RJ", "SK", "TN", "TG", "TS", "TR", "UP", "UK", "UT", "WB", "AN", "CH", "DN", "DD", "DL", "JK", "LA", "LD", "PY"]);
+
 function detectEligibleCountries(description: string): string[] {
   const matches: string[] = [];
   for (const rule of countryRules()) {

package/src/providers.ts

@@ -8,8 +8,8 @@ import type { Ats, Company, Job } from "./types.ts";
  */
 
 type Rec = Record<string, unknown>;
-/** Fetches a URL and returns its parsed JSON body; the caller supplies retry and pacing. */
-export type JsonGet = (url: string) => Promise<unknown>;
+/** Fetches a URL and returns its parsed JSON body (or the raw text when asked); the caller supplies retry and pacing. */
+export type JsonGet = (url: string, format?: "json" | "text") => Promise<unknown>;
 
 export interface ProviderSpec {
   ats: Ats;
@@ -21,8 +21,12 @@ export interface ProviderSpec {
   resolve(url: URL): string | null;
   canonicalUrl(token: string): string;
   endpoint(token: string): string;
+  /** "text" when the structured endpoint is not JSON (an RSS feed); `jobsFromBody` then receives the raw string. */
+  bodyFormat?: "json" | "text";
   jobsFromBody(body: unknown): Rec[] | null;
   providerName(jobs: Rec[], body: unknown): string;
+  /** Company identity from a separate tenant endpoint when the jobs payload carries no name; `website` is the company's own domain when the provider exposes it. */
+  companyInfo?(token: string, get: JsonGet): Promise<{ name: string; website?: string }>;
   payloadVersion(body: unknown): string;
   normalize(company: Company, job: Rec): Job;
   /** Fetches every record when the endpoint paginates. Defaults to one request to `endpoint`. */
@@ -179,7 +183,84 @@ const freshteam: ProviderSpec = {
   },
 };
 
-export const PROVIDERS: ReadonlyArray<ProviderSpec> = [smartrecruiters, workable, breezy, freshteam];
+/** Keka (India HRMS): the careers portal calls an unauthenticated embed API. Token is `tenant/orgId`; the org id sits in the portal shell. */
+const keka: ProviderSpec = {
+  ats: "keka",
+  label: "Keka",
+  hosts: ["keka.com"],
+  crawlPatterns: ["*.keka.com/careers*"],
+  resolve(url) {
+    const tenant = /^([a-z0-9-]+)\.keka\.com$/i.exec(url.hostname)?.[1]?.toLowerCase();
+    if (!tenant || ["www", "app", "hr", "academy", "developers", "help", "cdn", "api"].includes(tenant) && tenant !== "hr") return null;
+    const org = /\/(?:careers\/api\/embedjobs\/default\/active|ats\/documents)\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i.exec(url.pathname)?.[1];
+    return org ? `${tenant}/${org.toLowerCase()}` : null;
+  },
+  canonicalUrl: (token) => `https://${token.split("/")[0]}.keka.com/careers`,
+  endpoint: (token) => `https://${token.split("/")[0]}.keka.com/careers/api/embedjobs/default/active/${token.split("/")[1] ?? ""}`,
+  jobsFromBody: (body) => asRecords(body),
+  providerName: () => "",
+  async companyInfo(token, get) {
+    const body = await get(`https://${token.split("/")[0]}.keka.com/careers/api/organization/default/careerportalinfo`);
+    const website = isRecord(body) ? str(body.companyWebsite).replace(/^https?:\/\//, "").replace(/^www\./, "").split("/")[0] ?? "" : "";
+    return { name: isRecord(body) ? str(body.name) || str(body.shortName) : "", ...(website && !/keka\.com$/i.test(website) ? { website } : {}) };
+  },
+  payloadVersion: () => "keka-embedjobs:v1",
+  normalize(company, job) {
+    const places = asRecords(job.jobLocations) ?? [];
+    const location = places.map((place) => [str(place.city) || str(place.name), str(place.state), countryLabel(str(place.countryCode)) || str(place.countryName)].filter(Boolean).join(", ")).filter(Boolean).join("; ") || "Unspecified";
+    const remote = /remote/i.test(location) || job.jobType === 3;
+    return classifyJob({
+      id: `keka:${company.slug}:${str(job.id)}`, company: company.name, title: str(job.title), location,
+      remote, workMode: remote ? "remote" : "unknown",
+      eligibleCountries: [], excludedCountries: [], eligibleRegions: [], eligibilityConfidence: "unknown",
+      url: `https://${company.token.split("/")[0]}.keka.com/careers/jobdetails/${encodeURIComponent(str(job.id))}`,
+      updatedAt: str(job.publishedOn) || undefined, description: plainText(str(job.description)),
+    });
+  },
+};
+
+/** Zoho Recruit careers portals publish an RSS feed of open positions. Token is the portal host (tenant.zohorecruit.com or .in). */
+const zohorecruit: ProviderSpec = {
+  ats: "zohorecruit",
+  label: "Zoho Recruit",
+  hosts: ["zohorecruit.com", "zohorecruit.in", "zohorecruit.eu"],
+  crawlPatterns: ["*.zohorecruit.com/jobs/Careers*", "*.zohorecruit.in/jobs/Careers*"],
+  resolve(url) {
+    const match = /^([a-z0-9-]+)\.(zohorecruit\.(?:com|in|eu))$/i.exec(url.hostname);
+    if (!match || ["www", "help", "static", "img", "js", "css", "accounts"].includes(match[1]!.toLowerCase())) return null;
+    return /^\/jobs\/careers/i.test(url.pathname) ? url.hostname.toLowerCase() : null;
+  },
+  canonicalUrl: (token) => `https://${token}/jobs/Careers`,
+  endpoint: (token) => `https://${token}/jobs/Careers/rss`,
+  bodyFormat: "text",
+  jobsFromBody(body) {
+    if (typeof body !== "string" || !/<rss/i.test(body)) return null;
+    return [...body.matchAll(/<item>([\s\S]*?)<\/item>/gi)].map(([, item]) => ({
+      title: cdata(tag(item!, "title")), link: tag(item!, "link"), guid: cdata(tag(item!, "guid")), pubDate: tag(item!, "pubDate"), description: cdata(tag(item!, "description")),
+    }));
+  },
+  providerName: (_jobs, body) => (typeof body === "string" ? cdata(tag(body.split(/<item>/i)[0] ?? "", "title")).replace(/\s*[-–|]\s*careers?$/i, "").trim() : ""),
+  payloadVersion: () => "zohorecruit-rss:v1",
+  normalize(company, job) {
+    const html = str(job.description);
+    const locationText = plainText(/Location:\s*([\s\S]*?)(?:<br|<span|$)/i.exec(html)?.[1] ?? "").trim();
+    const description = plainText(html.replace(/^[\s\S]*?<span id="spandesc">/i, ""));
+    const remote = /remote/i.test(str(job.title)) || /remote/i.test(locationText);
+    const id = str(job.guid) || /\/jobs\/Careers\/(\d+)/i.exec(str(job.link))?.[1] || str(job.link);
+    const posted = Date.parse(str(job.pubDate));
+    return classifyJob({
+      id: `zohorecruit:${company.slug}:${id}`, company: company.name, title: str(job.title), location: locationText || "Unspecified",
+      remote, workMode: remote ? "remote" : "unknown",
+      eligibleCountries: [], excludedCountries: [], eligibleRegions: [], eligibilityConfidence: "unknown",
+      url: str(job.link).replace(/\?source=RSS$/i, ""), ...(Number.isFinite(posted) ? { updatedAt: new Date(posted).toISOString() } : {}), description,
+    });
+  },
+};
+
+function tag(xml: string, name: string): string { return new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)</${name}>`, "i").exec(xml)?.[1]?.trim() ?? ""; }
+function cdata(value: string): string { return value.replace(/^<!\[CDATA\[([\s\S]*?)\]\]>$/, "$1").trim(); }
+
+export const PROVIDERS: ReadonlyArray<ProviderSpec> = [smartrecruiters, workable, breezy, freshteam, keka, zohorecruit];
 
 export function providerSpec(ats: string): ProviderSpec | undefined {
   return PROVIDERS.find((spec) => spec.ats === ats);

package/src/site-admission.ts

@@ -0,0 +1,39 @@
+import { readFile } from "node:fs/promises";
+import { atomicJson } from "./atomic-file.ts";
+import type { SiteProbeReport } from "./site-probe.ts";
+import type { VerifiedCompany } from "./types.ts";
+
+export interface SiteAdmissionResult { admitted: string[]; skippedKnown: string[]; catalogSize: number }
+
+/**
+ * Promote company sites that published JobPosting markup in a probe run into the verified catalog. Identity is
+ * structural: the postings were read from pages on the company's own domain, so the evidence kind is `company_site`.
+ * A company already covered by an ATS board is skipped so it is never counted twice.
+ */
+export async function admitSites(reportPath: string, catalogPath: string, options: { minPostings?: number; now?: Date } = {}): Promise<SiteAdmissionResult> {
+  const report = JSON.parse(await readFile(reportPath, "utf8")) as SiteProbeReport;
+  const catalog = JSON.parse(await readFile(catalogPath, "utf8")) as Record<string, Omit<VerifiedCompany, "slug">>;
+  const knownDomains = new Set(Object.values(catalog).map((entry) => normalize(entry.companyDomain ?? "")).filter(Boolean));
+  const checkedAt = (options.now ?? new Date()).toISOString();
+  const result: SiteAdmissionResult = { admitted: [], skippedKnown: [], catalogSize: 0 };
+  for (const site of report.sites) {
+    if (site.postings < (options.minPostings ?? 1) || !site.careerUrl) continue;
+    const domain = normalize(site.companyDomain);
+    if (knownDomains.has(domain)) { result.skippedKnown.push(domain); continue; }
+    let slug = domain.split(".")[0]!.replace(/[^a-z0-9-]/g, "-");
+    if (catalog[slug] && normalize(catalog[slug]!.companyDomain ?? "") !== domain) slug = `${slug}-site`;
+    catalog[slug] = {
+      name: site.companyName.trim(), ats: "jobposting", token: site.careerUrl, companyDomain: domain, sourceUrl: site.careerUrl,
+      ...(site.indiaPostings > 0 ? { cohorts: ["IN"] } : {}),
+      discoveredFrom: { channel: "career_page", reference: site.careerUrl },
+      verification: { checkedAt, canonicalSourceUrl: site.careerUrl, observedCompanyName: site.companyName.trim(), identityEvidence: "company_site", contentType: "text/html", payloadVersion: "jobposting-jsonld:v1", jobCount: site.postings },
+    };
+    knownDomains.add(domain);
+    result.admitted.push(slug);
+  }
+  result.catalogSize = Object.keys(catalog).length;
+  await atomicJson(catalogPath, catalog);
+  return result;
+}
+
+function normalize(domain: string): string { return domain.toLowerCase().replace(/^www\./, ""); }

package/src/site-probe.ts

@@ -0,0 +1,43 @@
+import { readFile } from "node:fs/promises";
+import { atomicJson } from "./atomic-file.ts";
+import { crawlSite, sitePostingsToJobs, type SiteSeed } from "./jobposting-site.ts";
+
+export interface SiteProbeOptions { limit?: number; sample?: number; concurrency?: number; maxPages?: number; delayMs?: number; timeoutMs?: number }
+export interface SiteProbeReport {
+  generatedAt: string; seeds: number; checked: number; withCareersPage: number; withPostings: number; robotsBlocked: number;
+  postings: number; indiaPostings: number; withDate: number; withIdentifier: number; pagesFetched: number;
+  sites: Array<{ companyName: string; companyDomain: string; careerUrl?: string; candidatePages: number; pagesFetched: number; postings: number; indiaPostings: number; robotsBlocked: boolean; issues: number }>;
+}
+
+/** Bounded yield measurement over company seeds: how many company sites publish JobPosting markup, and how many roles that is. */
+export async function probeSites(inputPath: string, reportPath: string, options: SiteProbeOptions = {}): Promise<SiteProbeReport> {
+  const all = JSON.parse(await readFile(inputPath, "utf8")) as SiteSeed[];
+  const byDomain = new Map<string, SiteSeed>();
+  for (const seed of all) if (seed.companyDomain && !byDomain.has(seed.companyDomain)) byDomain.set(seed.companyDomain, seed);
+  let seeds = [...byDomain.values()];
+  if (options.sample && options.sample < seeds.length) { let state = 12345; const random = () => (state = (state * 1103515245 + 12345) % 2147483648) / 2147483648; seeds = [...seeds].sort(() => random() - 0.5).slice(0, options.sample); }
+  if (options.limit) seeds = seeds.slice(0, options.limit);
+  const report: SiteProbeReport = { generatedAt: new Date().toISOString(), seeds: all.length, checked: 0, withCareersPage: 0, withPostings: 0, robotsBlocked: 0, postings: 0, indiaPostings: 0, withDate: 0, withIdentifier: 0, pagesFetched: 0, sites: [] };
+  let cursor = 0;
+  async function worker() {
+    while (cursor < seeds.length) {
+      const seed = seeds[cursor++]!;
+      const result = await crawlSite(seed, { maxPages: options.maxPages, delayMs: options.delayMs ?? 300, timeoutMs: options.timeoutMs }).catch((error) => ({ robotsBlocked: false, candidatePages: 0, pagesFetched: 0, postings: [], issues: [String(error)], careerUrl: undefined }));
+      const jobs = sitePostingsToJobs({ slug: seed.companyDomain, name: seed.companyName }, result.postings);
+      const india = jobs.filter((job) => job.eligibleCountries.includes("IN")).length;
+      report.checked += 1;
+      if (result.careerUrl) report.withCareersPage += 1;
+      if (result.postings.length) report.withPostings += 1;
+      if (result.robotsBlocked) report.robotsBlocked += 1;
+      report.postings += result.postings.length; report.indiaPostings += india; report.pagesFetched += result.pagesFetched;
+      report.withDate += result.postings.filter((posting) => posting.datePosted).length;
+      report.withIdentifier += result.postings.filter((posting) => posting.identifier).length;
+      report.sites.push({ companyName: seed.companyName, companyDomain: seed.companyDomain, careerUrl: result.careerUrl, candidatePages: result.candidatePages, pagesFetched: result.pagesFetched, postings: result.postings.length, indiaPostings: india, robotsBlocked: result.robotsBlocked, issues: result.issues.length });
+      if (report.checked % 25 === 0) await atomicJson(reportPath, report);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.max(1, options.concurrency ?? 6) }, worker));
+  report.sites.sort((left, right) => right.postings - left.postings);
+  await atomicJson(reportPath, report);
+  return report;
+}

package/src/source-enrichment.ts

@@ -84,7 +84,7 @@ function isVerifiedCatalogRecord(company: unknown): boolean {
     && verification.canonicalSourceUrl === source.canonicalSourceUrl
     && typeof verification.checkedAt === "string" && Number.isFinite(Date.parse(verification.checkedAt))
     && typeof verification.observedCompanyName === "string" && verification.observedCompanyName.length > 0
-    && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect", "company_page_link"].includes(String(verification.identityEvidence))
+    && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect", "company_page_link", "provider_board", "company_site"].includes(String(verification.identityEvidence))
     && typeof verification.contentType === "string" && typeof verification.payloadVersion === "string" && verification.payloadVersion.length > 0
     && Number.isInteger(verification.jobCount) && Number(verification.jobCount) > 0;
 }