openings 0.1.17 → 0.1.19

package/src/site-probe.ts

@@ -0,0 +1,43 @@
+import { readFile } from "node:fs/promises";
+import { atomicJson } from "./atomic-file.ts";
+import { crawlSite, sitePostingsToJobs, type SiteSeed } from "./jobposting-site.ts";
+
+export interface SiteProbeOptions { limit?: number; sample?: number; concurrency?: number; maxPages?: number; delayMs?: number; timeoutMs?: number }
+export interface SiteProbeReport {
+  generatedAt: string; seeds: number; checked: number; withCareersPage: number; withPostings: number; robotsBlocked: number;
+  postings: number; indiaPostings: number; withDate: number; withIdentifier: number; pagesFetched: number;
+  sites: Array<{ companyName: string; companyDomain: string; careerUrl?: string; candidatePages: number; pagesFetched: number; postings: number; indiaPostings: number; robotsBlocked: boolean; issues: number }>;
+}
+
+/** Bounded yield measurement over company seeds: how many company sites publish JobPosting markup, and how many roles that is. */
+export async function probeSites(inputPath: string, reportPath: string, options: SiteProbeOptions = {}): Promise<SiteProbeReport> {
+  const all = JSON.parse(await readFile(inputPath, "utf8")) as SiteSeed[];
+  const byDomain = new Map<string, SiteSeed>();
+  for (const seed of all) if (seed.companyDomain && !byDomain.has(seed.companyDomain)) byDomain.set(seed.companyDomain, seed);
+  let seeds = [...byDomain.values()];
+  if (options.sample && options.sample < seeds.length) { let state = 12345; const random = () => (state = (state * 1103515245 + 12345) % 2147483648) / 2147483648; seeds = [...seeds].sort(() => random() - 0.5).slice(0, options.sample); }
+  if (options.limit) seeds = seeds.slice(0, options.limit);
+  const report: SiteProbeReport = { generatedAt: new Date().toISOString(), seeds: all.length, checked: 0, withCareersPage: 0, withPostings: 0, robotsBlocked: 0, postings: 0, indiaPostings: 0, withDate: 0, withIdentifier: 0, pagesFetched: 0, sites: [] };
+  let cursor = 0;
+  async function worker() {
+    while (cursor < seeds.length) {
+      const seed = seeds[cursor++]!;
+      const result = await crawlSite(seed, { maxPages: options.maxPages, delayMs: options.delayMs ?? 300, timeoutMs: options.timeoutMs }).catch((error) => ({ robotsBlocked: false, candidatePages: 0, pagesFetched: 0, postings: [], issues: [String(error)], careerUrl: undefined }));
+      const jobs = sitePostingsToJobs({ slug: seed.companyDomain, name: seed.companyName }, result.postings);
+      const india = jobs.filter((job) => job.eligibleCountries.includes("IN")).length;
+      report.checked += 1;
+      if (result.careerUrl) report.withCareersPage += 1;
+      if (result.postings.length) report.withPostings += 1;
+      if (result.robotsBlocked) report.robotsBlocked += 1;
+      report.postings += result.postings.length; report.indiaPostings += india; report.pagesFetched += result.pagesFetched;
+      report.withDate += result.postings.filter((posting) => posting.datePosted).length;
+      report.withIdentifier += result.postings.filter((posting) => posting.identifier).length;
+      report.sites.push({ companyName: seed.companyName, companyDomain: seed.companyDomain, careerUrl: result.careerUrl, candidatePages: result.candidatePages, pagesFetched: result.pagesFetched, postings: result.postings.length, indiaPostings: india, robotsBlocked: result.robotsBlocked, issues: result.issues.length });
+      if (report.checked % 25 === 0) await atomicJson(reportPath, report);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.max(1, options.concurrency ?? 6) }, worker));
+  report.sites.sort((left, right) => right.postings - left.postings);
+  await atomicJson(reportPath, report);
+  return report;
+}

package/src/source-enrichment.ts

@@ -84,7 +84,7 @@ function isVerifiedCatalogRecord(company: unknown): boolean {
     && verification.canonicalSourceUrl === source.canonicalSourceUrl
     && typeof verification.checkedAt === "string" && Number.isFinite(Date.parse(verification.checkedAt))
     && typeof verification.observedCompanyName === "string" && verification.observedCompanyName.length > 0
-    && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect", "company_page_link"].includes(String(verification.identityEvidence))
+    && ["provider_company_name", "provider_tenant", "structured_domain_link", "company_redirect", "company_page_link", "provider_board", "company_site"].includes(String(verification.identityEvidence))
     && typeof verification.contentType === "string" && typeof verification.payloadVersion === "string" && verification.payloadVersion.length > 0
     && Number.isInteger(verification.jobCount) && Number(verification.jobCount) > 0;
 }

package/src/source-verification.ts

@@ -58,7 +58,7 @@ export async function verifyCandidates(candidates: SourceCandidate[], options: V
       const providerFetch = providerCooldowns.get(source.ats)!.wrap(fetcher);
       try {
         const evidence = await probe(candidate, source, providerFetch, timeoutMs, options.resolveHost, options.headTransport, options.pageTransport);
-        const replayed = evidence.identityEvidence === "company_redirect" || evidence.identityEvidence === "company_page_link";
+        const replayed = evidence.identityEvidence === "company_redirect" || evidence.identityEvidence === "company_page_link" || evidence.providerLinksDomain;
         if (!replayed && !identityMatches(candidate.companyName, candidate.companyDomain, evidence.observedCompanyName, source.token)) {
           rejected.push({ index, value: rejection(candidate, "identity_mismatch", `Expected ${candidate.companyName}; observed ${evidence.observedCompanyName}`) });
           continue;
@@ -164,23 +164,32 @@ async function probe(candidate: SourceCandidate, source: ResolvedSource, fetcher
       throw new VerificationError("unreachable", `HTTP ${response.status}`);
     }
     const contentType = response.headers.get("content-type") ?? "unknown";
-    let body: unknown;
-    try { body = await response.json(); } catch { throw new VerificationError("invalid_payload", "Endpoint did not return JSON"); }
     const spec = providerSpec(source.ats);
+    let body: unknown;
+    try { body = spec?.bodyFormat === "text" ? await response.text() : await response.json(); } catch { throw new VerificationError("invalid_payload", "Endpoint did not return JSON"); }
     const jobs = spec ? spec.jobsFromBody(body) : source.ats === "greenhouse" ? recordArray(body, "jobs") : source.ats === "lever" ? array(body) : source.ats === "workday" ? recordArray(body, "jobPostings") : source.ats === "recruitee" ? recordArray(body, "offers") : recordArray(body, "jobs");
     if (!jobs) throw new VerificationError("invalid_payload", "Payload does not contain the expected jobs array");
     if (jobs.length === 0) throw new VerificationError("empty_board", "Source has no jobs, so identity cannot be verified");
-    const providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
+    let providerName = spec ? spec.providerName(jobs, body) : source.ats === "greenhouse" ? majority(jobs.map((job) => stringField(job, "company_name")).filter(Boolean)) : source.ats === "workday" ? workday!.tenant : "";
+    let infoWebsite = "";
+    if (spec?.companyInfo) {
+      const info = await spec.companyInfo(source.token, async (url, format = "json") => { const reply = await fetchProbeWithRetry(fetcher, url, { signal: controller.signal }); if (!reply.ok) { await reply.body?.cancel().catch(() => undefined); throw new VerificationError("unreachable", `HTTP ${reply.status}`); } return format === "text" ? reply.text() : reply.json(); }).catch((): { name: string; website?: string } => ({ name: "" }));
+      providerName = providerName || info.name; infoWebsite = info.website ?? "";
+    }
+    const expectedDomain = candidate.companyDomain.toLowerCase().replace(/^www\./, "");
+    // The provider's own tenant record names the company website: that is the company's claim on its own board, stronger than any name heuristic.
+    const providerLinksDomain = Boolean(infoWebsite) && (infoWebsite === expectedDomain || infoWebsite.endsWith(`.${expectedDomain}`));
     const tenantMatches = source.ats === "workday" && identityMatches(candidate.companyName, candidate.companyDomain, workday!.tenant, source.token);
     const namedProvider = source.ats === "greenhouse" || tenantMatches || Boolean(spec && providerName);
-    const hasDomainLink = !namedProvider && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
+    const hasDomainLink = providerLinksDomain || !namedProvider && structuredIdentityLinksDomain(jobs, candidate.companyDomain);
     const hasRedirectEvidence = !namedProvider && await verifiedCompanyRedirect(candidate, source, resolveHost, headTransport, timeoutMs);
     const hasPageLink = !namedProvider && !hasDomainLink && !hasRedirectEvidence && await verifiedCompanyPageLink(candidate, source, resolveHost, pageTransport, timeoutMs);
     if (!namedProvider && !hasDomainLink && !hasRedirectEvidence && !hasPageLink) throw new VerificationError("identity_mismatch", `Neither structured identity fields, a verified company redirect, nor a company careers-page link point to ${candidate.companyDomain}`);
     const observedCompanyName = namedProvider && providerName ? providerName : candidate.companyName;
     return {
       observedCompanyName,
-      identityEvidence: tenantMatches ? "provider_tenant" as const : (namedProvider && providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : hasRedirectEvidence ? "company_redirect" as const : "company_page_link" as const),
+      providerLinksDomain,
+      identityEvidence: providerLinksDomain ? "structured_domain_link" as const : tenantMatches ? "provider_tenant" as const : (namedProvider && providerName ? "provider_company_name" as const : hasDomainLink ? "structured_domain_link" as const : hasRedirectEvidence ? "company_redirect" as const : "company_page_link" as const),
       contentType,
       payloadVersion: spec ? spec.payloadVersion(body) : source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "workday" ? "workday-cxs:v1" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`,
       jobCount: source.ats === "workday" && isRecord(body) && typeof body.total === "number" ? body.total : jobs.length,

package/src/types.ts

@@ -1,4 +1,5 @@
-export const ALL_PROVIDERS = ["greenhouse", "lever", "ashby", "workday", "recruitee", "smartrecruiters", "workable", "breezy", "freshteam"] as const;
+/** ATS boards plus "jobposting": a company's own career site read only through its schema.org JobPosting markup (token = careers URL). */
+export const ALL_PROVIDERS = ["greenhouse", "lever", "ashby", "workday", "recruitee", "smartrecruiters", "workable", "breezy", "freshteam", "keka", "zohorecruit", "jobposting"] as const;
 export type Ats = (typeof ALL_PROVIDERS)[number];
 
 export interface DomainEvidence {
@@ -40,7 +41,7 @@ export interface SourceVerification {
   checkedAt: string;
   canonicalSourceUrl: string;
   observedCompanyName: string;
-  identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect" | "company_page_link" | "provider_board";
+  identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect" | "company_page_link" | "provider_board" | "company_site";
   contentType: string;
   payloadVersion: string;
   jobCount: number;

package/src/update-check.ts

@@ -0,0 +1,39 @@
+import { VERSION } from "./version.ts";
+
+export interface UpdateNotice { installed: string; latest: string; run: string; message: string }
+type Fetch = (input: string, init?: RequestInit) => Promise<Response>;
+
+export const UPDATE_COMMAND = "bun add --global openings";
+
+/**
+ * One small request to the npm registry at startup, never blocking a tool call. The result rides on every tool
+ * response so the AI app can prompt the person to update; the server itself never modifies the install.
+ */
+export function startUpdateCheck(options: { current?: string; fetcher?: Fetch; timeoutMs?: number; enabled?: boolean } = {}): { get(): UpdateNotice | null; ready: Promise<void> } {
+  const current = options.current ?? VERSION;
+  let notice: UpdateNotice | null = null;
+  const ready = options.enabled === false ? Promise.resolve() : (async () => {
+    try {
+      const response = await (options.fetcher ?? globalThis.fetch)("https://registry.npmjs.org/openings/latest", { signal: AbortSignal.timeout(options.timeoutMs ?? 5_000), headers: { accept: "application/json" } });
+      if (!response.ok) return;
+      const body = await response.json() as { version?: unknown };
+      if (typeof body.version === "string" && isNewer(body.version, current)) notice = updateNotice(current, body.version);
+    } catch { /* offline or blocked: no notice */ }
+  })();
+  return { get: () => notice, ready };
+}
+
+export function updateNotice(installed: string, latest: string): UpdateNotice {
+  return { installed, latest, run: UPDATE_COMMAND, message: `Openings ${latest} is available; ${installed} is installed. Tell the person to run "${UPDATE_COMMAND}" and restart their AI app to get fixes and new coverage.` };
+}
+
+/** Numeric dot-version comparison; pre-release suffixes are ignored. */
+export function isNewer(candidate: string, current: string): boolean {
+  const parse = (value: string) => value.split(".").map((part) => Number.parseInt(part, 10) || 0);
+  const [left, right] = [parse(candidate), parse(current)];
+  for (let index = 0; index < Math.max(left.length, right.length); index += 1) {
+    const difference = (left[index] ?? 0) - (right[index] ?? 0);
+    if (difference !== 0) return difference > 0;
+  }
+  return false;
+}

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = "0.1.17";
+export const VERSION = "0.1.19";