openings 0.1.2 → 0.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "A free, candidate-safe job-search substrate for AI agents",
   "license": "MIT",
   "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },

package/src/board-verification.ts

@@ -0,0 +1,180 @@
+import { atomicJson } from "./atomic-file.ts";
+import { readEnrichmentRegistry, mergeEnrichmentLeads, type EnrichmentLead, type LeadAttempt } from "./enrichment-registry.ts";
+import { withFileLock } from "./file-lock.ts";
+import { resolveSource } from "./source-verification.ts";
+import type { Ats, Company, SourceVerification } from "./types.ts";
+
+type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
+
+/** Providers whose public board is accepted as identity on its own. Workday stays on the company-identity path because its crawls are expensive. */
+export const BOARD_TIER_PROVIDERS: ReadonlySet<Ats> = new Set(["greenhouse", "lever", "ashby", "recruitee"]);
+
+export interface BoardVerificationOptions {
+  fetch?: Fetch;
+  now?: () => Date;
+  limit?: number;
+  concurrency?: number;
+  timeoutMs?: number;
+  /** Days before a board that permanently failed is probed again. */
+  cooldownDays?: number;
+}
+
+export interface BoardVerificationReport {
+  generatedAt: string;
+  considered: number;
+  selected: number;
+  skipped: { inCatalog: number; coolingDown: number; unsupported: number; deferred: number };
+  verified: number;
+  added: string[];
+  rejected: Array<{ sourceKey: string; reason: "unreachable" | "invalid_payload" | "empty_board" | "duplicate_slug"; detail: string }>;
+  catalogSize: number;
+}
+
+type CatalogEntry = Omit<Company, "slug"> & { verification: SourceVerification };
+
+/**
+ * Board-verified tier: a public ATS board is admitted on the provider's own identity (board name or tenant) without proof of
+ * the company website. Entries carry `identityEvidence: "provider_board"` and no `companyDomain`, so every consumer can tell
+ * them apart from company-verified sources.
+ */
+export async function verifyBoards(registryPath: string, catalogPath: string, options: BoardVerificationOptions = {}): Promise<BoardVerificationReport> {
+  const fetcher = options.fetch ?? globalThis.fetch;
+  const now = options.now ?? (() => new Date());
+  const limit = Math.max(1, Math.trunc(options.limit ?? 200));
+  const concurrency = Math.max(1, Math.trunc(options.concurrency ?? 8));
+  const timeoutMs = Math.max(1, Math.trunc(options.timeoutMs ?? 30_000));
+  const cooldownMs = Math.max(0, options.cooldownDays ?? 30) * 86_400_000;
+
+  return withFileLock(catalogPath, async () => {
+    const catalog = await readCatalog(catalogPath);
+    const registry = await readEnrichmentRegistry(registryPath);
+    const knownSources = new Set(Object.values(catalog).map((entry) => `${entry.ats}:${entry.token.toLowerCase()}`));
+    const skipped = { inCatalog: 0, coolingDown: 0, unsupported: 0, deferred: 0 };
+    const eligible: EnrichmentLead[] = [];
+    for (const lead of registry.leads) {
+      if (!BOARD_TIER_PROVIDERS.has(lead.ats) || !resolveSource(lead.sourceUrl)) { skipped.unsupported += 1; continue; }
+      if (knownSources.has(`${lead.ats}:${lead.token.toLowerCase()}`)) { skipped.inCatalog += 1; continue; }
+      if (looksLikeTestBoard(lead.token)) { skipped.unsupported += 1; continue; }
+      if (coolingDown(lead, now(), cooldownMs)) { skipped.coolingDown += 1; continue; }
+      eligible.push(lead);
+    }
+    const selected = eligible.slice(0, limit);
+    skipped.deferred = eligible.length - selected.length;
+
+    const usedSlugs = new Set(Object.keys(catalog));
+    const added: string[] = [];
+    const rejected: BoardVerificationReport["rejected"] = [];
+    const updatedLeads: EnrichmentLead[] = [];
+    let cursor = 0;
+    async function worker() {
+      while (cursor < selected.length) {
+        const lead = selected[cursor++]!;
+        const attemptedAt = now().toISOString();
+        try {
+          const probe = await probeBoard(lead, fetcher, timeoutMs);
+          const name = probe.providerName || lead.companyMatches[0]?.companyName || humanize(lead.token);
+          const source = resolveSource(lead.sourceUrl)!;
+          const slug = uniqueSlug(lead, usedSlugs);
+          if (!slug) {
+            rejected.push({ sourceKey: lead.sourceKey, reason: "duplicate_slug", detail: `Catalog slug already used for ${lead.token}` });
+            updatedLeads.push(withAttempt(lead, { attemptedAt, outcome: "permanent_failure", category: "duplicate_slug", detail: "slug collision" }));
+            continue;
+          }
+          usedSlugs.add(slug);
+          catalog[slug] = {
+            name, ats: lead.ats, token: source.token, sourceUrl: source.canonicalSourceUrl,
+            discoveredFrom: lead.discoveredFrom[0] ?? { channel: "dataset", reference: registryPath },
+            verification: { observedCompanyName: name, identityEvidence: "provider_board", contentType: probe.contentType, payloadVersion: probe.payloadVersion, jobCount: probe.jobCount, checkedAt: attemptedAt, canonicalSourceUrl: source.canonicalSourceUrl },
+          };
+          added.push(slug);
+          updatedLeads.push({ ...withAttempt(lead, { attemptedAt, outcome: "success", category: "provider_board", detail: `${probe.jobCount} jobs` }), promotedAt: attemptedAt });
+        } catch (error) {
+          const reason = error instanceof BoardError ? error.reason : "unreachable";
+          const detail = error instanceof Error ? error.message : String(error);
+          const outcome = error instanceof BoardError && error.reason !== "unreachable" ? "permanent_failure" : "transient_failure";
+          rejected.push({ sourceKey: lead.sourceKey, reason, detail });
+          updatedLeads.push(withAttempt(lead, { attemptedAt, outcome, category: reason, detail }));
+        }
+      }
+    }
+    await Promise.all(Array.from({ length: Math.min(concurrency, selected.length) }, worker));
+
+    if (added.length) await atomicJson(catalogPath, Object.fromEntries(Object.entries(catalog).sort(([a], [b]) => a.localeCompare(b))));
+    if (updatedLeads.length) await mergeEnrichmentLeads(registryPath, updatedLeads, now());
+    return {
+      generatedAt: now().toISOString(), considered: registry.leads.length, selected: selected.length, skipped,
+      verified: added.length, added: added.sort(), rejected: rejected.sort((a, b) => a.sourceKey.localeCompare(b.sourceKey)), catalogSize: Object.keys(catalog).length,
+    };
+  }, { operation: "verify boards" });
+}
+
+/** Random-looking tokens are almost always someone's test board: a long digit run mixed with letters, or a long consonant string. Short brand tokens like 2k or bvnk pass. */
+export function looksLikeTestBoard(token: string): boolean {
+  const value = token.toLowerCase();
+  return /[a-z]/.test(value) && (/\d{5,}/.test(value) || (value.length >= 8 && !/[aeiouy]/.test(value)));
+}
+
+class BoardError extends Error {
+  constructor(readonly reason: "unreachable" | "invalid_payload" | "empty_board", message: string) { super(message); }
+}
+
+async function probeBoard(lead: EnrichmentLead, fetcher: Fetch, timeoutMs: number): Promise<{ providerName: string; contentType: string; payloadVersion: string; jobCount: number }> {
+  const source = resolveSource(lead.sourceUrl)!;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${timeoutMs}ms`)), timeoutMs);
+  try {
+    const response = await fetcher(source.structuredEndpoint, { signal: controller.signal });
+    if (!response.ok) { await response.body?.cancel().catch(() => undefined); throw new BoardError(response.status === 404 || response.status === 410 ? "invalid_payload" : "unreachable", `HTTP ${response.status}`); }
+    let body: unknown;
+    try { body = await response.json(); } catch { throw new BoardError("invalid_payload", "Endpoint did not return JSON"); }
+    const jobs = source.ats === "lever" ? asArray(body) : asArray(isRecord(body) ? body[source.ats === "recruitee" ? "offers" : "jobs"] : undefined);
+    if (!jobs) throw new BoardError("invalid_payload", "Payload does not contain the expected jobs array");
+    if (jobs.length === 0) throw new BoardError("empty_board", "Board has no jobs");
+    const providerName = source.ats === "greenhouse" ? majority(jobs.map((job) => typeof job.company_name === "string" ? job.company_name.trim() : "").filter(Boolean)) : "";
+    const payloadVersion = source.ats === "greenhouse" ? "greenhouse-job-board:v1" : source.ats === "lever" ? "lever-postings:v0" : source.ats === "recruitee" ? "recruitee-careers:v1" : `ashby-job-board:${isRecord(body) && typeof body.apiVersion === "string" ? body.apiVersion : "unknown"}`;
+    return { providerName, contentType: response.headers.get("content-type") ?? "unknown", payloadVersion, jobCount: jobs.length };
+  } finally { clearTimeout(timer); }
+}
+
+function coolingDown(lead: EnrichmentLead, now: Date, cooldownMs: number): boolean {
+  const last = lead.attempts[lead.attempts.length - 1];
+  if (!last) return false;
+  const age = now.getTime() - Date.parse(last.attemptedAt);
+  if (last.outcome === "permanent_failure") return age < cooldownMs;
+  if (last.outcome === "transient_failure") return age < 86_400_000;
+  return false;
+}
+
+function withAttempt(lead: EnrichmentLead, attempt: LeadAttempt): EnrichmentLead {
+  return { ...lead, attempts: [...lead.attempts, attempt] };
+}
+
+function uniqueSlug(lead: EnrichmentLead, used: Set<string>): string | null {
+  const base = lead.token.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || lead.ats;
+  for (const candidate of [base, `${lead.ats}-${base}`]) if (!used.has(candidate)) return candidate;
+  return null;
+}
+
+/** "acme-corp" -> "Acme Corp". Used only when the provider exposes no company name. */
+export function humanize(token: string): string {
+  return token.split(/[-_.]+/).filter(Boolean).map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(" ");
+}
+
+async function readCatalog(path: string): Promise<Record<string, CatalogEntry>> {
+  try {
+    const parsed = JSON.parse(await Bun.file(path).text()) as unknown;
+    if (!isRecord(parsed)) throw new Error(`Catalog is not an object: ${path}`);
+    return parsed as Record<string, CatalogEntry>;
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") return {};
+    throw error;
+  }
+}
+
+function majority(values: string[]): string {
+  const counts = new Map<string, number>();
+  for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1);
+  return [...counts].sort((a, b) => b[1] - a[1])[0]?.[0] ?? "";
+}
+function asArray(value: unknown): Record<string, unknown>[] | null { return Array.isArray(value) && value.every(isRecord) ? value : null; }
+function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }

package/src/cli.ts

@@ -2,6 +2,8 @@
 import { readFile } from "node:fs/promises";
 import { relative, resolve } from "node:path";
 import { createRuntime } from "./runtime.ts";
+import { atomicJson } from "./atomic-file.ts";
+import { verifyBoards } from "./board-verification.ts";
 import { runSourceVerification } from "./source-pipeline.ts";
 import { runSourceDiscovery, runYcSourceDiscovery } from "./source-discovery.ts";
 import { discoverAndPromote } from "./source-discovery-pipeline.ts";
@@ -21,6 +23,7 @@ Usage:
   openings crawl [--country CODE | --companies FILE] [--concurrency N] [--source-cache-hours N] [--source-limit N] [--delay-ms N] [--workday-page-delay-ms N] [--data-dir PATH]
   openings snapshot export [--input FILE] [--output-dir PATH]
   openings coverage report --country CODE [--snapshot FILE] [--catalog FILE] [--candidates FILE] [--registry FILE] [--output FILE] [--as-of ISO]
+  openings sources verify-boards REGISTRY.json [--output FILE] [--limit N] [--concurrency N] [--report FILE]
   openings sources verify CANDIDATES.json [--output FILE] [--state-file FILE] [--concurrency N] [--workday-concurrency N] [--limit N] [--require-country CODE] [--registry FILE] [--retry-deferred]
   openings sources discover FEED.json [--country CODE] [--registry FILE] [--output FILE] [--catalog FILE] [--report FILE]
   openings sources discover-yc --country CODE [--registry FILE] [--output FILE] [--catalog FILE] [--report FILE]
@@ -156,6 +159,14 @@ export async function run(args: string[]): Promise<number> {
       )), null, 2));
       return 0;
     }
+    if (rest[0] === "verify-boards") {
+      const parsed = parseVerifyBoards(rest.slice(1));
+      if (typeof parsed === "string") return fail(parsed);
+      const report = await verifyBoards(parsed.registry, parsed.output, { limit: parsed.limit, concurrency: parsed.concurrency });
+      if (parsed.report) await atomicJson(parsed.report, report);
+      console.log(JSON.stringify(report, null, 2));
+      return 0;
+    }
     if (rest[0] !== "verify") return fail("sources requires a discovery, tracing, or `verify` subcommand");
     const parsed = parseSourceVerification(rest.slice(1));
     if (typeof parsed === "string") return fail(parsed);
@@ -533,6 +544,24 @@ async function readCompanyFile(path: string): Promise<string[]> {
   return (await readFile(path, "utf8")).split(/\r?\n/).map((line) => line.trim()).filter((line) => line && !line.startsWith("#"));
 }
 
+function parseVerifyBoards(args: string[]): { registry: string; output: string; limit?: number; concurrency?: number; report?: string } | string {
+  const registry = args[0];
+  if (!registry || registry.startsWith("--")) return "sources verify-boards requires a registry JSON file";
+  let output = "data/companies.json";
+  let limit: number | undefined;
+  let concurrency: number | undefined;
+  let report: string | undefined;
+  for (let index = 1; index < args.length; index += 1) {
+    const arg = args[index];
+    if (arg === "--output") output = args[++index] ?? output;
+    else if (arg === "--report") report = args[++index];
+    else if (arg === "--limit") { limit = Number(args[++index]); if (!Number.isInteger(limit) || limit < 1) return "--limit must be a positive integer"; }
+    else if (arg === "--concurrency") { concurrency = Number(args[++index]); if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 50) return "--concurrency must be an integer from 1 to 50"; }
+    else return `Unknown option: ${arg}`;
+  }
+  return { registry, output, limit, concurrency, report };
+}
+
 function parseSourceVerification(args: string[]) {
   const candidatesPath = args[0];
   if (!candidatesPath || candidatesPath.startsWith("--")) return "sources verify requires a candidate JSON file";

package/src/country-coverage.ts

@@ -111,7 +111,7 @@ async function readCatalog(path: string): Promise<Record<string, Omit<VerifiedCo
   if (!isRecord(value)) throw new Error(`Invalid verified catalog: ${path}`);
   for (const [slug, company] of Object.entries(value)) {
     if (!isRecord(company) || !["greenhouse", "lever", "ashby", "workday", "recruitee"].includes(String(company.ats)) || typeof company.token !== "string"
-      || typeof company.companyDomain !== "string" || typeof company.sourceUrl !== "string"
+      || (typeof company.companyDomain !== "string" && !(isRecord(company.verification) && company.verification.identityEvidence === "provider_board")) || typeof company.sourceUrl !== "string"
       || company.cohorts !== undefined && (!Array.isArray(company.cohorts) || !company.cohorts.every(validCountryCode))
       || !validVerification(company.verification)) throw new Error(`Invalid verified catalog source: ${slug}`);
   }

package/src/job-coverage.ts

@@ -40,7 +40,7 @@ export function projectJobCoverage(sources: Company[], snapshot: JobSnapshot, co
         country,
         indexedSourcesWithEligibleJobs: eligibleSources.length,
         eligibleJobs: eligibleSources.reduce((total, { jobs }) => total + jobs.filter((job) => isEligibleForCountry(job, country)).length, 0),
-        distinctEligibleEmployers: new Set(eligibleSources.flatMap(({ source }) => source.companyDomain ? [normalizeDomain(source.companyDomain)] : [])).size,
+        distinctEligibleEmployers: new Set(eligibleSources.flatMap(({ source }) => source.companyDomain ? [normalizeDomain(source.companyDomain)] : source.verification?.identityEvidence === "provider_board" ? [`board:${source.ats}:${source.token.toLowerCase()}`] : [])).size,
       };
     }),
   };

package/src/mcp.ts

@@ -22,7 +22,7 @@ export function createMcpHandler(tools: ToolHandler) {
     const base = { jsonrpc: "2.0" as const, id: request.id ?? null };
     try {
       if (request.method === "initialize") {
-        return { ...base, result: { protocolVersion: "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: "openings", version: "0.1.2" } } };
+        return { ...base, result: { protocolVersion: "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: "openings", version: "0.1.3" } } };
       }
       if (request.method === "ping") return { ...base, result: {} };
       if (request.method === "tools/list") return { ...base, result: { tools: tools.list() } };

package/src/types.ts

@@ -39,7 +39,7 @@ export interface SourceVerification {
   checkedAt: string;
   canonicalSourceUrl: string;
   observedCompanyName: string;
-  identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect";
+  identityEvidence: "provider_company_name" | "provider_tenant" | "structured_domain_link" | "company_redirect" | "provider_board";
   contentType: string;
   payloadVersion: string;
   jobCount: number;