openings 0.1.60 → 0.1.62

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.60",
+  "version": "0.1.62",
   "description": "A free, candidate-safe job-search substrate for AI agents",
   "license": "MIT",
   "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },

package/src/board-verification.ts

@@ -1,3 +1,4 @@
+import excludedData from "../data/excluded.json" with { type: "json" };
 import { atomicJson } from "./atomic-file.ts";
 import { readEnrichmentRegistry, mergeEnrichmentLeads, type EnrichmentLead, type LeadAttempt } from "./enrichment-registry.ts";
 import { withFileLock } from "./file-lock.ts";
@@ -7,6 +8,22 @@ import type { Ats, Company, SourceVerification } from "./types.ts";
 
 type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
 
+/**
+ * Sources that must never be admitted, by provider and token.
+ *
+ * Deleting a row from the catalog does not hold: the lead stays in the registry and the next verification run
+ * puts it straight back, which is how two employers excluded on terms grounds could have quietly returned. The
+ * reason travels with the entry so a later reader does not have to guess why.
+ */
+const EXCLUDED: ReadonlySet<string> = new Set(
+  excludedData.entries.map((entry) => `${entry.ats}:${entry.token.toLowerCase()}`),
+);
+
+/** Why a given source is excluded, for reporting. */
+export function exclusionReason(ats: string, token: string): string | undefined {
+  return excludedData.entries.find((entry) => entry.ats === ats && entry.token.toLowerCase() === token.toLowerCase())?.reason;
+}
+
 /** Providers whose public board is accepted as identity on its own. Workday boards are identified by tenant; their crawls are heavier but they carry the large employers. */
 export const BOARD_TIER_PROVIDERS: ReadonlySet<Ats> = new Set(["greenhouse", "lever", "ashby", "recruitee", "smartrecruiters", "workable", "breezy", "workday", "freshteam", "keka", "zohorecruit"]);
 
@@ -24,7 +41,7 @@ export interface BoardVerificationReport {
   generatedAt: string;
   considered: number;
   selected: number;
-  skipped: { inCatalog: number; coolingDown: number; unsupported: number; deferred: number };
+  skipped: { inCatalog: number; coolingDown: number; unsupported: number; deferred: number; excluded: number };
   verified: number;
   added: string[];
   rejected: Array<{ sourceKey: string; reason: "unreachable" | "invalid_payload" | "empty_board" | "duplicate_slug"; detail: string }>;
@@ -50,10 +67,11 @@ export async function verifyBoards(registryPath: string, catalogPath: string, op
     const catalog = await readCatalog(catalogPath);
     const registry = await readEnrichmentRegistry(registryPath);
     const knownSources = new Set(Object.values(catalog).map((entry) => `${entry.ats}:${entry.token.toLowerCase()}`));
-    const skipped = { inCatalog: 0, coolingDown: 0, unsupported: 0, deferred: 0 };
+    const skipped = { inCatalog: 0, coolingDown: 0, unsupported: 0, deferred: 0, excluded: 0 };
     const eligible: EnrichmentLead[] = [];
     for (const lead of registry.leads) {
       if (!BOARD_TIER_PROVIDERS.has(lead.ats) || !resolveSource(lead.sourceUrl)) { skipped.unsupported += 1; continue; }
+      if (EXCLUDED.has(`${lead.ats}:${lead.token.toLowerCase()}`)) { skipped.excluded += 1; continue; }
       if (knownSources.has(`${lead.ats}:${lead.token.toLowerCase()}`)) { skipped.inCatalog += 1; continue; }
       // Keka tokens contain tenant/UUID; the provider-issued routing UUID is not a company name.
       const boardName = lead.ats === "keka" ? lead.token.split("/")[0]! : lead.token;

package/src/common-crawl-discovery.ts

@@ -8,7 +8,9 @@ import type { Ats } from "./types.ts";
 import { assertArtifactFile } from "./artifact-path.ts";
 
 type Fetch = (input: string | URL, init?: RequestInit) => Promise<Response>;
-interface Options { country?: string; fetch?: Fetch; indexUrl?: string; registryPath?: string; provider?: Ats; indexRecordLimit?: number; sampleTokenLimit?: number; excludeTokens?: string[] }
+interface Options {
+  /** Backoff between index retries. Only set in tests, where real seconds are dead time. */
+  retryDelayMs?: number; country?: string; fetch?: Fetch; indexUrl?: string; registryPath?: string; provider?: Ats; indexRecordLimit?: number; sampleTokenLimit?: number; excludeTokens?: string[] }
 interface Lead { sourceUrl: string; ats: string; token: string; discoveredFrom: { channel: "dataset"; reference: string } }
 interface Collection { id?: unknown; "cdx-api"?: unknown }
 
@@ -26,6 +28,10 @@ export interface CommonCrawlDiscoveryReport extends ReportMeta {
   rejected: number;
   truncated: boolean;
   truncatedPatterns: string[];
+  /** Patterns the index had no captures for, so a later run can tell an empty region from a missing one. */
+  emptyPatterns: string[];
+  /** Patterns the index refused, with the status it gave. Reported rather than thrown, so one bad pattern cannot discard the sweep. */
+  failedPatterns: Array<{ pattern: string; status: number }>;
   candidatesPath: string;
   reportPath: string;
   registryPath?: string;
@@ -62,6 +68,8 @@ export async function discoverCommonCrawlSources(candidatesPath: string, reportP
   const seenUrls = new Set<string>();
   const rejections: Array<{ value: string; reason: string }> = [];
   const truncatedPatterns: string[] = [];
+  const emptyPatterns: string[] = [];
+  const failedPatterns: Array<{ pattern: string; status: number }> = [];
   let indexRecordsExamined = 0;
 
   for (const pattern of options.provider ? providerPatterns[options.provider] : patterns) {
@@ -75,10 +83,20 @@ export async function discoverCommonCrawlSources(candidatesPath: string, reportP
     let response = await fetcher(query);
     for (let attempt = 1; !response.ok && response.status >= 500 && attempt <= 3; attempt += 1) { // the index server sheds load with 502s; back off and retry
       await response.body?.cancel().catch(() => undefined);
-      await new Promise((resolve) => setTimeout(resolve, attempt * 5_000));
+      await new Promise((resolve) => setTimeout(resolve, attempt * (options.retryDelayMs ?? 5_000)));
       response = await fetcher(query);
     }
-    if (!response.ok) throw new Error(`Common Crawl index returned HTTP ${response.status} for ${pattern}`);
+    // The index answers 404 when a pattern matched nothing at all. That is an empty region, not a broken run.
+    if (response.status === 404) { await response.body?.cancel().catch(() => undefined); emptyPatterns.push(pattern); continue; }
+    // Anything else that survived the retries is this pattern's problem, not the run's. A sparse pattern asked
+    // for too many records times out (collapse and filter make the index do real work), and throwing there
+    // discarded every pattern that had already answered. A partial sweep is worth keeping; only a sweep where
+    // nothing at all succeeded is a failure.
+    if (!response.ok) {
+      await response.body?.cancel().catch(() => undefined);
+      failedPatterns.push({ pattern, status: response.status });
+      continue;
+    }
     const lines = (await response.text()).split(/\r?\n/).filter(Boolean).slice(0, indexRecordLimit);
     indexRecordsExamined += lines.length;
     if (lines.length >= indexRecordLimit) truncatedPatterns.push(pattern);
@@ -93,6 +111,12 @@ export async function discoverCommonCrawlSources(candidatesPath: string, reportP
     }
   }
 
+  // Every pattern refused is a broken sweep, not an empty one, and must not be reported as a quiet zero.
+  const attempted = options.provider ? providerPatterns[options.provider] : patterns;
+  if (failedPatterns.length === attempted.length && attempted.length > 0) {
+    throw new Error(`Common Crawl index refused every pattern: ${failedPatterns.map((f) => `${f.pattern} HTTP ${f.status}`).join(", ")}`);
+  }
+
   const availableLeads: Lead[] = [];
   const excludedTokens = new Set((options.excludeTokens ?? []).map((token) => token.toLowerCase()));
   let alreadyKnown = 0;
@@ -113,7 +137,7 @@ export async function discoverCommonCrawlSources(candidatesPath: string, reportP
   const registry = options.registryPath ? await mergeEnrichmentLeads(options.registryPath, registryLeads) : { added: 0, bytes: undefined, lockHeldMs: undefined };
   const report: CommonCrawlDiscoveryReport = stampReport("common-crawl-discovery:1", 1, {
     country, provider: options.provider, index, indexRecordsExamined, sampleTokenLimit, sampleShortfall: sampleTokenLimit === undefined ? 0 : Math.max(0, sampleTokenLimit - leads.length), urlsSeen: seenUrls.size, sourcesFound: found.size, alreadyKnown, unresolved: leads.length, rejected: rejections.length,
-    truncated: truncatedPatterns.length > 0, truncatedPatterns,
+    truncated: truncatedPatterns.length > 0, truncatedPatterns, emptyPatterns, failedPatterns,
     candidatesPath, reportPath, registryPath: options.registryPath, registryAdded: registry.added, registryBytes: registry.bytes, registryLockHeldMs: registry.lockHeldMs, availableLeads, leads, rejections,
   });
   await atomicJson(reportPath, report);

package/src/locations.ts

@@ -106,6 +106,12 @@ const US_STATE_CODES = new Set(["AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE",
 const US_CITY_STATE = /,\s*([A-Z]{2})(?=\s*(?:$|,|\d{5}|\(|\/|-))/g;
 /** Workday's "ST-CITY, street" and "ST - City" shapes, e.g. "IN-INDIANAPOLIS, 220 VIRGINIA AVE" or "IN - Indianapolis". */
 const US_STATE_PREFIX = /^([A-Z]{2})\s*-\s*(?=[A-Za-z])/;
+/**
+ * Workday also writes the country in front of the state: "US-IN-FT WAYNE-150A ~ 1010 Production Rd".
+ * US_STATE_PREFIX reads the leading "US" as the state, finds it is not one, and gives up, so Fort Wayne
+ * was filed under India on the strength of a bare "IN" further along the string.
+ */
+const US_COUNTRY_STATE_PREFIX = /^US\s*-\s*([A-Z]{2})\s*-\s*(?=[A-Za-z0-9])/;
 
 function detectCountries(location: string): string[] {
   const byName: string[] = [];
@@ -117,7 +123,7 @@ function detectCountries(location: string): string[] {
   // "Indianapolis, IN" is Indiana, not India: a bare code in the US "City, ST" position is a state unless a country name
   // or a known Indian place says otherwise.
   // Case matters: "Berlin, de" is a lowercase country code, "Gary, IN" is a state.
-  const prefix = US_STATE_PREFIX.exec(location)?.[1];
+  const prefix = US_COUNTRY_STATE_PREFIX.exec(location)?.[1] ?? US_STATE_PREFIX.exec(location)?.[1];
   const stateCodes = [...[...location.matchAll(US_CITY_STATE)].map((match) => match[1]!), ...(prefix ? [prefix] : [])].filter((code) => US_STATE_CODES.has(code));
   if (stateCodes.length && byName.length === 0 && !INDIA_PATTERN.test(location)) {
     return [...new Set(["US", ...byCode.filter((code) => !US_STATE_CODES.has(code))])];

package/src/providers.ts

@@ -210,7 +210,15 @@ const oraclecloud: ProviderSpec = {
   ats: "oraclecloud",
   label: "Oracle Cloud Recruiting",
   hosts: ["oraclecloud.com"],
-  crawlPatterns: ["*.fa.oraclecloud.com/hcmUI/CandidateExperience*"],
+  // One pattern per Fusion region. A wildcard in the middle of the host plus a deep path suffix is more than
+  // the Common Crawl index will serve: "*.fa.oraclecloud.com/hcmUI/CandidateExperience*" answered 504 every
+  // time, which is why Oracle had no discovered tenants at all despite being supported. Per-region host
+  // patterns answer 200, and the CandidateExperience path is filtered from the returned URLs instead.
+  crawlPatterns: ["*.fa.us2.oraclecloud.com/*", "*.fa.us6.oraclecloud.com/*", "*.fa.us9.oraclecloud.com/*",
+                  "*.fa.em2.oraclecloud.com/*", "*.fa.em3.oraclecloud.com/*", "*.fa.em4.oraclecloud.com/*",
+                  "*.fa.ap1.oraclecloud.com/*", "*.fa.ap2.oraclecloud.com/*", "*.fa.ap3.oraclecloud.com/*",
+                  "*.fa.ca2.oraclecloud.com/*", "*.fa.ca3.oraclecloud.com/*", "*.fa.uk1.oraclecloud.com/*",
+                  "*.fa.sa1.oraclecloud.com/*"],
   resolve(url) {
     if (!ORACLE_HOST.test(url.hostname)) return null;
     const site = /\/sites\/([A-Za-z0-9_]{2,32})/.exec(url.pathname)?.[1] ?? url.searchParams.get("siteNumber");

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = "0.1.60";
+export const VERSION = "0.1.62";