openings 0.1.63 → 0.1.65

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.63",
+  "version": "0.1.65",
   "description": "A free, candidate-safe job-search substrate for AI agents",
   "license": "MIT",
   "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },

package/src/catalog.ts

@@ -258,14 +258,23 @@ async function fetchWorkdayJobs(company: Company, fetcher: Fetch, signal?: Abort
       previousPageStart = pacingNow();
     } finally { release(); }
   }
-  async function page(offset: number, appliedFacets: Record<string, string[]> = {}): Promise<{ total: number; jobs: WorkdayJob[]; facets?: unknown }> {
+  async function page(offset: number, appliedFacets: Record<string, string[]> = {}): Promise<{ total: number; jobs: WorkdayJob[]; facets?: unknown; unusable?: number }> {
     for (let attempt = 0; attempt < 3; attempt += 1) {
       await pacePageStart();
       const response = await fetcher(endpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ appliedFacets, limit, offset, searchText: "" }), signal });
       if (response.ok) {
         const body = await response.json() as { total?: unknown; jobPostings?: unknown; facets?: unknown };
         if (!Number.isInteger(body.total) || !Array.isArray(body.jobPostings)) throw new Error(`${company.name} Workday source returned an invalid payload`);
-        return { total: body.total as number, jobs: body.jobPostings as WorkdayJob[], facets: body.facets };
+        // Workday began returning postings with no externalPath on 13 October, and every one of them threw
+        // inside workdayJobKey, taking the whole tenant down with it: 48 sources failed in an afternoon. A
+        // posting with no path cannot produce an apply link, so it is of no use here either way — but it is
+        // dropped loudly, because a provider quietly shedding postings is how a source empties itself.
+        const postings = (body.jobPostings as WorkdayJob[]).filter((job) => typeof job?.externalPath === "string" && job.externalPath.length > 0);
+        if (postings.length < body.jobPostings.length) {
+          console.error(`${company.name} Workday source: skipped ${body.jobPostings.length - postings.length} posting(s) with no externalPath`);
+        }
+        return { total: body.total as number, jobs: postings, facets: body.facets,
+                 unusable: (body.jobPostings as unknown[]).length - postings.length };
       }
       if (!isTransientStatus(response.status) || attempt === 2) { await response.body?.cancel().catch(() => undefined); throw new Error(`${company.name} job board returned HTTP ${response.status}`); }
       const delayMs = retryDelayMs(response, attempt);
@@ -279,6 +288,7 @@ async function fetchWorkdayJobs(company: Company, fetcher: Fetch, signal?: Abort
   if (first.total < first.jobs.length || first.total > 100_000) throw new Error(`${company.name} Workday source reported an invalid total`);
   const offsets = Array.from({ length: Math.ceil(first.total / limit) - 1 }, (_, index) => (index + 1) * limit);
   const pages = new Array<WorkdayJob[]>(offsets.length);
+  const pageUnusable: number[] = [];
   let cursor = 0;
   async function worker() {
     while (cursor < offsets.length) {
@@ -287,12 +297,17 @@ async function fetchWorkdayJobs(company: Company, fetcher: Fetch, signal?: Abort
       const result = await page(offset);
       if (result.jobs.length === 0 && offset < first.total) throw new Error(`${company.name} Workday source truncated at offset ${offset} of ${first.total}`);
       pages[index] = result.jobs;
+      pageUnusable[index] = result.unusable ?? 0;
     }
   }
   const workerCount = pageDelayMs > 0 ? 1 : Math.min(4, offsets.length);
   await Promise.all(Array.from({ length: workerCount }, worker));
-  const jobs = [first.jobs, ...pages].flat().slice(0, first.total);
-  if (jobs.length !== first.total) throw new Error(`${company.name} Workday source returned ${jobs.length} of ${first.total} jobs`);
+  // Postings Workday sent without a path were never usable, so they are not evidence of a truncated read.
+  // The check still fires for a genuinely short page, which is what stops a partial crawl emptying a tenant.
+  const unusable = (first.unusable ?? 0) + pages.reduce((sum, _, index) => sum + (pageUnusable[index] ?? 0), 0);
+  const expected = first.total - unusable;
+  const jobs = [first.jobs, ...pages].flat().slice(0, expected);
+  if (jobs.length !== expected) throw new Error(`${company.name} Workday source returned ${jobs.length} of ${expected} jobs`);
   // Two reasons to run a country pass with the tenant's own filter: past 2,000 postings the unfiltered listing stops,
   // and multi-location postings read "2 Locations" there, so their countries are invisible without the filter.
   const countryFacets = workdayCountryFacets(first.facets);

package/src/crawl-reporting.ts

@@ -27,8 +27,12 @@ export function resolveAggregatorUrl(value: string | undefined): string | undefi
 }
 
 /** Posts each successfully crawled partition to the aggregator. Only job data is sent, never resume content. */
-export function createCrawlReporter(options: { url: string; fetcher?: Fetch; timeoutMs?: number; attempts?: number; sleep?: (ms: number) => Promise<void> }) {
+export function createCrawlReporter(options: { url: string; fetcher?: Fetch; timeoutMs?: number; attempts?: number; secret?: string; sleep?: (ms: number) => Promise<void> }) {
   const fetcher = options.fetcher ?? globalThis.fetch;
+  // Without this the only thing standing between a stranger and an employer's listings is the slug, ats and
+  // token triple — all three of which are published in data/companies.json inside the npm tarball. An empty
+  // report for a real slug empties that employer from the site and answers 200.
+  const secret = options.secret ?? process.env.OPENINGS_INGEST_SECRET;
   const target = endpoint(options.url, "v1/crawls");
   const attempts = Math.max(1, options.attempts ?? 3);
   const sleep = options.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
@@ -42,7 +46,8 @@ export function createCrawlReporter(options: { url: string; fetcher?: Fetch; tim
       try {
         const response = await fetcher(target, {
           method: "POST",
-          headers: { "content-type": "application/json", "content-encoding": "gzip" },
+          headers: { "content-type": "application/json", "content-encoding": "gzip",
+                     ...(secret ? { authorization: `Bearer ${secret}` } : {}) },
           body,
           // Big employers send tens of megabytes and the aggregator ingests one report at a time; give them room.
           signal: AbortSignal.timeout(options.timeoutMs ?? 120_000),

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = "0.1.63";
+export const VERSION = "0.1.65";