jd-intel 0.9.0 → 0.10.0

package/src/adapters/workday.js

@@ -1,5 +1,8 @@
 import { normalize } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { makeLocationMatcher } from '../filters.js';
+import { atsFetch } from '../http.js';
+import { orgHost } from '../boards.js';
 
 const MAX_DETAIL_FETCHES = 100;
 const LIST_PAGE_SIZE = 20;
@@ -30,7 +33,9 @@ const MULTI_LOCATION = /^\s*\d+\s+locations?\s*$/;
  * detail set.
  *
  * @param {string} slug - normalized company slug (registry routing key)
- * @param {object} [ctx] - { config:{tenant,env,site}, companyName, filterContext }
+ * @param {object} [ctx] - { config:{tenant,env,site}, companyName, filterContext, report };
+ *   report is called once, after hydration, with
+ *   { ats, listed, prefiltered, hydrated, capped, org_name, org_url } when given
  * @returns {Promise<Array>} Normalized job objects
  */
 export async function fetchWorkday(slug, ctx = {}) {
@@ -46,12 +51,25 @@ export async function fetchWorkday(slug, ctx = {}) {
   let offset = 0;
   let pages = 0;
   let firstTotal = 0;
-  while (pages < LIST_PAGE_HARD_CAP) {
-    const resp = await fetch(`${base}/jobs`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ appliedFacets: {}, limit: LIST_PAGE_SIZE, offset, searchText: '' }),
-    });
+  let listCapped = false;
+  while (true) {
+    if (pages >= LIST_PAGE_HARD_CAP) {
+      listCapped = true;
+      break;
+    }
+    let resp;
+    try {
+      resp = await atsFetch(`${base}/jobs`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ appliedFacets: {}, limit: LIST_PAGE_SIZE, offset, searchText: '' }),
+      });
+    } catch (err) {
+      // A 429 or 5xx that outlasted the retries, or a network failure.
+      // After the first page, keep the postings already read.
+      if (offset === 0) throw err;
+      break;
+    }
 
     if (!resp.ok) {
       if (offset === 0) {
@@ -83,21 +101,23 @@ export async function fetchWorkday(slug, ctx = {}) {
     const re = new RegExp(fc.titleFilter, 'i');
     candidates = candidates.filter(p => re.test(p.title || ''));
   }
+  // Location rows go through the applyFilters matcher, so the pre-filter
+  // keeps exactly the rows the pass after hydration would (issue #61).
+  // "2 Locations" says nothing about where: the row stays a candidate
+  // through both filters and that later pass decides on the detail's
+  // location list.
   if (Array.isArray(fc.locationIncludes) && fc.locationIncludes.length > 0) {
-    const inc = fc.locationIncludes.map(s => String(s).toLowerCase());
+    const inc = fc.locationIncludes.map(makeLocationMatcher);
     candidates = candidates.filter(p => {
       const loc = (p.locationsText || '').toLowerCase();
-      // "2 Locations" says nothing about where. The row stays a candidate
-      // and the pass after hydration decides on the detail's location list.
-      if (MULTI_LOCATION.test(loc)) return true;
-      return inc.some(s => loc.includes(s));
+      return MULTI_LOCATION.test(loc) || inc.some(m => m(loc));
     });
   }
   if (Array.isArray(fc.locationExcludes) && fc.locationExcludes.length > 0) {
-    const exc = fc.locationExcludes.map(s => String(s).toLowerCase());
+    const exc = fc.locationExcludes.map(makeLocationMatcher);
     candidates = candidates.filter(p => {
       const loc = (p.locationsText || '').toLowerCase();
-      return !exc.some(s => loc.includes(s));
+      return MULTI_LOCATION.test(loc) || !exc.some(m => m(loc));
     });
   }
   if (typeof fc.postedWithinDays === 'number') {
@@ -120,23 +140,29 @@ export async function fetchWorkday(slug, ctx = {}) {
   const limit = typeof fc.limit === 'number' && fc.limit > 0 ? fc.limit : 100;
   const skip = typeof fc.offset === 'number' && fc.offset > 0 ? fc.offset : 0;
   const cap = fc.filter ? MAX_DETAIL_FETCHES : Math.min(skip + limit, MAX_DETAIL_FETCHES);
-  candidates = candidates.slice(0, cap);
+  const hydrate = candidates.slice(0, cap);
 
-  // 4. Hydrate descriptions via the per-posting detail endpoint.
-  const jobs = await Promise.all(candidates.map(async (p) => {
+  // 4. Hydrate descriptions via the per-posting detail endpoint. The detail
+  //    also carries `hiringOrganization: { name, url }` next to
+  //    jobPostingInfo; the list does not. Kept per posting in list order so
+  //    the one reported is the first hydrated posting's, not whichever
+  //    detail answered first (a tenant can post under several entities).
+  const orgs = [];
+  const jobs = await Promise.all(hydrate.map(async (p, i) => {
     const externalPath = p.externalPath || ''; // already begins with '/job/...'
     let info = {};
     try {
       // externalPath already carries the '/job/...' segment, so it is
       // concatenated directly onto the CXS base. Inserting another
       // '/job' here yields '/job/job/...' which Workday rejects (422).
-      const dResp = await fetch(`${base}${externalPath}`);
+      const dResp = await atsFetch(`${base}${externalPath}`);
       if (dResp.ok) {
         const detail = await dResp.json();
         info = detail.jobPostingInfo || {};
+        orgs[i] = detail.hiringOrganization || null;
       }
     } catch {
-      // detail failed: fall back to list fields, empty description
+      // detail failed, retries included: fall back to list fields, empty description
     }
 
     return normalize({
@@ -160,6 +186,21 @@ export async function fetchWorkday(slug, ctx = {}) {
     }, 'workday');
   }));
 
+  // Nothing hydrated (a filter miss, an empty site) means no detail was
+  // read, so the org is unknown rather than absent: null, null.
+  if (typeof ctx.report === 'function') {
+    const org = orgs.find(Boolean) || {};
+    ctx.report({
+      ats: 'workday',
+      listed: postings.length,
+      prefiltered: candidates.length,
+      hydrated: hydrate.length,
+      capped: listCapped || hydrate.length < candidates.length,
+      org_name: org.name || null,
+      org_url: orgHost(org.url),
+    });
+  }
+
   return jobs;
 }
 

package/src/boards.js

@@ -0,0 +1,80 @@
+/**
+ * The boards[] entry of a fetchJobsDetailed result (issues #58, #60, #87).
+ *
+ * A board is one (ats, slug) the library fetched, with what came back. The
+ * fields fall in three groups: what the registry or the caller said about
+ * it (`name`, `site`), what the fetch found (`jobs_found`, `matched`,
+ * `scan`), and what the board says about itself (`org_name`, `org_url`).
+ * Only the adapters can read the third group, and they hand it over through
+ * ctx.report. Both fields are null where the platform exposes nothing, and
+ * they are never filled from the slug or the registry name: a slug is an
+ * address, not a confirmed identity.
+ */
+
+const BOARD_URLS = {
+  greenhouse: (slug) => `https://boards.greenhouse.io/${slug}`,
+  lever: (slug) => `https://jobs.lever.co/${slug}`,
+  ashby: (slug) => `https://jobs.ashbyhq.com/${slug}`,
+  smartrecruiters: (slug) => `https://careers.smartrecruiters.com/${slug}`,
+  teamtailor: (slug) => `https://${slug}.teamtailor.com`,
+  recruitee: (slug) => `https://${slug}.recruitee.com`,
+  workday: (slug, config) => (config ? `https://${config.tenant}.${config.env}.myworkdayjobs.com/${config.site}` : null),
+};
+
+// Domains the platforms own. A link there (boards.greenhouse.io,
+// jobs.lever.co, testco.recruitee.com, cisco.wd5.myworkdayjobs.com) says
+// which ATS hosts the board, nothing about whose board it is.
+const ATS_DOMAINS = ['greenhouse.io', 'lever.co', 'ashbyhq.com', 'smartrecruiters.com', 'teamtailor.com', 'recruitee.com', 'myworkdayjobs.com'];
+
+/**
+ * The page a person opens to see the board, or null when the ATS is unknown
+ * or, for Workday, no {tenant, env, site} is at hand.
+ */
+export function boardUrl(ats, slug, config) {
+  const build = BOARD_URLS[ats];
+  return build ? build(slug, config) : null;
+}
+
+/**
+ * The bare hostname a board's link points at ("jobs.example.com"), for
+ * org_url. Null when the link is missing or malformed, and null when the
+ * host belongs to an ATS, since that carries no signal about the company.
+ */
+export function orgHost(link) {
+  let host;
+  try {
+    host = new URL(link).hostname.toLowerCase();
+  } catch {
+    return null;
+  }
+  if (!host || ATS_DOMAINS.some(d => host === d || host.endsWith(`.${d}`))) return null;
+  return host;
+}
+
+/**
+ * @param {object} board
+ * @param {string} board.ats
+ * @param {string} board.slug - The slug the adapter was called with (canonical casing on a registry hit)
+ * @param {string|null} board.name - Registry row name; null for a probe or an override
+ * @param {object} [board.config] - Workday {tenant, env, site}, when one was used
+ * @param {string|null} board.org_name - The organization name the ATS response states, else null
+ * @param {string|null} board.org_url - The careers or company host the board links to (see orgHost), else null
+ * @param {number} board.jobs_found - Rows the board listed before any filter: the list count an adapter reported through ctx.report when it filters before hydrating, else the rows it returned
+ * @param {number} board.matched - Rows left after filters, before offset and limit
+ * @param {object|null} board.scan - The { listed, prefiltered, hydrated, capped } counts the adapter reported through ctx.report, else null
+ */
+export function describeBoard({ ats, slug, name = null, config, org_name = null, org_url = null, jobs_found, matched = 0, scan = null }) {
+  return {
+    ats,
+    slug,
+    name,
+    site: ats === 'workday' && config ? config.site : null,
+    board_url: boardUrl(ats, slug, config),
+    org_name,
+    org_url,
+    jobs_found,
+    matched,
+    selected: true,
+    scan,
+  };
+}

package/src/cli.js

@@ -12,7 +12,7 @@
 import { realpathSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { fetchJobs } from './index.js';
-import { detectAts, searchRegistry } from './registry.js';
+import { detectAtsDetailed, searchRegistry } from './registry.js';
 
 const [,, command, ...args] = process.argv;
 
@@ -113,13 +113,17 @@ async function main() {
       const company = args[0];
       if (!company) { console.error('Usage: jd-intel detect <company>'); process.exit(1); }
       console.log(`Detecting ATS for ${company}...`);
-      const results = await detectAts(company);
-      if (results.length === 0) {
-        console.log('No ATS board found for this company.');
-      } else {
-        for (const r of results) {
-          console.log(`  Found: ${r.ats} (slug: ${r.slug})`);
-        }
+      const { boards, failed } = await detectAtsDetailed(company);
+      for (const b of boards) {
+        console.log(`  Found: ${b.ats} (slug: ${b.slug}, ${b.source === 'registry' ? 'in the registry' : 'live probe'})`);
+      }
+      for (const f of failed) {
+        console.log(`  Could not check ${f.ats}: ${f.message}`);
+      }
+      if (boards.length === 0) {
+        console.log(failed.length > 0
+          ? 'No ATS board confirmed. At least one check failed, so this is not a definite miss. Retry in a moment.'
+          : 'No ATS board found for this company.');
       }
       break;
     }

package/src/errors.js

@@ -34,6 +34,20 @@ export class AtsError extends Error {
   }
 }
 
+/**
+ * Thrown when a call cannot proceed because of its arguments: a missing
+ * company, an unknown ATS name, a regex that does not compile. Carries
+ * `code: 'invalid_args'` so callers tell a bad request from a failed fetch
+ * (AtsError) without reading the message.
+ */
+export class ArgumentError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'ArgumentError';
+    this.code = ERROR_CODES.INVALID_ARGS;
+  }
+}
+
 /**
  * Helper for adapters: build an AtsError from an HTTP status (429 => rate
  * limited, anything else => unreachable) with the given message.

package/src/filters.js

@@ -1,3 +1,5 @@
+import { ArgumentError } from './errors.js';
+
 /**
  * Apply filters to a list of normalized jobs.
  *
@@ -23,30 +25,32 @@ export function applyFilters(jobs, options = {}) {
  * @returns {{ jobs: Array, total_matched: number }}
  */
 export function applyFiltersDetailed(jobs, options = {}) {
-  const {
-    titleFilter,
-    filter,
-    postedWithinDays,
-    locationIncludes,
-    locationExcludes,
-    order = 'newest',
-    offset = 0,
-    limit = 100,
-  } = options;
+  const { order = 'newest', offset = 0, limit = 100 } = options;
+  const matched = filterJobs(jobs, options);
+  return { jobs: pageJobs(matched, { order, offset, limit }), total_matched: matched.length };
+}
+
+/**
+ * The filter step on its own: every job that passes titleFilter, filter,
+ * postedWithinDays and the location filters, in the order given. No sort
+ * and no paging, so fetchJobsDetailed can count the matches per board
+ * before the page cut removes them.
+ */
+export function filterJobs(jobs, options = {}) {
+  const { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes } = options;
+  const { title, topic } = compileFilterPatterns({ titleFilter, filter });
 
   let result = jobs;
 
-  if (titleFilter) {
-    const pattern = new RegExp(titleFilter, 'i');
-    result = result.filter(j => pattern.test(j.title || ''));
+  if (title) {
+    result = result.filter(j => title.test(j.title || ''));
   }
 
-  if (filter) {
-    const pattern = new RegExp(filter, 'i');
+  if (topic) {
     result = result.filter(j =>
-      pattern.test(j.title || '') ||
-      pattern.test(j.department || '') ||
-      pattern.test(j.description || '')
+      topic.test(j.title || '') ||
+      topic.test(j.department || '') ||
+      topic.test(j.description || '')
     );
   }
 
@@ -69,7 +73,14 @@ export function applyFiltersDetailed(jobs, options = {}) {
     result = result.filter(j => !jobLocations(j).every(loc => matchers.some(m => m(loc))));
   }
 
-  const total_matched = result.length;
+  return result;
+}
+
+/**
+ * Sort, then cut the page (see applyFiltersDetailed for the order rules).
+ */
+export function pageJobs(jobs, { order = 'newest', offset = 0, limit = 100 } = {}) {
+  let result = jobs;
 
   if (order !== 'board') {
     result = [...result].sort(byNewest);
@@ -81,7 +92,30 @@ export function applyFiltersDetailed(jobs, options = {}) {
     result = result.slice(start, end);
   }
 
-  return { jobs: result, total_matched };
+  return result;
+}
+
+/**
+ * Compile the two regex arguments, or throw ArgumentError naming the one
+ * that does not compile. Both are case-insensitive. fetchJobsDetailed calls
+ * this before its first request, so a bad pattern is reported as a bad
+ * argument and costs no upstream traffic.
+ *
+ * @returns {{ title: RegExp|null, topic: RegExp|null }}
+ */
+export function compileFilterPatterns({ titleFilter, filter } = {}) {
+  return {
+    title: titleFilter ? compilePattern(titleFilter, 'titleFilter') : null,
+    topic: filter ? compilePattern(filter, 'filter') : null,
+  };
+}
+
+function compilePattern(source, name) {
+  try {
+    return new RegExp(source, 'i');
+  } catch (err) {
+    throw new ArgumentError(`${name}: ${err.message}`);
+  }
 }
 
 /**
@@ -118,14 +152,19 @@ function byNewest(a, b) {
 }
 
 /**
- * Build a matcher for a single location keyword.
+ * Build a matcher for a single location keyword. The matcher takes a
+ * lowercased location string.
  *
  * Short tokens (≤4 chars) use word-boundary matching to prevent substring
  * collisions like "US" matching "Australia", "Brussels", "Belarus", or "UK"
- * matching "Auckland". Longer tokens use substring matching so phrases like
+ * matching "Ukraine". Longer tokens use substring matching so phrases like
  * "United States" can match "United States of America".
+ *
+ * Exported for the Workday list pre-filter, so one rule (trim, empty
+ * keywords never match, word boundaries for short tokens) applies before
+ * and after detail hydration (issue #61).
  */
-function makeLocationMatcher(needle) {
+export function makeLocationMatcher(needle) {
   const lower = (needle || '').toLowerCase().trim();
   if (!lower) return () => false;
   if (lower.length <= 4) {

package/src/http.js

@@ -0,0 +1,184 @@
+import { AtsError, ERROR_CODES, atsErrorFromStatus } from './errors.js';
+
+/**
+ * The one HTTP door for every adapter request (issue #7).
+ *
+ * atsFetch() wraps the global fetch with the politeness every ATS expects
+ * and the failure handling the adapters used to leave out:
+ *   - a timeout per attempt on the wait for the response to start
+ *   - retries with exponential backoff and jitter on 429, any 5xx, and
+ *     network errors (DNS, reset, timeout), honoring Retry-After; a
+ *     certificate failure is thrown at once, since it cannot pass later
+ *   - a cap on requests in flight per host
+ *
+ * Every other status resolves normally, so an adapter keeps its own 404
+ * handling. Once the retries are used up the caller gets an AtsError:
+ * rate_limited for a 429, ats_unreachable for a 5xx or a network failure.
+ * The global fetch is read on every attempt so a test's mock of it applies.
+ *
+ * The timer stops once the headers are in. Reading the body is the caller's
+ * step (resp.json() in the adapter), and it is not timed: a signal left on
+ * the request would abort that read too, so a large board on a slow link
+ * would fail at the timeout with a raw TimeoutError thrown from resp.json(),
+ * outside this retry loop, where master downloaded it fine. undici's own
+ * body timeout (300s idle) still ends a stream that stalls.
+ */
+
+const DEFAULTS = {
+  timeoutMs: 10_000,
+  retries: 3,   // attempts per request in total; 1 turns retrying off
+  perHost: 4,   // requests in flight per hostname
+  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+};
+
+const BASE_BACKOFF_MS = 1000; // 1s, 2s, 4s, ...
+const JITTER_MS = 250;
+const MAX_RETRY_AFTER_MS = 30_000;
+
+let settings = { ...DEFAULTS };
+
+/**
+ * Replace the HTTP settings with the defaults plus `overrides`. For tests
+ * and scripts. `configureHttp()` restores the defaults. Returns a copy of
+ * the settings now in force.
+ */
+export function configureHttp(overrides = {}) {
+  settings = { ...DEFAULTS, ...overrides };
+  return { ...settings };
+}
+
+/**
+ * fetch(url, init) with a timeout, retries and a per-host queue.
+ *
+ * Resolves with the Response for any status that is not retried (2xx, 3xx,
+ * and 4xx other than 429). Throws AtsError once the retries are used up on
+ * a 429 or 5xx (with `.status`), or on a network error or a timeout waiting
+ * for the response to start.
+ */
+export async function atsFetch(url, init = {}) {
+  const host = new URL(url).hostname;
+  const { retries, sleep, timeoutMs } = settings;
+
+  for (let attempt = 1; ; attempt++) {
+    let resp;
+    try {
+      resp = await withHostSlot(host, () => fetchWithTimeout(url, init, timeoutMs));
+    } catch (err) {
+      if (attempt >= retries || isCertError(err)) {
+        const error = new AtsError(
+          ERROR_CODES.ATS_UNREACHABLE,
+          `${host}: ${describeCause(err, timeoutMs)} after ${attempts(attempt)}`
+        );
+        error.cause = err;
+        throw error;
+      }
+      await sleep(backoffMs(attempt));
+      continue;
+    }
+
+    if (!isRetried(resp.status)) return resp;
+    if (attempt >= retries) {
+      throw atsErrorFromStatus(resp.status, `${host}: HTTP ${resp.status} after ${attempts(attempt)}`);
+    }
+    await sleep(retryAfterMs(resp) ?? backoffMs(attempt));
+  }
+}
+
+/**
+ * Three-state probe outcome for an adapter's has(): true on 2xx, false on
+ * a 404, and an AtsError for anything else (401, 403, ...), so a board the
+ * probe could not check never reads as "not here" (issue #55). A 429 or
+ * 5xx never reaches this point: atsFetch throws on those itself.
+ */
+export function probeResult(resp, label) {
+  if (resp.ok) return true;
+  if (resp.status === 404) return false;
+  throw atsErrorFromStatus(resp.status, `${label}: ${resp.status}`);
+}
+
+// The abort covers connecting and waiting for the headers, and is cleared as
+// soon as fetch resolves so the body read that follows is never aborted
+// (see the header comment). The reason is a TimeoutError like the one
+// AbortSignal.timeout would raise, so describeCause reads both the same way.
+async function fetchWithTimeout(url, init, timeoutMs) {
+  const controller = new AbortController();
+  const timer = setTimeout(
+    () => controller.abort(new DOMException(`Timed out after ${timeoutMs}ms`, 'TimeoutError')),
+    timeoutMs
+  );
+  try {
+    return await globalThis.fetch(url, { ...init, signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function isRetried(status) {
+  return status === 429 || status >= 500;
+}
+
+function backoffMs(attempt) {
+  return BASE_BACKOFF_MS * 2 ** (attempt - 1) + Math.floor(Math.random() * JITTER_MS);
+}
+
+// Retry-After is either delay-seconds or an HTTP-date. Mocked responses may
+// carry no headers at all.
+function retryAfterMs(resp) {
+  const raw = resp.headers?.get?.('retry-after');
+  if (!raw) return null;
+  const seconds = Number(raw);
+  const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(raw) - Date.now();
+  if (!Number.isFinite(ms)) return null;
+  return Math.min(Math.max(ms, 0), MAX_RETRY_AFTER_MS);
+}
+
+// A certificate that fails validation fails the same way on the next
+// attempt, so retrying it only costs time. Seen live: every
+// {slug}.eu.teamtailor.com answers ERR_TLS_CERT_ALTNAME_INVALID.
+const CERT_ERROR = /^(?:ERR_TLS_CERT_ALTNAME_INVALID|CERT_HAS_EXPIRED|CERT_NOT_YET_VALID|DEPTH_ZERO_SELF_SIGNED_CERT|SELF_SIGNED_CERT_IN_CHAIN|UNABLE_TO_VERIFY_LEAF_SIGNATURE|UNABLE_TO_GET_ISSUER_CERT(?:_LOCALLY)?)$/;
+
+function isCertError(err) {
+  return CERT_ERROR.test(err?.cause?.code || '');
+}
+
+// undici reports socket failures as TypeError('fetch failed') with the OS
+// code on `cause`, and rejects an aborted request with the signal's reason,
+// here the TimeoutError from fetchWithTimeout.
+function describeCause(err, timeoutMs) {
+  if (err?.name === 'TimeoutError') return `timed out after ${timeoutMs}ms`;
+  const detail = err?.cause?.code || err?.cause?.message;
+  const message = err?.message || String(err);
+  return detail ? `${message} (${detail})` : message;
+}
+
+function attempts(n) {
+  return `${n} attempt${n === 1 ? '' : 's'}`;
+}
+
+// Per-host queue. A finished request hands its slot straight to the next
+// waiter (the count never dips in between), so the cap holds even when a new
+// caller arrives while a waiter is being woken.
+const hosts = new Map(); // hostname -> { active, waiting: [resolve] }
+
+async function withHostSlot(host, run) {
+  let slot = hosts.get(host);
+  if (!slot) {
+    slot = { active: 0, waiting: [] };
+    hosts.set(host, slot);
+  }
+  if (slot.active >= settings.perHost) {
+    await new Promise((resolve) => slot.waiting.push(resolve));
+  } else {
+    slot.active += 1;
+  }
+  try {
+    return await run();
+  } finally {
+    const next = slot.waiting.shift();
+    if (next) next();
+    else {
+      slot.active -= 1;
+      if (slot.active === 0) hosts.delete(host);
+    }
+  }
+}