jd-intel 0.8.3 → 0.10.0

package/src/http.js

@@ -0,0 +1,184 @@
+import { AtsError, ERROR_CODES, atsErrorFromStatus } from './errors.js';
+
+/**
+ * The one HTTP door for every adapter request (issue #7).
+ *
+ * atsFetch() wraps the global fetch with the politeness every ATS expects
+ * and the failure handling the adapters used to leave out:
+ *   - a timeout per attempt on the wait for the response to start
+ *   - retries with exponential backoff and jitter on 429, any 5xx, and
+ *     network errors (DNS, reset, timeout), honoring Retry-After; a
+ *     certificate failure is thrown at once, since it cannot pass later
+ *   - a cap on requests in flight per host
+ *
+ * Every other status resolves normally, so an adapter keeps its own 404
+ * handling. Once the retries are used up the caller gets an AtsError:
+ * rate_limited for a 429, ats_unreachable for a 5xx or a network failure.
+ * The global fetch is read on every attempt so a test's mock of it applies.
+ *
+ * The timer stops once the headers are in. Reading the body is the caller's
+ * step (resp.json() in the adapter), and it is not timed: a signal left on
+ * the request would abort that read too, so a large board on a slow link
+ * would fail at the timeout with a raw TimeoutError thrown from resp.json(),
+ * outside this retry loop, where master downloaded it fine. undici's own
+ * body timeout (300s idle) still ends a stream that stalls.
+ */
+
+const DEFAULTS = {
+  timeoutMs: 10_000,
+  retries: 3,   // attempts per request in total; 1 turns retrying off
+  perHost: 4,   // requests in flight per hostname
+  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+};
+
+const BASE_BACKOFF_MS = 1000; // 1s, 2s, 4s, ...
+const JITTER_MS = 250;
+const MAX_RETRY_AFTER_MS = 30_000;
+
+let settings = { ...DEFAULTS };
+
+/**
+ * Replace the HTTP settings with the defaults plus `overrides`. For tests
+ * and scripts. `configureHttp()` restores the defaults. Returns a copy of
+ * the settings now in force.
+ */
+export function configureHttp(overrides = {}) {
+  settings = { ...DEFAULTS, ...overrides };
+  return { ...settings };
+}
+
+/**
+ * fetch(url, init) with a timeout, retries and a per-host queue.
+ *
+ * Resolves with the Response for any status that is not retried (2xx, 3xx,
+ * and 4xx other than 429). Throws AtsError once the retries are used up on
+ * a 429 or 5xx (with `.status`), or on a network error or a timeout waiting
+ * for the response to start.
+ */
+export async function atsFetch(url, init = {}) {
+  const host = new URL(url).hostname;
+  const { retries, sleep, timeoutMs } = settings;
+
+  for (let attempt = 1; ; attempt++) {
+    let resp;
+    try {
+      resp = await withHostSlot(host, () => fetchWithTimeout(url, init, timeoutMs));
+    } catch (err) {
+      if (attempt >= retries || isCertError(err)) {
+        const error = new AtsError(
+          ERROR_CODES.ATS_UNREACHABLE,
+          `${host}: ${describeCause(err, timeoutMs)} after ${attempts(attempt)}`
+        );
+        error.cause = err;
+        throw error;
+      }
+      await sleep(backoffMs(attempt));
+      continue;
+    }
+
+    if (!isRetried(resp.status)) return resp;
+    if (attempt >= retries) {
+      throw atsErrorFromStatus(resp.status, `${host}: HTTP ${resp.status} after ${attempts(attempt)}`);
+    }
+    await sleep(retryAfterMs(resp) ?? backoffMs(attempt));
+  }
+}
+
+/**
+ * Three-state probe outcome for an adapter's has(): true on 2xx, false on
+ * a 404, and an AtsError for anything else (401, 403, ...), so a board the
+ * probe could not check never reads as "not here" (issue #55). A 429 or
+ * 5xx never reaches this point: atsFetch throws on those itself.
+ */
+export function probeResult(resp, label) {
+  if (resp.ok) return true;
+  if (resp.status === 404) return false;
+  throw atsErrorFromStatus(resp.status, `${label}: ${resp.status}`);
+}
+
+// The abort covers connecting and waiting for the headers, and is cleared as
+// soon as fetch resolves so the body read that follows is never aborted
+// (see the header comment). The reason is a TimeoutError like the one
+// AbortSignal.timeout would raise, so describeCause reads both the same way.
+async function fetchWithTimeout(url, init, timeoutMs) {
+  const controller = new AbortController();
+  const timer = setTimeout(
+    () => controller.abort(new DOMException(`Timed out after ${timeoutMs}ms`, 'TimeoutError')),
+    timeoutMs
+  );
+  try {
+    return await globalThis.fetch(url, { ...init, signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function isRetried(status) {
+  return status === 429 || status >= 500;
+}
+
+function backoffMs(attempt) {
+  return BASE_BACKOFF_MS * 2 ** (attempt - 1) + Math.floor(Math.random() * JITTER_MS);
+}
+
+// Retry-After is either delay-seconds or an HTTP-date. Mocked responses may
+// carry no headers at all.
+function retryAfterMs(resp) {
+  const raw = resp.headers?.get?.('retry-after');
+  if (!raw) return null;
+  const seconds = Number(raw);
+  const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(raw) - Date.now();
+  if (!Number.isFinite(ms)) return null;
+  return Math.min(Math.max(ms, 0), MAX_RETRY_AFTER_MS);
+}
+
+// A certificate that fails validation fails the same way on the next
+// attempt, so retrying it only costs time. Seen live: every
+// {slug}.eu.teamtailor.com answers ERR_TLS_CERT_ALTNAME_INVALID.
+const CERT_ERROR = /^(?:ERR_TLS_CERT_ALTNAME_INVALID|CERT_HAS_EXPIRED|CERT_NOT_YET_VALID|DEPTH_ZERO_SELF_SIGNED_CERT|SELF_SIGNED_CERT_IN_CHAIN|UNABLE_TO_VERIFY_LEAF_SIGNATURE|UNABLE_TO_GET_ISSUER_CERT(?:_LOCALLY)?)$/;
+
+function isCertError(err) {
+  return CERT_ERROR.test(err?.cause?.code || '');
+}
+
+// undici reports socket failures as TypeError('fetch failed') with the OS
+// code on `cause`, and rejects an aborted request with the signal's reason,
+// here the TimeoutError from fetchWithTimeout.
+function describeCause(err, timeoutMs) {
+  if (err?.name === 'TimeoutError') return `timed out after ${timeoutMs}ms`;
+  const detail = err?.cause?.code || err?.cause?.message;
+  const message = err?.message || String(err);
+  return detail ? `${message} (${detail})` : message;
+}
+
+function attempts(n) {
+  return `${n} attempt${n === 1 ? '' : 's'}`;
+}
+
+// Per-host queue. A finished request hands its slot straight to the next
+// waiter (the count never dips in between), so the cap holds even when a new
+// caller arrives while a waiter is being woken.
+const hosts = new Map(); // hostname -> { active, waiting: [resolve] }
+
+async function withHostSlot(host, run) {
+  let slot = hosts.get(host);
+  if (!slot) {
+    slot = { active: 0, waiting: [] };
+    hosts.set(host, slot);
+  }
+  if (slot.active >= settings.perHost) {
+    await new Promise((resolve) => slot.waiting.push(resolve));
+  } else {
+    slot.active += 1;
+  }
+  try {
+    return await run();
+  } finally {
+    const next = slot.waiting.shift();
+    if (next) next();
+    else {
+      slot.active -= 1;
+      if (slot.active === 0) hosts.delete(host);
+    }
+  }
+}

package/src/index.js

@@ -7,12 +7,40 @@
  */
 
 import { ADAPTERS, ATS_NAMES } from './adapters/index.js';
-import { loadRegistry, searchRegistry, detectAts, findAtsBySlug, findEntryBySlug, getRegistrySource } from './registry.js';
-import { applyFilters } from './filters.js';
+import { loadRegistry, searchRegistry, detectAts, detectAtsDetailed, findAtsBySlug, findEntryBySlug, getRegistrySource, normSlug } from './registry.js';
+import { filterJobs, pageJobs, compileFilterPatterns } from './filters.js';
+import { AtsError, ArgumentError, ERROR_CODES } from './errors.js';
+import { describeBoard } from './boards.js';
+
+// The counts an adapter reports about its scan. boards[].scan carries these
+// and nothing else; org_name and org_url from the same report go on the
+// board itself.
+const SCAN_KEYS = ['listed', 'prefiltered', 'hydrated', 'capped'];
 
 /**
  * Fetch jobs from a company's ATS board.
  *
+ * Same options as fetchJobsDetailed; returns the page as an array.
+ *
+ * Discovery that found no board while at least one check failed throws an
+ * AtsError (rate_limited when any failure was a 429, else ats_unreachable)
+ * instead of returning []: an array has no place for `failed`, and an empty
+ * one would read as "not found" (issue #55).
+ *
+ * @returns {Promise<Array>} Normalized, filtered job objects
+ */
+export async function fetchJobs(options = {}) {
+  const result = await fetchJobsDetailed(options);
+  if (result.match === 'probe' && result.boards.length === 0 && result.failed.length > 0) {
+    throw discoveryFailure(options.company, result.failed);
+  }
+  return result.jobs;
+}
+
+/**
+ * Fetch jobs from a company's ATS board, with the counts and the boards
+ * behind the page.
+ *
  * @param {Object} options
  * @param {string} options.company - Company slug or name
  * @param {string} [options.ats] - Specific ATS platform. If omitted, auto-detects.
@@ -20,12 +48,34 @@ import { applyFilters } from './filters.js';
  * @param {string} [options.titleFilter] - Regex matched against title only. Use for role identity ("product manager", "staff engineer").
  * @param {string} [options.filter] - Regex matched across title, department, description. Use for topic/scope.
  * @param {number} [options.postedWithinDays] - Only return jobs posted within N days.
- * @param {string[]} [options.locationIncludes] - Keep jobs whose location contains any of these (case-insensitive).
- * @param {string[]} [options.locationExcludes] - Drop jobs whose location contains any of these (case-insensitive).
- * @param {number} [options.limit=100] - Maximum jobs to return after filtering.
- * @returns {Promise<Array>} Normalized, filtered job objects
+ * @param {string[]} [options.locationIncludes] - Keep jobs where any listed location contains any of these (case-insensitive).
+ * @param {string[]} [options.locationExcludes] - Drop jobs only when every listed location contains one of these (case-insensitive).
+ * @param {'newest'|'board'} [options.order='newest'] - 'newest': by postedAt descending, undated last, ties by id. 'board': the adapter's own order.
+ * @param {number} [options.offset=0] - Matches to skip after sorting (paging).
+ * @param {number} [options.limit=100] - Maximum jobs to return after offset.
+ * @returns {Promise<{
+ *   jobs: Array,
+ *   total_matched: number,
+ *   total_before_filters: number,
+ *   match: 'registry'|'probe'|'workday_override',
+ *   company: { key: string, name: string }|null,
+ *   boards: Array<object>,
+ *   failed: Array<{ ats: string, slug: string, name: string|null, code: string, message: string }>,
+ * }>}
+ *   jobs: the page. total_matched: matches before offset and limit.
+ *   total_before_filters: rows every board listed before any filter (on
+ *   Workday and SmartRecruiters the list count, not the rows they hydrated).
+ *   match: how the company was resolved. company: the registry row's name
+ *   and its key (normalized name); null unless match is 'registry'.
+ *   boards: one entry per board that answered (see src/boards.js), with
+ *   jobs_found and matched counted per board, and org_name and org_url as
+ *   the board states them (null where its ATS exposes nothing). failed:
+ *   adapters that threw an AtsError during discovery; on a registry hit
+ *   the error propagates.
+ * @throws {ArgumentError} No company, unknown ats, or a filter regex that does not compile.
+ * @throws {AtsError} The board's ATS failed on a registry hit, an explicit ats, or a Workday override.
  */
-export async function fetchJobs({
+export async function fetchJobsDetailed({
   company,
   ats,
   config,
@@ -34,66 +84,135 @@ export async function fetchJobs({
   postedWithinDays,
   locationIncludes,
   locationExcludes,
+  order = 'newest',
+  offset = 0,
   limit = 100,
 } = {}) {
-  if (!company) throw new Error('Company slug required');
+  if (!company) throw new ArgumentError('company is required');
+  compileFilterPatterns({ titleFilter, filter });
 
-  // Unified slug normalization: strip all non-alphanumeric (matches detectAts)
-  const slug = company.toLowerCase().replace(/[^a-z0-9]/g, '');
+  const slug = normSlug(company);
+  const filters = { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes };
 
-  // Filter context is passed as an additive 2nd arg to adapters. Existing
-  // adapters declare fetch{Name}(slug) and ignore extra positional args
-  // (JS no-op), so this is backward-compatible. Filter-aware adapters
-  // (e.g. Workday) use it to avoid mass detail-fetching on huge tenants.
-  const filterContext = { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes, limit };
+  // Which boards to fetch, and how the company was resolved. A registry hit
+  // and a Workday override are one board each and their errors propagate.
+  // Discovery (not in the registry, no ats) asks every adapter, keeps the
+  // ones that answered, and records the ones that failed.
+  let match = 'probe';
+  let discovery = false;
+  let targets;
 
-  let jobs;
   if (ats) {
-    const adapter = ADAPTERS[ats];
-    if (!adapter) throw new Error(`Unknown ATS: ${ats}. Supported: ${ATS_NAMES.join(', ')}`);
+    if (!ADAPTERS[ats]) throw new ArgumentError(`Unknown ATS: ${ats}. Supported: ${ATS_NAMES.join(', ')}`);
     // Explicit ATS: an explicitly passed config wins (the only path that
     // can reach a Workday company not in the registry). With no explicit
-    // config, fall back to the registry so config-keyed adapters
-    // (Workday) and canonically-cased registry slugs (SmartRecruiters
-    // "Visa") also work on the explicit path, not just under auto-detect.
-    let fetchSlug = slug;
-    let cfg = config;
-    let companyName;
-    if (!cfg) {
-      const hit = await findEntryBySlug(slug);
-      if (hit && hit.ats === ats) {
-        fetchSlug = hit.entry.slug;
-        cfg = hit.entry.config;
-        companyName = hit.entry.name;
-      }
+    // config, the registry supplies the canonical slug, config and name
+    // when it lists the company on that ATS (Workday's triple,
+    // SmartRecruiters' PascalCase slugs).
+    const hit = config ? null : await findEntryBySlug(slug);
+    if (hit && hit.ats === ats) {
+      match = 'registry';
+      targets = [registryTarget(hit, config)];
+    } else {
+      if (ats === 'workday' && config) match = 'workday_override';
+      targets = [{ ats, slug, name: null, config }];
     }
-    jobs = await adapter.fetch(fetchSlug, { config: cfg, companyName, filterContext });
   } else {
-    // Consult registry first — if we know which ATS this company uses,
-    // skip probing the others (saves API calls, clearer error semantics).
-    // The registry entry carries the canonical slug (so the adapter is
-    // called with the ATS's own casing, e.g. SmartRecruiters "Visa") and
-    // any adapter-specific config (the Workday {tenant,env,site} triple).
+    // Registry first: a known company costs one adapter call, with the
+    // ATS's own slug casing and any adapter config the row carries.
     const hit = await findEntryBySlug(slug);
     if (hit) {
-      jobs = await ADAPTERS[hit.ats].fetch(hit.entry.slug, {
-        config: config || hit.entry.config,
-        companyName: hit.entry.name,
-        filterContext,
-      });
+      match = 'registry';
+      targets = [registryTarget(hit, config)];
     } else {
-      // Discovery mode: company not in registry, probe all adapters.
-      // (Registry-only adapters like Workday bail here via their guard.)
-      const results = await Promise.allSettled(
-        Object.entries(ADAPTERS).map(async ([name, adapter]) => adapter.fetch(slug, { filterContext }))
-      );
-      jobs = results
-        .filter(r => r.status === 'fulfilled')
-        .flatMap(r => r.value);
+      discovery = true;
+      targets = ATS_NAMES.map(atsName => ({ ats: atsName, slug, name: null, config: undefined }));
+    }
+  }
+
+  // Adapters get the filters and the page so filter-aware ones (Workday,
+  // SmartRecruiters) hydrate only what the page needs, plus `report`, which
+  // records what an adapter chooses to say about its own board: the scan
+  // counts, and the org name and host the board states. Every call for one
+  // adapter merges into one record, so the two can arrive together or apart.
+  const reports = {};
+  const outcomes = await Promise.allSettled(targets.map(t =>
+    ADAPTERS[t.ats].fetch(t.slug, {
+      config: t.config,
+      companyName: t.name ?? undefined,
+      filterContext: { ...filters, offset, limit },
+      report: (fields) => {
+        reports[t.ats] = { ...reports[t.ats], ...fields };
+      },
+    })
+  ));
+
+  const boards = [];
+  const failed = [];
+  const rows = [];
+  targets.forEach((t, i) => {
+    const outcome = outcomes[i];
+    if (outcome.status === 'rejected') {
+      const err = outcome.reason;
+      if (!discovery || !(err instanceof AtsError)) throw err;
+      failed.push({ ats: t.ats, slug: t.slug, name: t.name, code: err.code, message: err.message });
+      return;
     }
+    // jobs_found is the board's list before any filter. Workday and
+    // SmartRecruiters filter their list before hydrating and report what
+    // they listed; counting their rows instead would make a filter miss on
+    // a hiring company read as an empty board (issue #60). Every other
+    // adapter returns its whole list.
+    const report = reports[t.ats] ?? {};
+    const scan = SCAN_KEYS.some(k => report[k] !== undefined)
+      ? Object.fromEntries(SCAN_KEYS.map(k => [k, report[k]]))
+      : null;
+    const listed = scan?.listed ?? outcome.value.length;
+    // A probed board exists when it listed rows: a 404 and an empty board
+    // both come back as []. A registry hit or an override is a board
+    // whatever it returned.
+    if (match === 'probe' && listed === 0) return;
+    rows.push(...outcome.value);
+    boards.push(describeBoard({
+      ...t,
+      org_name: report.org_name ?? null,
+      org_url: report.org_url ?? null,
+      jobs_found: listed,
+      scan,
+    }));
+  });
+
+  const matched = filterJobs(rows, filters);
+  const perBoard = new Map();
+  for (const job of matched) {
+    const key = `${job.ats}|${job.companySlug}`;
+    perBoard.set(key, (perBoard.get(key) || 0) + 1);
   }
+  for (const board of boards) board.matched = perBoard.get(`${board.ats}|${board.slug}`) || 0;
 
-  return applyFilters(jobs, { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes, limit });
+  return {
+    jobs: pageJobs(matched, { order, offset, limit }),
+    total_matched: matched.length,
+    total_before_filters: boards.reduce((n, b) => n + b.jobs_found, 0),
+    match,
+    company: match === 'registry' ? { key: normSlug(targets[0].name), name: targets[0].name } : null,
+    boards,
+    failed,
+  };
+}
+
+function registryTarget(hit, config) {
+  return { ats: hit.ats, slug: hit.entry.slug, name: hit.entry.name, config: config || hit.entry.config };
+}
+
+function discoveryFailure(company, failed) {
+  const limited = failed.some(f => f.code === ERROR_CODES.RATE_LIMITED);
+  const checks = failed.map(f => `${f.ats} (${f.message})`).join('; ');
+  return new AtsError(
+    limited ? ERROR_CODES.RATE_LIMITED : ERROR_CODES.ATS_UNREACHABLE,
+    `No board answered for "${company}" and the check failed on ${checks}`,
+    limited ? 429 : undefined
+  );
 }
 
 /**
@@ -101,14 +220,16 @@ export async function fetchJobs({
  */
 export async function search({ keyword, location, ats } = {}) {
   // For now, search is registry-based. With SQLite store, this becomes a full-text search.
-  if (!keyword) throw new Error('Keyword required');
+  if (!keyword) throw new ArgumentError('keyword is required');
   return searchRegistry(keyword);
 }
 
 /**
- * Detect which ATS platform a company uses (probes each adapter).
+ * Detect which ATS platform a company uses: the registry first, then a
+ * probe of each adapter. detectAts returns [{ ats, slug }]; detectAtsDetailed
+ * adds each board's source and the probes that failed.
  */
-export { detectAts } from './registry.js';
+export { detectAts, detectAtsDetailed } from './registry.js';
 
 /**
  * Look up which ATS a slug belongs to in the registry (cached, no network).
@@ -123,6 +244,7 @@ export const registry = {
   load: loadRegistry,
   search: searchRegistry,
   detect: detectAts,
+  detectDetailed: detectAtsDetailed,
   findAtsBySlug,
   findEntryBySlug,
   getSource: getRegistrySource,
@@ -134,13 +256,18 @@ export { fetchLever } from './adapters/lever.js';
 export { fetchAshby } from './adapters/ashby.js';
 
 // Re-export filter logic for reuse (e.g., by the MCP server)
-export { applyFilters } from './filters.js';
+export { applyFilters, applyFiltersDetailed } from './filters.js';
 
 // Re-export the list of supported ATS names (e.g. so the MCP layer can report
 // the full set detectAts probes, instead of hardcoding a stale subset).
 export { ATS_NAMES };
 
-// Error taxonomy + typed error. Adapters throw AtsError with a stable .code
-// (ats_unreachable / rate_limited) so the MCP layer maps failures without
-// parsing messages. ERROR_CODES is the single source of truth for both.
-export { ERROR_CODES, AtsError } from './errors.js';
+// Error taxonomy + typed errors. Adapters throw AtsError with a stable .code
+// (ats_unreachable / rate_limited); the library throws ArgumentError
+// (invalid_args) for a call it cannot make. The MCP layer maps both by code
+// without parsing messages. ERROR_CODES is the single source of truth.
+export { ERROR_CODES, AtsError, ArgumentError } from './errors.js';
+
+// HTTP settings for every adapter request: timeout, retries, per-host cap.
+// For tests and scripts; the defaults are right for normal use.
+export { configureHttp } from './http.js';