jd-intel 0.9.0 → 0.10.0

package/src/index.js

@@ -7,23 +7,39 @@
  */
 
 import { ADAPTERS, ATS_NAMES } from './adapters/index.js';
-import { loadRegistry, searchRegistry, detectAts, findAtsBySlug, findEntryBySlug, getRegistrySource } from './registry.js';
-import { applyFiltersDetailed } from './filters.js';
+import { loadRegistry, searchRegistry, detectAts, detectAtsDetailed, findAtsBySlug, findEntryBySlug, getRegistrySource, normSlug } from './registry.js';
+import { filterJobs, pageJobs, compileFilterPatterns } from './filters.js';
+import { AtsError, ArgumentError, ERROR_CODES } from './errors.js';
+import { describeBoard } from './boards.js';
+
+// The counts an adapter reports about its scan. boards[].scan carries these
+// and nothing else; org_name and org_url from the same report go on the
+// board itself.
+const SCAN_KEYS = ['listed', 'prefiltered', 'hydrated', 'capped'];
 
 /**
  * Fetch jobs from a company's ATS board.
  *
  * Same options as fetchJobsDetailed; returns the page as an array.
  *
+ * Discovery that found no board while at least one check failed throws an
+ * AtsError (rate_limited when any failure was a 429, else ats_unreachable)
+ * instead of returning []: an array has no place for `failed`, and an empty
+ * one would read as "not found" (issue #55).
+ *
  * @returns {Promise<Array>} Normalized, filtered job objects
  */
 export async function fetchJobs(options = {}) {
-  const { jobs } = await fetchJobsDetailed(options);
-  return jobs;
+  const result = await fetchJobsDetailed(options);
+  if (result.match === 'probe' && result.boards.length === 0 && result.failed.length > 0) {
+    throw discoveryFailure(options.company, result.failed);
+  }
+  return result.jobs;
 }
 
 /**
- * Fetch jobs from a company's ATS board, with the match count.
+ * Fetch jobs from a company's ATS board, with the counts and the boards
+ * behind the page.
  *
  * @param {Object} options
  * @param {string} options.company - Company slug or name
@@ -37,7 +53,27 @@ export async function fetchJobs(options = {}) {
  * @param {'newest'|'board'} [options.order='newest'] - 'newest': by postedAt descending, undated last, ties by id. 'board': the adapter's own order.
  * @param {number} [options.offset=0] - Matches to skip after sorting (paging).
  * @param {number} [options.limit=100] - Maximum jobs to return after offset.
- * @returns {Promise<{ jobs: Array, total_matched: number }>} The page, plus the match count before offset and limit
+ * @returns {Promise<{
+ *   jobs: Array,
+ *   total_matched: number,
+ *   total_before_filters: number,
+ *   match: 'registry'|'probe'|'workday_override',
+ *   company: { key: string, name: string }|null,
+ *   boards: Array<object>,
+ *   failed: Array<{ ats: string, slug: string, name: string|null, code: string, message: string }>,
+ * }>}
+ *   jobs: the page. total_matched: matches before offset and limit.
+ *   total_before_filters: rows every board listed before any filter (on
+ *   Workday and SmartRecruiters the list count, not the rows they hydrated).
+ *   match: how the company was resolved. company: the registry row's name
+ *   and its key (normalized name); null unless match is 'registry'.
+ *   boards: one entry per board that answered (see src/boards.js), with
+ *   jobs_found and matched counted per board, and org_name and org_url as
+ *   the board states them (null where its ATS exposes nothing). failed:
+ *   adapters that threw an AtsError during discovery; on a registry hit
+ *   the error propagates.
+ * @throws {ArgumentError} No company, unknown ats, or a filter regex that does not compile.
+ * @throws {AtsError} The board's ATS failed on a registry hit, an explicit ats, or a Workday override.
  */
 export async function fetchJobsDetailed({
   company,
@@ -52,64 +88,131 @@ export async function fetchJobsDetailed({
   offset = 0,
   limit = 100,
 } = {}) {
-  if (!company) throw new Error('Company slug required');
+  if (!company) throw new ArgumentError('company is required');
+  compileFilterPatterns({ titleFilter, filter });
 
-  // Unified slug normalization: strip all non-alphanumeric (matches detectAts)
-  const slug = company.toLowerCase().replace(/[^a-z0-9]/g, '');
+  const slug = normSlug(company);
+  const filters = { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes };
 
-  // Filter context is passed as an additive 2nd arg to adapters. Existing
-  // adapters declare fetch{Name}(slug) and ignore extra positional args
-  // (JS no-op), so this is backward-compatible. Filter-aware adapters
-  // (e.g. Workday) use it to avoid mass detail-fetching on huge tenants.
-  const filterContext = { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes, offset, limit };
+  // Which boards to fetch, and how the company was resolved. A registry hit
+  // and a Workday override are one board each and their errors propagate.
+  // Discovery (not in the registry, no ats) asks every adapter, keeps the
+  // ones that answered, and records the ones that failed.
+  let match = 'probe';
+  let discovery = false;
+  let targets;
 
-  let jobs;
   if (ats) {
-    const adapter = ADAPTERS[ats];
-    if (!adapter) throw new Error(`Unknown ATS: ${ats}. Supported: ${ATS_NAMES.join(', ')}`);
+    if (!ADAPTERS[ats]) throw new ArgumentError(`Unknown ATS: ${ats}. Supported: ${ATS_NAMES.join(', ')}`);
     // Explicit ATS: an explicitly passed config wins (the only path that
     // can reach a Workday company not in the registry). With no explicit
-    // config, fall back to the registry so config-keyed adapters
-    // (Workday) and canonically-cased registry slugs (SmartRecruiters
-    // "Visa") also work on the explicit path, not just under auto-detect.
-    let fetchSlug = slug;
-    let cfg = config;
-    let companyName;
-    if (!cfg) {
-      const hit = await findEntryBySlug(slug);
-      if (hit && hit.ats === ats) {
-        fetchSlug = hit.entry.slug;
-        cfg = hit.entry.config;
-        companyName = hit.entry.name;
-      }
+    // config, the registry supplies the canonical slug, config and name
+    // when it lists the company on that ATS (Workday's triple,
+    // SmartRecruiters' PascalCase slugs).
+    const hit = config ? null : await findEntryBySlug(slug);
+    if (hit && hit.ats === ats) {
+      match = 'registry';
+      targets = [registryTarget(hit, config)];
+    } else {
+      if (ats === 'workday' && config) match = 'workday_override';
+      targets = [{ ats, slug, name: null, config }];
     }
-    jobs = await adapter.fetch(fetchSlug, { config: cfg, companyName, filterContext });
   } else {
-    // Consult registry first — if we know which ATS this company uses,
-    // skip probing the others (saves API calls, clearer error semantics).
-    // The registry entry carries the canonical slug (so the adapter is
-    // called with the ATS's own casing, e.g. SmartRecruiters "Visa") and
-    // any adapter-specific config (the Workday {tenant,env,site} triple).
+    // Registry first: a known company costs one adapter call, with the
+    // ATS's own slug casing and any adapter config the row carries.
     const hit = await findEntryBySlug(slug);
     if (hit) {
-      jobs = await ADAPTERS[hit.ats].fetch(hit.entry.slug, {
-        config: config || hit.entry.config,
-        companyName: hit.entry.name,
-        filterContext,
-      });
+      match = 'registry';
+      targets = [registryTarget(hit, config)];
     } else {
-      // Discovery mode: company not in registry, probe all adapters.
-      // (Registry-only adapters like Workday bail here via their guard.)
-      const results = await Promise.allSettled(
-        Object.entries(ADAPTERS).map(async ([name, adapter]) => adapter.fetch(slug, { filterContext }))
-      );
-      jobs = results
-        .filter(r => r.status === 'fulfilled')
-        .flatMap(r => r.value);
+      discovery = true;
+      targets = ATS_NAMES.map(atsName => ({ ats: atsName, slug, name: null, config: undefined }));
+    }
+  }
+
+  // Adapters get the filters and the page so filter-aware ones (Workday,
+  // SmartRecruiters) hydrate only what the page needs, plus `report`, which
+  // records what an adapter chooses to say about its own board: the scan
+  // counts, and the org name and host the board states. Every call for one
+  // adapter merges into one record, so the two can arrive together or apart.
+  const reports = {};
+  const outcomes = await Promise.allSettled(targets.map(t =>
+    ADAPTERS[t.ats].fetch(t.slug, {
+      config: t.config,
+      companyName: t.name ?? undefined,
+      filterContext: { ...filters, offset, limit },
+      report: (fields) => {
+        reports[t.ats] = { ...reports[t.ats], ...fields };
+      },
+    })
+  ));
+
+  const boards = [];
+  const failed = [];
+  const rows = [];
+  targets.forEach((t, i) => {
+    const outcome = outcomes[i];
+    if (outcome.status === 'rejected') {
+      const err = outcome.reason;
+      if (!discovery || !(err instanceof AtsError)) throw err;
+      failed.push({ ats: t.ats, slug: t.slug, name: t.name, code: err.code, message: err.message });
+      return;
     }
+    // jobs_found is the board's list before any filter. Workday and
+    // SmartRecruiters filter their list before hydrating and report what
+    // they listed; counting their rows instead would make a filter miss on
+    // a hiring company read as an empty board (issue #60). Every other
+    // adapter returns its whole list.
+    const report = reports[t.ats] ?? {};
+    const scan = SCAN_KEYS.some(k => report[k] !== undefined)
+      ? Object.fromEntries(SCAN_KEYS.map(k => [k, report[k]]))
+      : null;
+    const listed = scan?.listed ?? outcome.value.length;
+    // A probed board exists when it listed rows: a 404 and an empty board
+    // both come back as []. A registry hit or an override is a board
+    // whatever it returned.
+    if (match === 'probe' && listed === 0) return;
+    rows.push(...outcome.value);
+    boards.push(describeBoard({
+      ...t,
+      org_name: report.org_name ?? null,
+      org_url: report.org_url ?? null,
+      jobs_found: listed,
+      scan,
+    }));
+  });
+
+  const matched = filterJobs(rows, filters);
+  const perBoard = new Map();
+  for (const job of matched) {
+    const key = `${job.ats}|${job.companySlug}`;
+    perBoard.set(key, (perBoard.get(key) || 0) + 1);
   }
+  for (const board of boards) board.matched = perBoard.get(`${board.ats}|${board.slug}`) || 0;
+
+  return {
+    jobs: pageJobs(matched, { order, offset, limit }),
+    total_matched: matched.length,
+    total_before_filters: boards.reduce((n, b) => n + b.jobs_found, 0),
+    match,
+    company: match === 'registry' ? { key: normSlug(targets[0].name), name: targets[0].name } : null,
+    boards,
+    failed,
+  };
+}
 
-  return applyFiltersDetailed(jobs, { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes, order, offset, limit });
+function registryTarget(hit, config) {
+  return { ats: hit.ats, slug: hit.entry.slug, name: hit.entry.name, config: config || hit.entry.config };
+}
+
+function discoveryFailure(company, failed) {
+  const limited = failed.some(f => f.code === ERROR_CODES.RATE_LIMITED);
+  const checks = failed.map(f => `${f.ats} (${f.message})`).join('; ');
+  return new AtsError(
+    limited ? ERROR_CODES.RATE_LIMITED : ERROR_CODES.ATS_UNREACHABLE,
+    `No board answered for "${company}" and the check failed on ${checks}`,
+    limited ? 429 : undefined
+  );
 }
 
 /**
@@ -117,14 +220,16 @@ export async function fetchJobsDetailed({
  */
 export async function search({ keyword, location, ats } = {}) {
   // For now, search is registry-based. With SQLite store, this becomes a full-text search.
-  if (!keyword) throw new Error('Keyword required');
+  if (!keyword) throw new ArgumentError('keyword is required');
   return searchRegistry(keyword);
 }
 
 /**
- * Detect which ATS platform a company uses (probes each adapter).
+ * Detect which ATS platform a company uses: the registry first, then a
+ * probe of each adapter. detectAts returns [{ ats, slug }]; detectAtsDetailed
+ * adds each board's source and the probes that failed.
  */
-export { detectAts } from './registry.js';
+export { detectAts, detectAtsDetailed } from './registry.js';
 
 /**
  * Look up which ATS a slug belongs to in the registry (cached, no network).
@@ -139,6 +244,7 @@ export const registry = {
   load: loadRegistry,
   search: searchRegistry,
   detect: detectAts,
+  detectDetailed: detectAtsDetailed,
   findAtsBySlug,
   findEntryBySlug,
   getSource: getRegistrySource,
@@ -156,7 +262,12 @@ export { applyFilters, applyFiltersDetailed } from './filters.js';
 // the full set detectAts probes, instead of hardcoding a stale subset).
 export { ATS_NAMES };
 
-// Error taxonomy + typed error. Adapters throw AtsError with a stable .code
-// (ats_unreachable / rate_limited) so the MCP layer maps failures without
-// parsing messages. ERROR_CODES is the single source of truth for both.
-export { ERROR_CODES, AtsError } from './errors.js';
+// Error taxonomy + typed errors. Adapters throw AtsError with a stable .code
+// (ats_unreachable / rate_limited); the library throws ArgumentError
+// (invalid_args) for a call it cannot make. The MCP layer maps both by code
+// without parsing messages. ERROR_CODES is the single source of truth.
+export { ERROR_CODES, AtsError, ArgumentError } from './errors.js';
+
+// HTTP settings for every adapter request: timeout, retries, per-host cap.
+// For tests and scripts; the defaults are right for normal use.
+export { configureHttp } from './http.js';

package/src/registry.js

@@ -1,10 +1,14 @@
 import { readFile } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { AtsError } from './errors.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const REGISTRY_DIR = join(__dirname, '..', 'registry');
 
+// The one order the registry is ever walked in. Lookups, detectAts and the
+// loaded object all follow it, so which file answers for a slug does not
+// depend on which file's load finished first (issue #87).
 const PLATFORMS = ['greenhouse', 'lever', 'ashby', 'smartrecruiters', 'teamtailor', 'recruitee', 'workday'];
 
 // Network-first registry. A hosted copy lets installed bundles AND npx users
@@ -68,11 +72,8 @@ async function loadPlatform(platform) {
  */
 export async function loadRegistry(ats) {
   if (ats) return loadPlatform(ats);
-  const all = {};
-  await Promise.all(PLATFORMS.map(async (platform) => {
-    all[platform] = await loadPlatform(platform);
-  }));
-  return all;
+  const lists = await Promise.all(PLATFORMS.map(loadPlatform));
+  return Object.fromEntries(PLATFORMS.map((platform, i) => [platform, lists[i]]));
 }
 
 /**
@@ -117,34 +118,43 @@ export async function searchRegistry(query) {
 // in each ATS's canonical form (SmartRecruiters uses PascalCase, e.g.
 // "Visa"), but callers pass a lowercased/alnum-stripped slug. Comparing
 // normalized forms keeps registry-first routing working for those.
-const normSlug = (s) => String(s).toLowerCase().replace(/[^a-z0-9]/g, '');
+export const normSlug = (s) => String(s).toLowerCase().replace(/[^a-z0-9]/g, '');
+
+// The loaded registry as [ats, companies] pairs in PLATFORMS order, whatever
+// order the object's keys are in.
+function platformEntries(all) {
+  return PLATFORMS.map(ats => [ats, all[ats] || []]);
+}
+
+function platformIndex(ats) {
+  const i = PLATFORMS.indexOf(ats);
+  return i === -1 ? PLATFORMS.length : i;
+}
+
+const byPlatform = (a, b) => platformIndex(a.ats) - platformIndex(b.ats);
 
 /**
  * Look up which ATS a slug belongs to in the registry.
  * Returns the ATS name (e.g., "greenhouse") or null if not in registry.
  */
 export async function findAtsBySlug(slug) {
-  const all = await loadRegistry();
-  const key = normSlug(slug);
-  for (const [ats, companies] of Object.entries(all)) {
-    if (companies.some(c => normSlug(c.slug) === key)) return ats;
-  }
-  return null;
+  const hit = await findEntryBySlug(slug);
+  return hit ? hit.ats : null;
 }
 
 /**
  * Look up the full registry entry for a slug, with its ATS.
  * Unlike findAtsBySlug (returns just the ats name), this returns the
  * whole entry so callers can read adapter-specific config (e.g. the
- * Workday {tenant, env, site} triple). Additive — does not change
- * findAtsBySlug, which has other callers.
+ * Workday {tenant, env, site} triple). The files are searched in
+ * PLATFORMS order, so the first match is the same on every call.
  *
  * @returns {Promise<{ats: string, entry: object}|null>}
  */
 export async function findEntryBySlug(slug) {
   const all = await loadRegistry();
   const key = normSlug(slug);
-  for (const [ats, companies] of Object.entries(all)) {
+  for (const [ats, companies] of platformEntries(all)) {
     const entry = companies.find(c => normSlug(c.slug) === key);
     if (entry) return { ats, entry };
   }
@@ -152,18 +162,62 @@ export async function findEntryBySlug(slug) {
 }
 
 /**
- * Auto-detect which ATS a company uses.
+ * Where a company answers: the registry first, then a live probe of every
+ * adapter the registry did not already answer for.
+ *
+ * A slug the registry knows is listed with source 'registry' and not probed
+ * (Workday included, whose boards cannot be probed at all). Every remaining
+ * adapter's has() then runs: true adds a board with source 'probe', false
+ * adds nothing, and an AtsError (429, 5xx, 401, network) goes to `failed`
+ * with its code, so a board the probe could not check never reads as
+ * absent (issue #55). Any other error is a bug and is rethrown. Both lists
+ * come back in PLATFORMS order, never in completion order.
+ *
+ * @returns {Promise<{
+ *   boards: Array<{ ats: string, slug: string, source: 'registry'|'probe' }>,
+ *   failed: Array<{ ats: string, slug: string, code: string, message: string }>,
+ * }>}
  */
-export async function detectAts(companyName) {
+export async function detectAtsDetailed(companyName) {
   const { ADAPTERS } = await import('./adapters/index.js');
-  const slug = companyName.toLowerCase().replace(/[^a-z0-9]/g, '');
+  const slug = normSlug(companyName);
+  const all = await loadRegistry();
 
-  const results = [];
-  const checks = Object.entries(ADAPTERS).map(async ([ats, adapter]) => {
-    const found = await adapter.has(slug);
-    if (found) results.push({ ats, slug });
-  });
+  const boards = [];
+  const failed = [];
+  const known = new Set();
+  for (const [ats, companies] of platformEntries(all)) {
+    const entry = companies.find(c => normSlug(c.slug) === slug);
+    if (entry) {
+      boards.push({ ats, slug: entry.slug, source: 'registry' });
+      known.add(ats);
+    }
+  }
 
-  await Promise.allSettled(checks);
-  return results;
+  const probes = Object.entries(ADAPTERS).filter(([ats]) => !known.has(ats));
+  const outcomes = await Promise.all(probes.map(async ([ats, adapter]) => {
+    try {
+      return { ats, found: await adapter.has(slug) };
+    } catch (err) {
+      if (!(err instanceof AtsError)) throw err;
+      return { ats, error: err };
+    }
+  }));
+  for (const { ats, found, error } of outcomes) {
+    if (error) failed.push({ ats, slug, code: error.code, message: error.message });
+    else if (found) boards.push({ ats, slug, source: 'probe' });
+  }
+
+  return { boards: boards.sort(byPlatform), failed: failed.sort(byPlatform) };
+}
+
+/**
+ * Auto-detect which ATS a company uses: the boards from detectAtsDetailed
+ * as [{ ats, slug }]. A failed probe never rejects this; the detailed
+ * variant is where those are reported. An adapter throwing anything but an
+ * AtsError is a bug and propagates, as it does through fetchJobs.
+ */
+export async function detectAts(companyName) {
+  const { boards } = await detectAtsDetailed(companyName);
+  return boards.map(({ ats, slug }) => ({ ats, slug }));
 }