jd-intel 0.9.0 → 0.11.0

package/src/index.js

@@ -7,23 +7,39 @@
  */
 
 import { ADAPTERS, ATS_NAMES } from './adapters/index.js';
-import { loadRegistry, searchRegistry, detectAts, findAtsBySlug, findEntryBySlug, getRegistrySource } from './registry.js';
-import { applyFiltersDetailed } from './filters.js';
+import { loadRegistry, searchRegistry, detectAts, detectAtsDetailed, findAtsBySlug, findEntryBySlug, getRegistrySource, normSlug } from './registry.js';
+import { filterJobs, pageJobs, compileFilterPatterns } from './filters.js';
+import { AtsError, ArgumentError, ERROR_CODES } from './errors.js';
+import { describeBoard } from './boards.js';
+
+// The counts an adapter reports about its scan. boards[].scan carries these
+// and nothing else; org_name and org_url from the same report go on the
+// board itself.
+const SCAN_KEYS = ['listed', 'prefiltered', 'hydrated', 'capped'];
 
 /**
  * Fetch jobs from a company's ATS board.
  *
  * Same options as fetchJobsDetailed; returns the page as an array.
  *
+ * Discovery that found no board while at least one check failed throws an
+ * AtsError (rate_limited when any failure was a 429, else ats_unreachable)
+ * instead of returning []: an array has no place for `failed`, and an empty
+ * one would read as "not found" (issue #55).
+ *
  * @returns {Promise<Array>} Normalized, filtered job objects
  */
 export async function fetchJobs(options = {}) {
-  const { jobs } = await fetchJobsDetailed(options);
-  return jobs;
+  const result = await fetchJobsDetailed(options);
+  if (result.match === 'probe' && result.boards.length === 0 && result.failed.length > 0) {
+    throw discoveryFailure(options.company, result.failed);
+  }
+  return result.jobs;
 }
 
 /**
- * Fetch jobs from a company's ATS board, with the match count.
+ * Fetch jobs from a company's ATS board, with the counts and the boards
+ * behind the page.
  *
  * @param {Object} options
  * @param {string} options.company - Company slug or name
@@ -37,7 +53,31 @@ export async function fetchJobs(options = {}) {
  * @param {'newest'|'board'} [options.order='newest'] - 'newest': by postedAt descending, undated last, ties by id. 'board': the adapter's own order.
  * @param {number} [options.offset=0] - Matches to skip after sorting (paging).
  * @param {number} [options.limit=100] - Maximum jobs to return after offset.
- * @returns {Promise<{ jobs: Array, total_matched: number }>} The page, plus the match count before offset and limit
+ * @returns {Promise<{
+ *   jobs: Array,
+ *   total_matched: number,
+ *   total_before_filters: number,
+ *   content_missing: number,
+ *   match: 'registry'|'probe'|'workday_override',
+ *   company: { key: string, name: string }|null,
+ *   boards: Array<object>,
+ *   failed: Array<{ ats: string, slug: string, name: string|null, code: string, message: string }>,
+ * }>}
+ *   jobs: the page. total_matched: matches before offset and limit.
+ *   total_before_filters: rows every board listed before any filter (on
+ *   Workday and SmartRecruiters the list count, not the rows they hydrated).
+ *   content_missing: jobs whose posting was not read because the detail
+ *   request failed (Workday, SmartRecruiters), counted before the filters:
+ *   a description filter cannot match text that never arrived.
+ *   match: how the company was resolved. company: the registry row's name
+ *   and its key (normalized name); null unless match is 'registry'.
+ *   boards: one entry per board that answered (see src/boards.js), with
+ *   jobs_found and matched counted per board, and org_name and org_url as
+ *   the board states them (null where its ATS exposes nothing). failed:
+ *   adapters that threw an AtsError during discovery; on a registry hit
+ *   the error propagates.
+ * @throws {ArgumentError} No company, unknown ats, or a filter regex that does not compile.
+ * @throws {AtsError} The board's ATS failed on a registry hit, an explicit ats, or a Workday override.
  */
 export async function fetchJobsDetailed({
   company,
@@ -52,64 +92,138 @@ export async function fetchJobsDetailed({
   offset = 0,
   limit = 100,
 } = {}) {
-  if (!company) throw new Error('Company slug required');
+  if (!company) throw new ArgumentError('company is required');
+  compileFilterPatterns({ titleFilter, filter });
 
-  // Unified slug normalization: strip all non-alphanumeric (matches detectAts)
-  const slug = company.toLowerCase().replace(/[^a-z0-9]/g, '');
+  const slug = normSlug(company);
+  const filters = { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes };
 
-  // Filter context is passed as an additive 2nd arg to adapters. Existing
-  // adapters declare fetch{Name}(slug) and ignore extra positional args
-  // (JS no-op), so this is backward-compatible. Filter-aware adapters
-  // (e.g. Workday) use it to avoid mass detail-fetching on huge tenants.
-  const filterContext = { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes, offset, limit };
+  // Which boards to fetch, and how the company was resolved. A registry hit
+  // and a Workday override are one board each and their errors propagate.
+  // Discovery (not in the registry, no ats) asks every adapter, keeps the
+  // ones that answered, and records the ones that failed.
+  let match = 'probe';
+  let discovery = false;
+  let targets;
 
-  let jobs;
   if (ats) {
-    const adapter = ADAPTERS[ats];
-    if (!adapter) throw new Error(`Unknown ATS: ${ats}. Supported: ${ATS_NAMES.join(', ')}`);
+    if (!ADAPTERS[ats]) throw new ArgumentError(`Unknown ATS: ${ats}. Supported: ${ATS_NAMES.join(', ')}`);
     // Explicit ATS: an explicitly passed config wins (the only path that
     // can reach a Workday company not in the registry). With no explicit
-    // config, fall back to the registry so config-keyed adapters
-    // (Workday) and canonically-cased registry slugs (SmartRecruiters
-    // "Visa") also work on the explicit path, not just under auto-detect.
-    let fetchSlug = slug;
-    let cfg = config;
-    let companyName;
-    if (!cfg) {
-      const hit = await findEntryBySlug(slug);
-      if (hit && hit.ats === ats) {
-        fetchSlug = hit.entry.slug;
-        cfg = hit.entry.config;
-        companyName = hit.entry.name;
-      }
+    // config, the registry supplies the canonical slug, config and name
+    // when it lists the company on that ATS (Workday's triple,
+    // SmartRecruiters' PascalCase slugs).
+    const hit = config ? null : await findEntryBySlug(slug);
+    if (hit && hit.ats === ats) {
+      match = 'registry';
+      targets = [registryTarget(hit, config)];
+    } else {
+      if (ats === 'workday' && config) match = 'workday_override';
+      targets = [{ ats, slug, name: null, config }];
     }
-    jobs = await adapter.fetch(fetchSlug, { config: cfg, companyName, filterContext });
   } else {
-    // Consult registry first — if we know which ATS this company uses,
-    // skip probing the others (saves API calls, clearer error semantics).
-    // The registry entry carries the canonical slug (so the adapter is
-    // called with the ATS's own casing, e.g. SmartRecruiters "Visa") and
-    // any adapter-specific config (the Workday {tenant,env,site} triple).
+    // Registry first: a known company costs one adapter call, with the
+    // ATS's own slug casing and any adapter config the row carries.
     const hit = await findEntryBySlug(slug);
     if (hit) {
-      jobs = await ADAPTERS[hit.ats].fetch(hit.entry.slug, {
-        config: config || hit.entry.config,
-        companyName: hit.entry.name,
-        filterContext,
-      });
+      match = 'registry';
+      targets = [registryTarget(hit, config)];
     } else {
-      // Discovery mode: company not in registry, probe all adapters.
-      // (Registry-only adapters like Workday bail here via their guard.)
-      const results = await Promise.allSettled(
-        Object.entries(ADAPTERS).map(async ([name, adapter]) => adapter.fetch(slug, { filterContext }))
-      );
-      jobs = results
-        .filter(r => r.status === 'fulfilled')
-        .flatMap(r => r.value);
+      discovery = true;
+      targets = ATS_NAMES.map(atsName => ({ ats: atsName, slug, name: null, config: undefined }));
+    }
+  }
+
+  // Adapters get the filters and the page so filter-aware ones (Workday,
+  // SmartRecruiters) hydrate only what the page needs, plus `report`, which
+  // records what an adapter chooses to say about its own board: the scan
+  // counts, and the org name and host the board states. Every call for one
+  // adapter merges into one record, so the two can arrive together or apart.
+  const reports = {};
+  const outcomes = await Promise.allSettled(targets.map(t =>
+    ADAPTERS[t.ats].fetch(t.slug, {
+      config: t.config,
+      companyName: t.name ?? undefined,
+      filterContext: { ...filters, offset, limit },
+      report: (fields) => {
+        reports[t.ats] = { ...reports[t.ats], ...fields };
+      },
+    })
+  ));
+
+  const boards = [];
+  const failed = [];
+  const rows = [];
+  targets.forEach((t, i) => {
+    const outcome = outcomes[i];
+    if (outcome.status === 'rejected') {
+      const err = outcome.reason;
+      if (!discovery || !(err instanceof AtsError)) throw err;
+      failed.push({ ats: t.ats, slug: t.slug, name: t.name, code: err.code, message: err.message });
+      return;
     }
+    // jobs_found is the board's list before any filter. Workday and
+    // SmartRecruiters filter their list before hydrating and report what
+    // they listed; counting their rows instead would make a filter miss on
+    // a hiring company read as an empty board (issue #60). Every other
+    // adapter returns its whole list.
+    const report = reports[t.ats] ?? {};
+    const scan = SCAN_KEYS.some(k => report[k] !== undefined)
+      ? Object.fromEntries(SCAN_KEYS.map(k => [k, report[k]]))
+      : null;
+    const listed = scan?.listed ?? outcome.value.length;
+    // A probed board exists when it listed rows: a 404 and an empty board
+    // both come back as []. A registry hit or an override is a board
+    // whatever it returned.
+    if (match === 'probe' && listed === 0) return;
+    rows.push(...outcome.value);
+    boards.push(describeBoard({
+      ...t,
+      org_name: report.org_name ?? null,
+      org_url: report.org_url ?? null,
+      jobs_found: listed,
+      scan,
+    }));
+  });
+
+  const matched = filterJobs(rows, filters);
+  const perBoard = new Map();
+  for (const job of matched) {
+    const key = `${job.ats}|${job.companySlug}`;
+    perBoard.set(key, (perBoard.get(key) || 0) + 1);
   }
+  for (const board of boards) board.matched = perBoard.get(`${board.ats}|${board.slug}`) || 0;
+
+  return {
+    jobs: pageJobs(matched, { order, offset, limit }),
+    total_matched: matched.length,
+    total_before_filters: boards.reduce((n, b) => n + b.jobs_found, 0),
+    content_missing: rows.filter(j => j.content?.status === 'missing').length,
+    match,
+    company: match === 'registry' ? { key: normSlug(targets[0].name), name: targets[0].name } : null,
+    boards,
+    failed,
+  };
+}
+
+function registryTarget(hit, config) {
+  return { ats: hit.ats, slug: hit.entry.slug, name: hit.entry.name, config: config || hit.entry.config };
+}
 
-  return applyFiltersDetailed(jobs, { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes, order, offset, limit });
+/**
+ * The AtsError for a lookup where no board answered and at least one check
+ * failed: rate_limited when any failure was a 429, else ats_unreachable,
+ * with a message naming each failed check. `failed` is the list
+ * fetchJobsDetailed and detectAtsDetailed return.
+ */
+export function discoveryFailure(company, failed) {
+  const limited = failed.some(f => f.code === ERROR_CODES.RATE_LIMITED);
+  const checks = failed.map(f => `${f.ats} (${f.message})`).join('; ');
+  return new AtsError(
+    limited ? ERROR_CODES.RATE_LIMITED : ERROR_CODES.ATS_UNREACHABLE,
+    `No board answered for "${company}" and the check failed on ${checks}`,
+    limited ? 429 : undefined
+  );
 }
 
 /**
@@ -117,14 +231,16 @@ export async function fetchJobsDetailed({
  */
 export async function search({ keyword, location, ats } = {}) {
   // For now, search is registry-based. With SQLite store, this becomes a full-text search.
-  if (!keyword) throw new Error('Keyword required');
+  if (!keyword) throw new ArgumentError('keyword is required');
   return searchRegistry(keyword);
 }
 
 /**
- * Detect which ATS platform a company uses (probes each adapter).
+ * Detect which ATS platform a company uses: the registry first, then a
+ * probe of each adapter. detectAts returns [{ ats, slug }]; detectAtsDetailed
+ * adds each board's source and the probes that failed.
  */
-export { detectAts } from './registry.js';
+export { detectAts, detectAtsDetailed } from './registry.js';
 
 /**
  * Look up which ATS a slug belongs to in the registry (cached, no network).
@@ -139,6 +255,7 @@ export const registry = {
   load: loadRegistry,
   search: searchRegistry,
   detect: detectAts,
+  detectDetailed: detectAtsDetailed,
   findAtsBySlug,
   findEntryBySlug,
   getSource: getRegistrySource,
@@ -156,7 +273,12 @@ export { applyFilters, applyFiltersDetailed } from './filters.js';
 // the full set detectAts probes, instead of hardcoding a stale subset).
 export { ATS_NAMES };
 
-// Error taxonomy + typed error. Adapters throw AtsError with a stable .code
-// (ats_unreachable / rate_limited) so the MCP layer maps failures without
-// parsing messages. ERROR_CODES is the single source of truth for both.
-export { ERROR_CODES, AtsError } from './errors.js';
+// Error taxonomy + typed errors. Adapters throw AtsError with a stable .code
+// (ats_unreachable / rate_limited); the library throws ArgumentError
+// (invalid_args) for a call it cannot make. The MCP layer maps both by code
+// without parsing messages. ERROR_CODES is the single source of truth.
+export { ERROR_CODES, AtsError, ArgumentError } from './errors.js';
+
+// HTTP settings for every adapter request: timeout, retries, per-host cap.
+// For tests and scripts; the defaults are right for normal use.
+export { configureHttp } from './http.js';

package/src/normalizer.js

@@ -23,6 +23,9 @@ export function jobId(company, title, ats, location = '') {
  * adapter to 'remote' | 'hybrid' | 'onsite', or null when the platform
  * gives no signal. `raw.locations` lists every place the posting is open
  * in; `location` stays the primary because it feeds the id (issue #68).
+ *
+ * `raw.content` is set only by a two-step adapter whose detail request
+ * failed (see missingContent). Every other job was read in full.
  */
 export function normalize(raw, ats) {
   const now = new Date().toISOString();
@@ -47,10 +50,21 @@ export function normalize(raw, ats) {
     firstSeen: now,
     lastSeen: now,
     status: 'open',
+    content: raw.content || { status: 'complete', reason: null },
     metadata: raw.metadata || {},
   };
 }
 
+/**
+ * The `content` of a job whose detail request failed, so its description
+ * and pay were never read (issue #85). `failure` is the non-OK Response or
+ * the error atsFetch threw: an HTTP status gives "http_503", anything else
+ * "network_error".
+ */
+export function missingContent(failure) {
+  return { status: 'missing', reason: failure?.status ? `http_${failure.status}` : 'network_error' };
+}
+
 const CURRENCY_CODES = 'USD|EUR|GBP|CAD|AUD|NZD|CHF|SEK|NOK|DKK|PLN|CZK|HUF|INR|SGD|HKD|JPY|CNY|BRL|MXN|ZAR|AED|ILS';
 const SYMBOL_CURRENCY = { $: 'USD', '€': 'EUR', '£': 'GBP' };
 
@@ -140,14 +154,14 @@ function detectPeriod(before, after, min) {
   return min >= 10000 ? 'year' : null;
 }
 
-function periodWord(text) {
+export function periodWord(text) {
   if (HOUR_RE.test(text)) return 'hour';
   if (MONTH_RE.test(text)) return 'month';
   if (YEAR_RE.test(text)) return 'year';
   return null;
 }
 
-const WORKPLACE_TYPES = new Set(['remote', 'hybrid', 'onsite']);
+export const WORKPLACE_TYPES = new Set(['remote', 'hybrid', 'onsite']);
 
 /**
  * The platform's own value wins. Without one, a keyword in the location

package/src/registry.js

@@ -1,11 +1,16 @@
 import { readFile } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { AtsError } from './errors.js';
+import { ADAPTERS, ATS_NAMES } from './adapters/index.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const REGISTRY_DIR = join(__dirname, '..', 'registry');
 
-const PLATFORMS = ['greenhouse', 'lever', 'ashby', 'smartrecruiters', 'teamtailor', 'recruitee', 'workday'];
+// The one order the registry is ever walked in. Lookups, detectAts and the
+// loaded object all follow it, so which file answers for a slug does not
+// depend on which file's load finished first (issue #87).
+const PLATFORMS = ATS_NAMES;
 
 // Network-first registry. A hosted copy lets installed bundles AND npx users
 // pick up newly-added companies without reinstalling; the on-disk copy that
@@ -68,11 +73,8 @@ async function loadPlatform(platform) {
  */
 export async function loadRegistry(ats) {
   if (ats) return loadPlatform(ats);
-  const all = {};
-  await Promise.all(PLATFORMS.map(async (platform) => {
-    all[platform] = await loadPlatform(platform);
-  }));
-  return all;
+  const lists = await Promise.all(PLATFORMS.map(loadPlatform));
+  return Object.fromEntries(PLATFORMS.map((platform, i) => [platform, lists[i]]));
 }
 
 /**
@@ -93,7 +95,10 @@ export function getRegistrySource() {
 }
 
 /**
- * Search registry for companies matching a query.
+ * Search registry for companies matching a query, best match first: an
+ * exact name or slug, then a name that starts with the query, then a name
+ * that contains it, then a sector-only match. Ties keep platform order, so
+ * a caller that cuts the list drops the weakest matches (issue #62).
  */
 export async function searchRegistry(query) {
   const all = await loadRegistry();
@@ -104,40 +109,41 @@ export async function searchRegistry(query) {
     for (const company of companies) {
       const name = (company.name || company.slug || '').toLowerCase();
       const sector = (company.sector || '').toLowerCase();
-      if (name.includes(lower) || sector.includes(lower)) {
-        results.push({ ...company, ats });
-      }
+      const rank = name === lower || String(company.slug).toLowerCase() === lower ? 0
+        : name.startsWith(lower) ? 1
+        : name.includes(lower) ? 2
+        : sector.includes(lower) ? 3
+        : -1;
+      if (rank >= 0) results.push({ rank, row: { ...company, ats } });
     }
   }
 
-  return results;
+  return results.sort((a, b) => a.rank - b.rank).map(r => r.row);
 }
 
 // Slug match is case/punctuation-insensitive: registry slugs are stored
 // in each ATS's canonical form (SmartRecruiters uses PascalCase, e.g.
 // "Visa"), but callers pass a lowercased/alnum-stripped slug. Comparing
 // normalized forms keeps registry-first routing working for those.
-const normSlug = (s) => String(s).toLowerCase().replace(/[^a-z0-9]/g, '');
+export const normSlug = (s) => String(s).toLowerCase().replace(/[^a-z0-9]/g, '');
+
+const byPlatform = (a, b) => PLATFORMS.indexOf(a.ats) - PLATFORMS.indexOf(b.ats);
 
 /**
  * Look up which ATS a slug belongs to in the registry.
  * Returns the ATS name (e.g., "greenhouse") or null if not in registry.
  */
 export async function findAtsBySlug(slug) {
-  const all = await loadRegistry();
-  const key = normSlug(slug);
-  for (const [ats, companies] of Object.entries(all)) {
-    if (companies.some(c => normSlug(c.slug) === key)) return ats;
-  }
-  return null;
+  const hit = await findEntryBySlug(slug);
+  return hit ? hit.ats : null;
 }
 
 /**
  * Look up the full registry entry for a slug, with its ATS.
  * Unlike findAtsBySlug (returns just the ats name), this returns the
  * whole entry so callers can read adapter-specific config (e.g. the
- * Workday {tenant, env, site} triple). Additive — does not change
- * findAtsBySlug, which has other callers.
+ * Workday {tenant, env, site} triple). The files are searched in
+ * PLATFORMS order, so the first match is the same on every call.
  *
  * @returns {Promise<{ats: string, entry: object}|null>}
  */
@@ -152,18 +158,61 @@ export async function findEntryBySlug(slug) {
 }
 
 /**
- * Auto-detect which ATS a company uses.
+ * Where a company answers: the registry first, then a live probe of every
+ * adapter the registry did not already answer for.
+ *
+ * A slug the registry knows is listed with source 'registry' and not probed
+ * (Workday included, whose boards cannot be probed at all). Every remaining
+ * adapter's has() then runs: true adds a board with source 'probe', false
+ * adds nothing, and an AtsError (429, 5xx, 401, network) goes to `failed`
+ * with its code, so a board the probe could not check never reads as
+ * absent (issue #55). Any other error is a bug and is rethrown. Both lists
+ * come back in PLATFORMS order, never in completion order.
+ *
+ * @returns {Promise<{
+ *   boards: Array<{ ats: string, slug: string, source: 'registry'|'probe' }>,
+ *   failed: Array<{ ats: string, slug: string, code: string, message: string }>,
+ * }>}
  */
-export async function detectAts(companyName) {
-  const { ADAPTERS } = await import('./adapters/index.js');
-  const slug = companyName.toLowerCase().replace(/[^a-z0-9]/g, '');
+export async function detectAtsDetailed(companyName) {
+  const slug = normSlug(companyName);
+  const all = await loadRegistry();
 
-  const results = [];
-  const checks = Object.entries(ADAPTERS).map(async ([ats, adapter]) => {
-    const found = await adapter.has(slug);
-    if (found) results.push({ ats, slug });
-  });
+  const boards = [];
+  const failed = [];
+  const known = new Set();
+  for (const [ats, companies] of Object.entries(all)) {
+    const entry = companies.find(c => normSlug(c.slug) === slug);
+    if (entry) {
+      boards.push({ ats, slug: entry.slug, source: 'registry' });
+      known.add(ats);
+    }
+  }
+
+  const probes = Object.entries(ADAPTERS).filter(([ats]) => !known.has(ats));
+  const outcomes = await Promise.all(probes.map(async ([ats, adapter]) => {
+    try {
+      return { ats, found: await adapter.has(slug) };
+    } catch (err) {
+      if (!(err instanceof AtsError)) throw err;
+      return { ats, error: err };
+    }
+  }));
+  for (const { ats, found, error } of outcomes) {
+    if (error) failed.push({ ats, slug, code: error.code, message: error.message });
+    else if (found) boards.push({ ats, slug, source: 'probe' });
+  }
+
+  return { boards: boards.sort(byPlatform), failed: failed.sort(byPlatform) };
+}
 
-  await Promise.allSettled(checks);
-  return results;
+/**
+ * Auto-detect which ATS a company uses: the boards from detectAtsDetailed
+ * as [{ ats, slug }]. A failed probe never rejects this; the detailed
+ * variant is where those are reported. An adapter throwing anything but an
+ * AtsError is a bug and propagates, as it does through fetchJobs.
+ */
+export async function detectAts(companyName) {
+  const { boards } = await detectAtsDetailed(companyName);
+  return boards.map(({ ats, slug }) => ({ ats, slug }));
 }