jd-intel 0.9.0 → 0.11.0

package/src/cli.js

@@ -11,38 +11,44 @@
 
 import { realpathSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
 import { fetchJobs } from './index.js';
-import { detectAts, searchRegistry } from './registry.js';
+import { detectAtsDetailed, searchRegistry } from './registry.js';
 
 const [,, command, ...args] = process.argv;
 
 async function main() {
   switch (command) {
     case 'fetch': {
-      const company = args[0];
+      const string = { type: 'string' };
+      const { values: flags, positionals } = parseArgs({
+        args,
+        allowPositionals: true,
+        options: {
+          ats: string, 'title-filter': string, filter: string, 'posted-within-days': string,
+          'location-include': string, 'location-exclude': string, limit: string,
+          'workday-tenant': string, 'workday-env': string, 'workday-site': string,
+          json: { type: 'boolean' },
+        },
+      });
+      const company = positionals[0];
       if (!company) { console.error('Usage: jd-intel fetch <company> [--ats <platform>]  (omit --ats to auto-detect; run "jd-intel" for the platform list)'); process.exit(1); }
-      const getArg = (flag) => {
-        const idx = args.indexOf(flag);
-        return idx >= 0 ? args[idx + 1] : undefined;
-      };
-      let ats = getArg('--ats');
-      const titleFilter = getArg('--title-filter');
-      const filter = getArg('--filter');
-      const postedWithinRaw = getArg('--posted-within-days');
-      const postedWithinDays = postedWithinRaw !== undefined ? Number(postedWithinRaw) : undefined;
-      const locIncludeRaw = getArg('--location-include');
-      const locationIncludes = locIncludeRaw ? locIncludeRaw.split(',').map(s => s.trim()).filter(Boolean) : undefined;
-      const locExcludeRaw = getArg('--location-exclude');
-      const locationExcludes = locExcludeRaw ? locExcludeRaw.split(',').map(s => s.trim()).filter(Boolean) : undefined;
-      const limitRaw = getArg('--limit');
-      const limit = limitRaw !== undefined ? Number(limitRaw) : undefined;
+      const number = (v) => (v !== undefined ? Number(v) : undefined);
+      const list = (v) => (v ? v.split(',').map(s => s.trim()).filter(Boolean) : undefined);
+      let ats = flags.ats;
+      const titleFilter = flags['title-filter'];
+      const filter = flags.filter;
+      const postedWithinDays = number(flags['posted-within-days']);
+      const locationIncludes = list(flags['location-include']);
+      const locationExcludes = list(flags['location-exclude']);
+      const limit = number(flags.limit);
 
       // Workday is keyed by a {tenant, env, site} triple, not a slug.
       // Supplying it here makes a Workday board reachable without a
       // registry entry; presence of the flags infers --ats workday.
-      const wdTenant = getArg('--workday-tenant');
-      const wdEnv = getArg('--workday-env');
-      const wdSite = getArg('--workday-site');
+      const wdTenant = flags['workday-tenant'];
+      const wdEnv = flags['workday-env'];
+      const wdSite = flags['workday-site'];
       let config;
       if (wdTenant || wdEnv || wdSite) {
         if (!wdTenant || !wdEnv || !wdSite) {
@@ -91,8 +97,10 @@ async function main() {
         const loc = job.location ? ` | ${job.location}` : '';
         const dept = job.department ? ` [${job.department}]` : '';
         console.log(`  ${job.title}${dept}${loc}${salary}`);
-        console.log(`  ${job.url}`);
-        if (job.description) {
+        console.log(`  ${job.url || '(no URL: posting not read)'}`);
+        if (job.content?.status === 'missing') {
+          console.log(`  posting not read: ${job.content.reason}`);
+        } else if (job.description) {
           const preview = job.description.substring(0, 120).replace(/\n/g, ' ');
           console.log(`  ${preview}...`);
         }
@@ -103,7 +111,7 @@ async function main() {
         console.log(`  ... and ${jobs.length - 20} more. Use --json for full output.`);
       }
 
-      if (args.includes('--json')) {
+      if (flags.json) {
         console.log(JSON.stringify(jobs, null, 2));
       }
       break;
@@ -113,13 +121,17 @@ async function main() {
       const company = args[0];
       if (!company) { console.error('Usage: jd-intel detect <company>'); process.exit(1); }
       console.log(`Detecting ATS for ${company}...`);
-      const results = await detectAts(company);
-      if (results.length === 0) {
-        console.log('No ATS board found for this company.');
-      } else {
-        for (const r of results) {
-          console.log(`  Found: ${r.ats} (slug: ${r.slug})`);
-        }
+      const { boards, failed } = await detectAtsDetailed(company);
+      for (const b of boards) {
+        console.log(`  Found: ${b.ats} (slug: ${b.slug}, ${b.source === 'registry' ? 'in the registry' : 'live probe'})`);
+      }
+      for (const f of failed) {
+        console.log(`  Could not check ${f.ats}: ${f.message}`);
+      }
+      if (boards.length === 0) {
+        console.log(failed.length > 0
+          ? 'No ATS board confirmed. At least one check failed, so this is not a definite miss. Retry in a moment.'
+          : 'No ATS board found for this company.');
       }
       break;
     }

package/src/errors.js

@@ -34,6 +34,20 @@ export class AtsError extends Error {
   }
 }
 
+/**
+ * Thrown when a call cannot proceed because of its arguments: a missing
+ * company, an unknown ATS name, a regex that does not compile. Carries
+ * `code: 'invalid_args'` so callers tell a bad request from a failed fetch
+ * (AtsError) without reading the message.
+ */
+export class ArgumentError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'ArgumentError';
+    this.code = ERROR_CODES.INVALID_ARGS;
+  }
+}
+
 /**
  * Helper for adapters: build an AtsError from an HTTP status (429 => rate
  * limited, anything else => unreachable) with the given message.

package/src/filters.js

@@ -1,3 +1,5 @@
+import { ArgumentError } from './errors.js';
+
 /**
  * Apply filters to a list of normalized jobs.
  *
@@ -23,30 +25,32 @@ export function applyFilters(jobs, options = {}) {
  * @returns {{ jobs: Array, total_matched: number }}
  */
 export function applyFiltersDetailed(jobs, options = {}) {
-  const {
-    titleFilter,
-    filter,
-    postedWithinDays,
-    locationIncludes,
-    locationExcludes,
-    order = 'newest',
-    offset = 0,
-    limit = 100,
-  } = options;
+  const { order = 'newest', offset = 0, limit = 100 } = options;
+  const matched = filterJobs(jobs, options);
+  return { jobs: pageJobs(matched, { order, offset, limit }), total_matched: matched.length };
+}
+
+/**
+ * The filter step on its own: every job that passes titleFilter, filter,
+ * postedWithinDays and the location filters, in the order given. No sort
+ * and no paging, so fetchJobsDetailed can count the matches per board
+ * before the page cut removes them.
+ */
+export function filterJobs(jobs, options = {}) {
+  const { titleFilter, filter, postedWithinDays, locationIncludes, locationExcludes } = options;
+  const { title, topic } = compileFilterPatterns({ titleFilter, filter });
 
   let result = jobs;
 
-  if (titleFilter) {
-    const pattern = new RegExp(titleFilter, 'i');
-    result = result.filter(j => pattern.test(j.title || ''));
+  if (title) {
+    result = result.filter(j => title.test(j.title || ''));
   }
 
-  if (filter) {
-    const pattern = new RegExp(filter, 'i');
+  if (topic) {
     result = result.filter(j =>
-      pattern.test(j.title || '') ||
-      pattern.test(j.department || '') ||
-      pattern.test(j.description || '')
+      topic.test(j.title || '') ||
+      topic.test(j.department || '') ||
+      topic.test(j.description || '')
     );
   }
 
@@ -69,7 +73,14 @@ export function applyFiltersDetailed(jobs, options = {}) {
     result = result.filter(j => !jobLocations(j).every(loc => matchers.some(m => m(loc))));
   }
 
-  const total_matched = result.length;
+  return result;
+}
+
+/**
+ * Sort, then cut the page (see applyFiltersDetailed for the order rules).
+ */
+export function pageJobs(jobs, { order = 'newest', offset = 0, limit = 100 } = {}) {
+  let result = jobs;
 
   if (order !== 'board') {
     result = [...result].sort(byNewest);
@@ -77,11 +88,30 @@ export function applyFiltersDetailed(jobs, options = {}) {
 
   const start = typeof offset === 'number' && offset > 0 ? offset : 0;
   const end = typeof limit === 'number' ? start + limit : undefined;
-  if (start > 0 || (end !== undefined && result.length > end)) {
-    result = result.slice(start, end);
-  }
+  return result.slice(start, end);
+}
+
+/**
+ * Compile the two regex arguments, or throw ArgumentError naming the one
+ * that does not compile. Both are case-insensitive. fetchJobsDetailed calls
+ * this before its first request, so a bad pattern is reported as a bad
+ * argument and costs no upstream traffic.
+ *
+ * @returns {{ title: RegExp|null, topic: RegExp|null }}
+ */
+export function compileFilterPatterns({ titleFilter, filter } = {}) {
+  return {
+    title: titleFilter ? compilePattern(titleFilter, 'titleFilter') : null,
+    topic: filter ? compilePattern(filter, 'filter') : null,
+  };
+}
 
-  return { jobs: result, total_matched };
+function compilePattern(source, name) {
+  try {
+    return new RegExp(source, 'i');
+  } catch (err) {
+    throw new ArgumentError(`${name}: ${err.message}`);
+  }
 }
 
 /**
@@ -118,14 +148,19 @@ function byNewest(a, b) {
 }
 
 /**
- * Build a matcher for a single location keyword.
+ * Build a matcher for a single location keyword. The matcher takes a
+ * lowercased location string.
  *
  * Short tokens (≤4 chars) use word-boundary matching to prevent substring
  * collisions like "US" matching "Australia", "Brussels", "Belarus", or "UK"
- * matching "Auckland". Longer tokens use substring matching so phrases like
+ * matching "Ukraine". Longer tokens use substring matching so phrases like
  * "United States" can match "United States of America".
+ *
+ * Exported for the Workday list pre-filter, so one rule (trim, empty
+ * keywords never match, word boundaries for short tokens) applies before
+ * and after detail hydration (issue #61).
  */
-function makeLocationMatcher(needle) {
+export function makeLocationMatcher(needle) {
   const lower = (needle || '').toLowerCase().trim();
   if (!lower) return () => false;
   if (lower.length <= 4) {
@@ -135,3 +170,51 @@ function makeLocationMatcher(needle) {
   }
   return (loc) => loc.includes(lower);
 }
+
+/**
+ * The list pre-filter and detail budget the two-step adapters (Workday,
+ * SmartRecruiters) share: narrow the cheap list rows with the filters a row
+ * can answer, then bound how many get a detail request.
+ *
+ * The library re-applies every filter after hydration, so a keep here is
+ * never final. `location(row)` returns the lowercased location, or null
+ * when the row cannot say where it is (it then stays a candidate through
+ * both location filters). `postedWithin(row, days)` decides recency.
+ *
+ * A description `filter` runs only after hydration, so that case keeps the
+ * full `max` budget instead of truncating to the page (which could hydrate
+ * rows that all fail the regex while better matches go unscanned). Without
+ * one, the budget is the page plus the offset before it. List order is kept.
+ *
+ * @returns {{ candidates: Array, hydrate: Array }}
+ */
+export function prefilterRows(rows, fc, { title, location, postedWithin, max }) {
+  let candidates = rows;
+
+  if (fc.titleFilter) {
+    const re = new RegExp(fc.titleFilter, 'i');
+    candidates = candidates.filter(p => re.test(title(p)));
+  }
+  if (Array.isArray(fc.locationIncludes) && fc.locationIncludes.length > 0) {
+    const inc = fc.locationIncludes.map(makeLocationMatcher);
+    candidates = candidates.filter(p => {
+      const loc = location(p);
+      return loc === null || inc.some(m => m(loc));
+    });
+  }
+  if (Array.isArray(fc.locationExcludes) && fc.locationExcludes.length > 0) {
+    const exc = fc.locationExcludes.map(makeLocationMatcher);
+    candidates = candidates.filter(p => {
+      const loc = location(p);
+      return loc === null || !exc.some(m => m(loc));
+    });
+  }
+  if (typeof fc.postedWithinDays === 'number') {
+    candidates = candidates.filter(p => postedWithin(p, fc.postedWithinDays));
+  }
+
+  const limit = typeof fc.limit === 'number' && fc.limit > 0 ? fc.limit : 100;
+  const skip = typeof fc.offset === 'number' && fc.offset > 0 ? fc.offset : 0;
+  const cap = fc.filter ? max : Math.min(skip + limit, max);
+  return { candidates, hydrate: candidates.slice(0, cap) };
+}

package/src/http.js

@@ -0,0 +1,184 @@
+import { AtsError, ERROR_CODES, atsErrorFromStatus } from './errors.js';
+
+/**
+ * The one HTTP door for every adapter request (issue #7).
+ *
+ * atsFetch() wraps the global fetch with the politeness every ATS expects
+ * and the failure handling the adapters used to leave out:
+ *   - a timeout per attempt on the wait for the response to start
+ *   - retries with exponential backoff and jitter on 429, any 5xx, and
+ *     network errors (DNS, reset, timeout), honoring Retry-After; a
+ *     certificate failure is thrown at once, since it cannot pass later
+ *   - a cap on requests in flight per host
+ *
+ * Every other status resolves normally, so an adapter keeps its own 404
+ * handling. Once the retries are used up the caller gets an AtsError:
+ * rate_limited for a 429, ats_unreachable for a 5xx or a network failure.
+ * The global fetch is read on every attempt so a test's mock of it applies.
+ *
+ * The timer stops once the headers are in. Reading the body is the caller's
+ * step (resp.json() in the adapter), and it is not timed: a signal left on
+ * the request would abort that read too, so a large board on a slow link
+ * would fail at the timeout with a raw TimeoutError thrown from resp.json(),
+ * outside this retry loop, where master downloaded it fine. undici's own
+ * body timeout (300s idle) still ends a stream that stalls.
+ */
+
+const DEFAULTS = {
+  timeoutMs: 10_000,
+  retries: 3,   // attempts per request in total; 1 turns retrying off
+  perHost: 4,   // requests in flight per hostname
+  sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+};
+
+const BASE_BACKOFF_MS = 1000; // 1s, 2s, 4s, ...
+const JITTER_MS = 250;
+const MAX_RETRY_AFTER_MS = 30_000;
+
+let settings = { ...DEFAULTS };
+
+/**
+ * Replace the HTTP settings with the defaults plus `overrides`. For tests
+ * and scripts. `configureHttp()` restores the defaults. Returns a copy of
+ * the settings now in force.
+ */
+export function configureHttp(overrides = {}) {
+  settings = { ...DEFAULTS, ...overrides };
+  return { ...settings };
+}
+
+/**
+ * fetch(url, init) with a timeout, retries and a per-host queue.
+ *
+ * Resolves with the Response for any status that is not retried (2xx, 3xx,
+ * and 4xx other than 429). Throws AtsError once the retries are used up on
+ * a 429 or 5xx (with `.status`), or on a network error or a timeout waiting
+ * for the response to start.
+ */
+export async function atsFetch(url, init = {}) {
+  const host = new URL(url).hostname;
+  const { retries, sleep, timeoutMs } = settings;
+
+  for (let attempt = 1; ; attempt++) {
+    let resp;
+    try {
+      resp = await withHostSlot(host, () => fetchWithTimeout(url, init, timeoutMs));
+    } catch (err) {
+      if (attempt >= retries || isCertError(err)) {
+        const error = new AtsError(
+          ERROR_CODES.ATS_UNREACHABLE,
+          `${host}: ${describeCause(err, timeoutMs)} after ${attempts(attempt)}`
+        );
+        error.cause = err;
+        throw error;
+      }
+      await sleep(backoffMs(attempt));
+      continue;
+    }
+
+    if (!isRetried(resp.status)) return resp;
+    if (attempt >= retries) {
+      throw atsErrorFromStatus(resp.status, `${host}: HTTP ${resp.status} after ${attempts(attempt)}`);
+    }
+    await sleep(retryAfterMs(resp) ?? backoffMs(attempt));
+  }
+}
+
+/**
+ * Three-state probe outcome for an adapter's has(): true on 2xx, false on
+ * a 404, and an AtsError for anything else (401, 403, ...), so a board the
+ * probe could not check never reads as "not here" (issue #55). A 429 or
+ * 5xx never reaches this point: atsFetch throws on those itself.
+ */
+export function probeResult(resp, label) {
+  if (resp.ok) return true;
+  if (resp.status === 404) return false;
+  throw atsErrorFromStatus(resp.status, `${label}: ${resp.status}`);
+}
+
+// The abort covers connecting and waiting for the headers, and is cleared as
+// soon as fetch resolves so the body read that follows is never aborted
+// (see the header comment). The reason is a TimeoutError like the one
+// AbortSignal.timeout would raise, so describeCause reads both the same way.
+async function fetchWithTimeout(url, init, timeoutMs) {
+  const controller = new AbortController();
+  const timer = setTimeout(
+    () => controller.abort(new DOMException(`Timed out after ${timeoutMs}ms`, 'TimeoutError')),
+    timeoutMs
+  );
+  try {
+    return await globalThis.fetch(url, { ...init, signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function isRetried(status) {
+  return status === 429 || status >= 500;
+}
+
+function backoffMs(attempt) {
+  return BASE_BACKOFF_MS * 2 ** (attempt - 1) + Math.floor(Math.random() * JITTER_MS);
+}
+
+// Retry-After is either delay-seconds or an HTTP-date. Mocked responses may
+// carry no headers at all.
+function retryAfterMs(resp) {
+  const raw = resp.headers?.get?.('retry-after');
+  if (!raw) return null;
+  const seconds = Number(raw);
+  const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(raw) - Date.now();
+  if (!Number.isFinite(ms)) return null;
+  return Math.min(Math.max(ms, 0), MAX_RETRY_AFTER_MS);
+}
+
+// A certificate that fails validation fails the same way on the next
+// attempt, so retrying it only costs time. Seen live: every
+// {slug}.eu.teamtailor.com answers ERR_TLS_CERT_ALTNAME_INVALID.
+const CERT_ERROR = /^(?:ERR_TLS_CERT_ALTNAME_INVALID|CERT_HAS_EXPIRED|CERT_NOT_YET_VALID|DEPTH_ZERO_SELF_SIGNED_CERT|SELF_SIGNED_CERT_IN_CHAIN|UNABLE_TO_VERIFY_LEAF_SIGNATURE|UNABLE_TO_GET_ISSUER_CERT(?:_LOCALLY)?)$/;
+
+function isCertError(err) {
+  return CERT_ERROR.test(err?.cause?.code || '');
+}
+
+// undici reports socket failures as TypeError('fetch failed') with the OS
+// code on `cause`, and rejects an aborted request with the signal's reason,
+// here the TimeoutError from fetchWithTimeout.
+function describeCause(err, timeoutMs) {
+  if (err?.name === 'TimeoutError') return `timed out after ${timeoutMs}ms`;
+  const detail = err?.cause?.code || err?.cause?.message;
+  const message = err?.message || String(err);
+  return detail ? `${message} (${detail})` : message;
+}
+
+function attempts(n) {
+  return `${n} attempt${n === 1 ? '' : 's'}`;
+}
+
+// Per-host queue. A finished request hands its slot straight to the next
+// waiter (the count never dips in between), so the cap holds even when a new
+// caller arrives while a waiter is being woken.
+const hosts = new Map(); // hostname -> { active, waiting: [resolve] }
+
+async function withHostSlot(host, run) {
+  let slot = hosts.get(host);
+  if (!slot) {
+    slot = { active: 0, waiting: [] };
+    hosts.set(host, slot);
+  }
+  if (slot.active >= settings.perHost) {
+    await new Promise((resolve) => slot.waiting.push(resolve));
+  } else {
+    slot.active += 1;
+  }
+  try {
+    return await run();
+  } finally {
+    const next = slot.waiting.shift();
+    if (next) next();
+    else {
+      slot.active -= 1;
+      if (slot.active === 0) hosts.delete(host);
+    }
+  }
+}