jd-intel 0.9.0 → 0.11.0

package/README.md

@@ -144,8 +144,22 @@ const { jobs, total_matched } = await fetchJobsDetailed({
 });
 ```
 
+The same result says how the company was resolved and which boards answered:
+
+- `match`: `registry` (a known company, one adapter call), `probe` (not in the registry, every adapter asked) or `workday_override` (an explicit Workday config).
+- `company`: `{ key, name }` from the registry row on a registry match, null otherwise.
+- `boards`: one entry per board that answered, `{ ats, slug, name, site, board_url, org_name, org_url, jobs_found, matched, selected, scan }`. `jobs_found` counts the rows the board listed before any filter. Workday and SmartRecruiters filter their list before fetching details, so for them it is the list count, not the rows that came back (a lower bound when the list scan caps). `matched` is the rows left after filters and before `offset` and `limit`, so a board whose rows were all cut from the page still shows up. `board_url` is built from the slug. `org_name` and `org_url` are what the board states about itself, the organization name in the ATS response and the careers or company host its links point at, and they are null where the platform exposes nothing (Lever and Ashby expose neither); nothing is filled in from the slug or the registry name. A `probe` match is a slug match, not a confirmed identity: compare `org_name` and `org_url` with the company you meant, and open `board_url` when they are null, before treating the jobs as that company's.
+- `failed`: adapters that threw during discovery, `{ ats, slug, name, code, message }`, with `code` either `rate_limited` or `ats_unreachable`. When no board answered and a check failed, `fetchJobs` throws that error instead of returning `[]`, so an outage never reads as "not found".
+- `total_before_filters`: the sum of `jobs_found`. Zero matches with a positive `total_before_filters` means the company is hiring and nothing passed the filters, on every ATS.
+
+`detectAtsDetailed(company)` returns `{ boards, failed }`: registry rows first (`source: 'registry'`, never probed), then every live probe that answered (`source: 'probe'`), in platform order; `failed` lists the probes that could not be checked, with the same codes. `detectAts` keeps returning `[{ ats, slug }]`.
+
+A call the library cannot make throws `ArgumentError` (`code: 'invalid_args'`): no company, an unknown `ats`, or a `titleFilter` or `filter` that does not compile as a regex. It is thrown before any request goes out.
+
 CLI usage: `npx jd-intel fetch <company-slug> --title-filter "engineer" --posted-within-days 14`. Full filter reference [below](#filters-quick-reference).
 
+Each ATS request gives the server 10 seconds to start responding. A 429, a 5xx or a network error is retried up to three attempts with backoff (1s, 2s), honoring `Retry-After` when the ATS sends one, and at most 4 requests run at a time per host. A failure that outlasts the retries throws an `AtsError` whose `code` is `rate_limited` or `ats_unreachable`.
+
 Node.js 18+. No API keys. No configuration.
 
 ### Manual install (fallback)
@@ -226,7 +240,8 @@ No custom parsing per company.
 | `locationType` | `remote`, `hybrid`, `onsite`, or `unknown` when neither the platform nor the location text says |
 | `workplace` | `{ type, source }`. `type` repeats `locationType`; `source` is `ats` when the platform stated it, `text` when read from the location string, null when unknown |
 | `salary` | Min-max range with `currency`, plus `period` (`year`, `month`, `hour`, or null) and `source` (`ats` when the platform supplied it, `text` when parsed from the posting). Null when nothing is stated |
-| `description` | Full JD in clean markdown |
+| `description` | Full JD in clean markdown. Empty when `content.status` is `missing` |
+| `content` | `{ status, reason }`. `complete` when the posting was read. `missing` when Workday or SmartRecruiters listed the job but its detail request failed (`reason`: `http_503`, `http_429`, `network_error`), so `description` and `salary` are unknown |
 | `url` | Direct link to the posting |
 | `postedAt` | Publication date (when provided) |
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jd-intel",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "description": "Fetch and normalize job descriptions across seven major ATS (Greenhouse, Lever, Ashby, Workday, and more), for your AI assistant. No copy-paste.",
   "type": "module",
   "main": "src/index.js",

package/src/adapters/ashby.js

@@ -1,7 +1,7 @@
-import { normalize, extractSalaryFromText } from '../normalizer.js';
+import { normalize, extractSalaryFromText, WORKPLACE_TYPES } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { atsFetch, probeResult } from '../http.js';
 
-const API_URL = 'https://jobs.ashbyhq.com/api/non-user-graphql';
 const BOARD_URL = 'https://api.ashbyhq.com/posting-api/job-board';
 
 /**
@@ -9,23 +9,16 @@ const BOARD_URL = 'https://api.ashbyhq.com/posting-api/job-board';
  * Public API, no auth required.
  * Docs: https://developers.ashbyhq.com/docs/public-job-posting-api
  *
+ * REST only. The GraphQL fallback this adapter once carried never named a
+ * board, so it never returned a job, and it turned every REST 429 or 5xx
+ * into a silent empty result (issue #55).
+ *
  * @param {string} slug - Company slug (e.g., 'notion', 'linear')
  * @returns {Promise<Array>} Normalized job objects
  */
 export async function fetchAshby(slug) {
-  // Try the REST API first (simpler, includes compensation)
-  try {
-    const restJobs = await fetchAshbyRest(slug);
-    if (restJobs.length > 0) return restJobs;
-  } catch { /* fall through to GraphQL */ }
-
-  // Fallback: GraphQL API
-  return fetchAshbyGraphQL(slug);
-}
-
-async function fetchAshbyRest(slug) {
   const url = `${BOARD_URL}/${slug}?includeCompensation=true`;
-  const resp = await fetch(url);
+  const resp = await atsFetch(url);
 
   if (!resp.ok) {
     if (resp.status === 404) return [];
@@ -35,6 +28,10 @@ async function fetchAshbyRest(slug) {
   const data = await resp.json();
   const jobs = data.jobs || [];
 
+  // The REST response is { jobs, apiVersion }: no organization name, and
+  // every link is on jobs.ashbyhq.com. Nothing to report, so the board's
+  // org_name and org_url stay null (issue #58).
+
   return jobs.map(job => {
     const comp = job.compensation || {};
 
@@ -68,68 +65,14 @@ async function fetchAshbyRest(slug) {
   });
 }
 
-async function fetchAshbyGraphQL(slug) {
-  const query = `{
-    jobBoard {
-      title
-      jobPostings {
-        id
-        title
-        locationName
-        employmentType
-        descriptionHtml
-        publishedDate
-        compensationTierSummary
-      }
-    }
-  }`;
-
-  const resp = await fetch(API_URL, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({
-      operationName: 'ApiJobBoardWithTeams',
-      variables: { organizationHostedJobsPageName: slug },
-      query,
-    }),
-  });
-
-  if (!resp.ok) return [];
-
-  const data = await resp.json();
-  const board = data.data?.jobBoard;
-  if (!board) return [];
-
-  const postings = board.jobPostings || [];
-
-  return postings.map(job => normalize({
-    companySlug: slug,
-    company: board.title || slug,
-    title: job.title || '',
-    department: '',
-    location: job.locationName || '',
-    description: job.descriptionHtml || '',
-    url: `https://jobs.ashbyhq.com/${slug}/${job.id}`,
-    postedAt: job.publishedDate || null,
-    salary: null,
-    metadata: {
-      ashbyId: job.id,
-      employmentType: job.employmentType || '',
-      compensationSummary: job.compensationTierSummary || '',
-    },
-  }, 'ashby'));
-}
-
-const WORKPLACE_TYPES = { remote: 'remote', hybrid: 'hybrid', onsite: 'onsite' };
-
 /**
  * `workplaceType` is 'Remote', 'Hybrid' or 'OnSite'. `isRemote` is the
  * older flag and can only say remote, so it is the fallback when the type
  * is absent. false means nothing: the role may be hybrid or onsite.
  */
 function parseAshbyWorkplace(job) {
-  const type = WORKPLACE_TYPES[String(job.workplaceType || '').toLowerCase()];
-  if (type) return type;
+  const type = String(job.workplaceType || '').toLowerCase();
+  if (WORKPLACE_TYPES.has(type)) return type;
   return job.isRemote === true ? 'remote' : null;
 }
 
@@ -163,11 +106,10 @@ function parseAshbyCompensation(comp) {
   return parsed ? { ...parsed, source: 'ats' } : null;
 }
 
+/**
+ * Check if a company has an Ashby board. See probeResult for the outcomes.
+ */
 export async function hasAshby(slug) {
-  try {
-    const resp = await fetch(`${BOARD_URL}/${slug}`, { method: 'HEAD' });
-    return resp.ok;
-  } catch {
-    return false;
-  }
+  const resp = await atsFetch(`${BOARD_URL}/${slug}`, { method: 'HEAD' });
+  return probeResult(resp, `Ashby probe for ${slug}`);
 }

package/src/adapters/greenhouse.js

@@ -1,5 +1,6 @@
-import { normalize, decodeEntities } from '../normalizer.js';
+import { normalize, decodeEntities, periodWord } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { atsFetch, probeResult } from '../http.js';
 
 const BASE_URL = 'https://boards-api.greenhouse.io/v1/boards';
 
@@ -9,11 +10,14 @@ const BASE_URL = 'https://boards-api.greenhouse.io/v1/boards';
  * Docs: https://developers.greenhouse.io/job-board.html
  *
  * @param {string} slug - Company slug (e.g., 'stripe', 'notion')
+ * @param {object} [ctx] - { report }; report is called once with
+ *   { org_name, org_url } when given
  * @returns {Promise<Array>} Normalized job objects
  */
-export async function fetchGreenhouse(slug) {
-  const url = `${BASE_URL}/${slug}/jobs?content=true`;
-  const resp = await fetch(url);
+export async function fetchGreenhouse(slug, ctx = {}) {
+  // pay_transparency adds pay_input_ranges to each row (issue #86).
+  const url = `${BASE_URL}/${slug}/jobs?content=true&pay_transparency=true`;
+  const resp = await atsFetch(url);
 
   if (!resp.ok) {
     if (resp.status === 404) return []; // Company not found or no jobs
@@ -23,29 +27,91 @@ export async function fetchGreenhouse(slug) {
   const data = await resp.json();
   const jobs = data.jobs || [];
 
-  return jobs.map(job => normalize({
-    companySlug: slug,
-    company: data.name || slug,
-    title: job.title || '',
-    department: job.departments?.[0]?.name || '',
-    location: job.location?.name || '',
-    workplace: parseGreenhouseWorkplace(job.metadata),
-    // `content` arrives HTML-escaped (`&lt;p&gt;`). Decode that outer layer
-    // once so normalize() sees real tags; it strips and decodes the rest.
-    description: decodeEntities(job.content || ''),
-    url: job.absolute_url || '',
-    // updated_at is an edit time that many boards bulk-refresh, so it is not
-    // a posting date. first_published is. Fallback covers boards without it (#69).
-    postedAt: job.first_published || job.updated_at || null,
-    salary: null, // list endpoint has no structured pay; normalizer parses the pay transparency text
-    metadata: {
-      greenhouseId: job.id,
-      internal_job_id: job.internal_job_id,
-      departments: job.departments?.map(d => d.name) || [],
-      offices: job.offices?.map(o => o.name) || [],
-      updatedAt: job.updated_at,
-    },
-  }, 'greenhouse'));
+  // The list response has no top-level name, but each row carries the
+  // board's company_name. Its only links are job-boards.greenhouse.io, so
+  // there is no company host to report (issue #58).
+  if (typeof ctx.report === 'function') {
+    ctx.report({
+      org_name: jobs.find(j => j.company_name)?.company_name || null,
+      org_url: null,
+    });
+  }
+
+  return jobs.map(job => {
+    const payRanges = parsePayRanges(job.pay_input_ranges);
+    return normalize({
+      companySlug: slug,
+      company: data.name || slug,
+      title: job.title || '',
+      department: job.departments?.[0]?.name || '',
+      location: job.location?.name || '',
+      workplace: parseGreenhouseWorkplace(job.metadata),
+      // `content` arrives HTML-escaped (`&lt;p&gt;`). Decode that outer layer
+      // once so normalize() sees real tags; it strips and decodes the rest.
+      description: decodeEntities(job.content || ''),
+      url: job.absolute_url || '',
+      // updated_at is an edit time that many boards bulk-refresh, so it is not
+      // a posting date. first_published is. Fallback covers boards without it (#69).
+      postedAt: job.first_published || job.updated_at || null,
+      salary: salaryFromRanges(payRanges), // null without structured pay; normalize() then parses the text
+      metadata: {
+        greenhouseId: job.id,
+        internal_job_id: job.internal_job_id,
+        departments: job.departments?.map(d => d.name) || [],
+        offices: job.offices?.map(o => o.name) || [],
+        updatedAt: job.updated_at,
+        payRanges,
+      },
+    }, 'greenhouse');
+  });
+}
+
+/**
+ * `pay_input_ranges` is the board's pay transparency data:
+ * [{ min_cents, max_cents, currency_type, title, blurb }]. The blurb is
+ * boilerplate already rendered in the description, so it is dropped, and
+ * the platform can send the same entry twice, so entries are deduplicated.
+ * A board that does not publish pay sends an empty array or no key.
+ */
+function parsePayRanges(ranges) {
+  const seen = new Set();
+  const out = [];
+  for (const r of Array.isArray(ranges) ? ranges : []) {
+    const range = {
+      title: r.title || '',
+      min: Number.isFinite(r.min_cents) ? r.min_cents / 100 : null,
+      max: Number.isFinite(r.max_cents) ? r.max_cents / 100 : null,
+      currency: r.currency_type || 'USD',
+    };
+    const key = JSON.stringify(range);
+    if ((range.min === null && range.max === null) || seen.has(key)) continue;
+    seen.add(key);
+    out.push(range);
+  }
+  return out;
+}
+
+/**
+ * One salary from the ranges. Ranges in one currency span (lowest min,
+ * highest max), as the Ashby adapter does for its tiers; with mixed
+ * currencies the first range stands alone. Every range stays in
+ * metadata.payRanges. The field has no period, so the period is the one
+ * the range titles state ("Annual", "Hourly") and null when they state
+ * none or disagree: an 'ats' value carries no guessed period.
+ */
+function salaryFromRanges(ranges) {
+  if (ranges.length === 0) return null;
+  const used = ranges.every(r => r.currency === ranges[0].currency) ? ranges : [ranges[0]];
+  const mins = used.map(r => r.min).filter(v => v !== null);
+  const maxes = used.map(r => r.max).filter(v => v !== null);
+  const periods = new Set(used.map(r => periodWord(r.title)));
+  return {
+    min: mins.length ? Math.min(...mins) : null,
+    max: maxes.length ? Math.max(...maxes) : null,
+    currency: used[0].currency,
+    period: periods.size === 1 ? [...periods][0] : null,
+    source: 'ats',
+  };
 }
 
 /**
@@ -66,13 +132,9 @@ function parseGreenhouseWorkplace(metadata) {
 }
 
 /**
- * Check if a company has a Greenhouse board.
+ * Check if a company has a Greenhouse board. See probeResult for the outcomes.
  */
 export async function hasGreenhouse(slug) {
-  try {
-    const resp = await fetch(`${BASE_URL}/${slug}`, { method: 'HEAD' });
-    return resp.ok;
-  } catch {
-    return false;
-  }
+  const resp = await atsFetch(`${BASE_URL}/${slug}`, { method: 'HEAD' });
+  return probeResult(resp, `Greenhouse probe for ${slug}`);
 }

package/src/adapters/index.js

@@ -1,19 +1,29 @@
-export { fetchGreenhouse, hasGreenhouse } from './greenhouse.js';
-export { fetchLever, hasLever } from './lever.js';
-export { fetchAshby, hasAshby } from './ashby.js';
-export { fetchSmartrecruiters, hasSmartrecruiters } from './smartrecruiters.js';
-export { fetchTeamtailor, hasTeamtailor } from './teamtailor.js';
-export { fetchRecruitee, hasRecruitee } from './recruitee.js';
-export { fetchWorkday, hasWorkday } from './workday.js';
+import { fetchGreenhouse, hasGreenhouse } from './greenhouse.js';
+import { fetchLever, hasLever } from './lever.js';
+import { fetchAshby, hasAshby } from './ashby.js';
+import { fetchSmartrecruiters, hasSmartrecruiters } from './smartrecruiters.js';
+import { fetchTeamtailor, hasTeamtailor } from './teamtailor.js';
+import { fetchRecruitee, hasRecruitee } from './recruitee.js';
+import { fetchWorkday, hasWorkday } from './workday.js';
+
+export {
+  fetchGreenhouse, hasGreenhouse,
+  fetchLever, hasLever,
+  fetchAshby, hasAshby,
+  fetchSmartrecruiters, hasSmartrecruiters,
+  fetchTeamtailor, hasTeamtailor,
+  fetchRecruitee, hasRecruitee,
+  fetchWorkday, hasWorkday,
+};
 
 export const ADAPTERS = {
-  greenhouse: { fetch: (...args) => import('./greenhouse.js').then(m => m.fetchGreenhouse(...args)), has: (...args) => import('./greenhouse.js').then(m => m.hasGreenhouse(...args)) },
-  lever: { fetch: (...args) => import('./lever.js').then(m => m.fetchLever(...args)), has: (...args) => import('./lever.js').then(m => m.hasLever(...args)) },
-  ashby: { fetch: (...args) => import('./ashby.js').then(m => m.fetchAshby(...args)), has: (...args) => import('./ashby.js').then(m => m.hasAshby(...args)) },
-  smartrecruiters: { fetch: (...args) => import('./smartrecruiters.js').then(m => m.fetchSmartrecruiters(...args)), has: (...args) => import('./smartrecruiters.js').then(m => m.hasSmartrecruiters(...args)) },
-  teamtailor: { fetch: (...args) => import('./teamtailor.js').then(m => m.fetchTeamtailor(...args)), has: (...args) => import('./teamtailor.js').then(m => m.hasTeamtailor(...args)) },
-  recruitee: { fetch: (...args) => import('./recruitee.js').then(m => m.fetchRecruitee(...args)), has: (...args) => import('./recruitee.js').then(m => m.hasRecruitee(...args)) },
-  workday: { fetch: (...args) => import('./workday.js').then(m => m.fetchWorkday(...args)), has: (...args) => import('./workday.js').then(m => m.hasWorkday(...args)) },
+  greenhouse: { fetch: fetchGreenhouse, has: hasGreenhouse },
+  lever: { fetch: fetchLever, has: hasLever },
+  ashby: { fetch: fetchAshby, has: hasAshby },
+  smartrecruiters: { fetch: fetchSmartrecruiters, has: hasSmartrecruiters },
+  teamtailor: { fetch: fetchTeamtailor, has: hasTeamtailor },
+  recruitee: { fetch: fetchRecruitee, has: hasRecruitee },
+  workday: { fetch: fetchWorkday, has: hasWorkday },
 };
 
 export const ATS_NAMES = Object.keys(ADAPTERS);

package/src/adapters/lever.js

@@ -1,11 +1,10 @@
 import { normalize, extractSalaryFromText } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { atsFetch, probeResult } from '../http.js';
 
 const BASE_URL = 'https://api.lever.co/v0/postings';
 
 const PERIODS = { 'per-year-salary': 'year', 'per-month-salary': 'month', 'per-hour-wage': 'hour' };
-// Lever's workplaceType is one of these or 'unspecified'.
-const WORKPLACE_TYPES = new Set(['remote', 'hybrid', 'onsite']);
 
 /**
  * Fetch all jobs from a Lever job board.
@@ -17,7 +16,7 @@ const WORKPLACE_TYPES = new Set(['remote', 'hybrid', 'onsite']);
  */
 export async function fetchLever(slug) {
   const url = `${BASE_URL}/${slug}?mode=json`;
-  const resp = await fetch(url);
+  const resp = await atsFetch(url);
 
   if (!resp.ok) {
     if (resp.status === 404) return [];
@@ -27,6 +26,10 @@ export async function fetchLever(slug) {
   const jobs = await resp.json();
   if (!Array.isArray(jobs)) return [];
 
+  // The postings response is a bare array of jobs: no organization name
+  // anywhere, and every link is on jobs.lever.co. Nothing to report, so
+  // the board's org_name and org_url stay null (issue #58).
+
   return jobs.map(job => normalize({
     companySlug: slug,
     // Lever's API doesn't return the company name at the board or job level,
@@ -37,7 +40,8 @@ export async function fetchLever(slug) {
     department: job.categories?.department || job.categories?.team || '',
     location: job.categories?.location || '',
     locations: job.categories?.allLocations || [],
-    workplace: WORKPLACE_TYPES.has(job.workplaceType) ? job.workplaceType : null,
+    // 'remote', 'hybrid', 'onsite' or 'unspecified'; normalize() ignores the last.
+    workplace: job.workplaceType,
     description: buildDescription(job),
     url: job.hostedUrl || '',
     postedAt: job.createdAt ? new Date(job.createdAt).toISOString() : null,
@@ -102,11 +106,10 @@ function titleCaseSlug(slug) {
   return slug.charAt(0).toUpperCase() + slug.slice(1);
 }
 
+/**
+ * Check if a company has a Lever board. See probeResult for the outcomes.
+ */
 export async function hasLever(slug) {
-  try {
-    const resp = await fetch(`${BASE_URL}/${slug}?mode=json`, { method: 'HEAD' });
-    return resp.ok;
-  } catch {
-    return false;
-  }
+  const resp = await atsFetch(`${BASE_URL}/${slug}?mode=json`, { method: 'HEAD' });
+  return probeResult(resp, `Lever probe for ${slug}`);
 }

package/src/adapters/recruitee.js

@@ -1,5 +1,7 @@
 import { normalize } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { atsFetch, probeResult } from '../http.js';
+import { orgHost } from '../boards.js';
 
 /**
  * Fetch jobs from a Recruitee career site.
@@ -18,11 +20,13 @@ import { atsErrorFromStatus } from '../errors.js';
  * both are joined before normalize() strips them (issue #65).
  *
  * @param {string} slug - Recruitee company subdomain (e.g., 'vandebron')
+ * @param {object} [ctx] - { report }; report is called once with
+ *   { org_name, org_url } when given
  * @returns {Promise<Array>} Normalized job objects
  */
-export async function fetchRecruitee(slug) {
+export async function fetchRecruitee(slug, ctx = {}) {
   const url = `https://${slug}.recruitee.com/api/offers/`;
-  const resp = await fetch(url);
+  const resp = await atsFetch(url);
 
   if (!resp.ok) {
     if (resp.status === 404) return []; // No Recruitee site for this slug
@@ -32,6 +36,16 @@ export async function fetchRecruitee(slug) {
   const data = await resp.json();
   const offers = data.offers || [];
 
+  // The response is { offers } only, so the identity lives on the rows:
+  // company_name, and careers_url, which sits on the company's own careers
+  // domain when the site has one and on {slug}.recruitee.com otherwise.
+  if (typeof ctx.report === 'function') {
+    ctx.report({
+      org_name: offers.find(o => o.company_name)?.company_name || null,
+      org_url: orgHost(offers.find(o => o.careers_url)?.careers_url),
+    });
+  }
+
   return offers.map(offer => {
     const place = [offer.city, offer.country].filter(Boolean).join(', ');
     let location = place;
@@ -111,13 +125,9 @@ function toAmount(value) {
 }
 
 /**
- * Check if a company has a Recruitee career site.
+ * Check if a company has a Recruitee career site. See probeResult for the outcomes.
  */
 export async function hasRecruitee(slug) {
-  try {
-    const resp = await fetch(`https://${slug}.recruitee.com/api/offers/`);
-    return resp.ok;
-  } catch {
-    return false;
-  }
+  const resp = await atsFetch(`https://${slug}.recruitee.com/api/offers/`);
+  return probeResult(resp, `Recruitee probe for ${slug}`);
 }