jd-intel 0.8.3 → 0.10.0

package/README.md

@@ -131,8 +131,35 @@ const jobs = await fetchJobs({
 });
 ```
 
+Results come back newest first by `postedAt`, undated last (`order: 'board'` keeps the ATS's own order). `fetchJobs` returns the page as an array. `fetchJobsDetailed` returns the same page plus `total_matched`, the number of matches before `offset` and `limit`, so you can tell a small board from a cut and page through the rest:
+
+```js
+import { fetchJobsDetailed } from 'jd-intel';
+
+const { jobs, total_matched } = await fetchJobsDetailed({
+  company: '<your-target-company>',
+  titleFilter: 'engineer',
+  limit: 20,
+  offset: 20, // second page
+});
+```
+
+The same result says how the company was resolved and which boards answered:
+
+- `match`: `registry` (a known company, one adapter call), `probe` (not in the registry, every adapter asked) or `workday_override` (an explicit Workday config).
+- `company`: `{ key, name }` from the registry row on a registry match, null otherwise.
+- `boards`: one entry per board that answered, `{ ats, slug, name, site, board_url, org_name, org_url, jobs_found, matched, selected, scan }`. `jobs_found` counts the rows the board listed before any filter. Workday and SmartRecruiters filter their list before fetching details, so for them it is the list count, not the rows that came back (a lower bound when the list scan caps). `matched` is the rows left after filters and before `offset` and `limit`, so a board whose rows were all cut from the page still shows up. `board_url` is built from the slug. `org_name` and `org_url` are what the board states about itself, the organization name in the ATS response and the careers or company host its links point at, and they are null where the platform exposes nothing (Lever and Ashby expose neither); nothing is filled in from the slug or the registry name. A `probe` match is a slug match, not a confirmed identity: compare `org_name` and `org_url` with the company you meant, and open `board_url` when they are null, before treating the jobs as that company's.
+- `failed`: adapters that threw during discovery, `{ ats, slug, name, code, message }`, with `code` either `rate_limited` or `ats_unreachable`. When no board answered and a check failed, `fetchJobs` throws that error instead of returning `[]`, so an outage never reads as "not found".
+- `total_before_filters`: the sum of `jobs_found`. Zero matches with a positive `total_before_filters` means the company is hiring and nothing passed the filters, on every ATS.
+
+`detectAtsDetailed(company)` returns `{ boards, failed }`: registry rows first (`source: 'registry'`, never probed), then every live probe that answered (`source: 'probe'`), in platform order; `failed` lists the probes that could not be checked, with the same codes. `detectAts` keeps returning `[{ ats, slug }]`.
+
+A call the library cannot make throws `ArgumentError` (`code: 'invalid_args'`): no company, an unknown `ats`, or a `titleFilter` or `filter` that does not compile as a regex. It is thrown before any request goes out.
+
 CLI usage: `npx jd-intel fetch <company-slug> --title-filter "engineer" --posted-within-days 14`. Full filter reference [below](#filters-quick-reference).
 
+Each ATS request gives the server 10 seconds to start responding. A 429, a 5xx or a network error is retried up to three attempts with backoff (1s, 2s), honoring `Retry-After` when the ATS sends one, and at most 4 requests run at a time per host. A failure that outlasts the retries throws an `AtsError` whose `code` is `rate_limited` or `ats_unreachable`.
+
 Node.js 18+. No API keys. No configuration.
 
 ### Manual install (fallback)
@@ -189,8 +216,10 @@ Every job normalizes to one schema, across every platform:
   "title": "Senior Software Engineer, Platform",
   "department": "Engineering",
   "location": "Remote - US",
+  "locations": ["Remote - US", "Toronto, Canada"],
   "locationType": "remote",
-  "salary": { "min": 180000, "max": 240000, "currency": "USD" },
+  "workplace": { "type": "remote", "source": "ats" },
+  "salary": { "min": 180000, "max": 240000, "currency": "USD", "period": "year", "source": "text" },
   "description": "Design and build the API surface our customers integrate against...",
   "url": "https://boards.example.com/jobs/12345",
   "postedAt": "2026-04-10T14:30:00Z"
@@ -206,9 +235,11 @@ No custom parsing per company.
 | `title` | Full job title |
 | `company` | Normalized company name |
 | `department` | Team or department (when provided) |
-| `location` | City, state, country, or remote |
-| `locationType` | `remote`, `hybrid`, or `onsite` |
-| `salary` | Min-max range with currency (when available) |
+| `location` | Primary location: city, state, country, or remote |
+| `locations` | Every location the posting is open in, primary first. The location filters check each entry |
+| `locationType` | `remote`, `hybrid`, `onsite`, or `unknown` when neither the platform nor the location text says |
+| `workplace` | `{ type, source }`. `type` repeats `locationType`; `source` is `ats` when the platform stated it, `text` when read from the location string, null when unknown |
+| `salary` | Min-max range with `currency`, plus `period` (`year`, `month`, `hour`, or null) and `source` (`ats` when the platform supplied it, `text` when parsed from the posting). Null when nothing is stated |
 | `description` | Full JD in clean markdown |
 | `url` | Direct link to the posting |
 | `postedAt` | Publication date (when provided) |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jd-intel",
-  "version": "0.8.3",
+  "version": "0.10.0",
   "description": "Fetch and normalize job descriptions across seven major ATS (Greenhouse, Lever, Ashby, Workday, and more), for your AI assistant. No copy-paste.",
   "type": "module",
   "main": "src/index.js",

package/src/adapters/ashby.js

@@ -1,7 +1,7 @@
-import { normalize } from '../normalizer.js';
+import { normalize, extractSalaryFromText } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { atsFetch, probeResult } from '../http.js';
 
-const API_URL = 'https://jobs.ashbyhq.com/api/non-user-graphql';
 const BOARD_URL = 'https://api.ashbyhq.com/posting-api/job-board';
 
 /**
@@ -9,23 +9,18 @@ const BOARD_URL = 'https://api.ashbyhq.com/posting-api/job-board';
  * Public API, no auth required.
  * Docs: https://developers.ashbyhq.com/docs/public-job-posting-api
  *
+ * REST only. The GraphQL fallback this adapter once carried never named a
+ * board, so it never returned a job, and it turned every REST 429 or 5xx
+ * into a silent empty result (issue #55).
+ *
  * @param {string} slug - Company slug (e.g., 'notion', 'linear')
+ * @param {object} [ctx] - { report }; report is called once with
+ *   { ats, org_name, org_url } when given
  * @returns {Promise<Array>} Normalized job objects
  */
-export async function fetchAshby(slug) {
-  // Try the REST API first (simpler, includes compensation)
-  try {
-    const restJobs = await fetchAshbyRest(slug);
-    if (restJobs.length > 0) return restJobs;
-  } catch { /* fall through to GraphQL */ }
-
-  // Fallback: GraphQL API
-  return fetchAshbyGraphQL(slug);
-}
-
-async function fetchAshbyRest(slug) {
+export async function fetchAshby(slug, ctx = {}) {
   const url = `${BOARD_URL}/${slug}?includeCompensation=true`;
-  const resp = await fetch(url);
+  const resp = await atsFetch(url);
 
   if (!resp.ok) {
     if (resp.status === 404) return [];
@@ -35,104 +30,92 @@ async function fetchAshbyRest(slug) {
   const data = await resp.json();
   const jobs = data.jobs || [];
 
+  // The REST response is { jobs, apiVersion }: no organization name, and
+  // every link is on jobs.ashbyhq.com. Both null (issue #58).
+  if (typeof ctx.report === 'function') {
+    ctx.report({ ats: 'ashby', org_name: null, org_url: null });
+  }
+
   return jobs.map(job => {
-    const salary = parseAshbyCompensation(job.compensation);
+    const comp = job.compensation || {};
 
     return normalize({
       companySlug: slug,
       company: data.organizationName || slug,
       title: job.title || '',
-      department: job.departmentName || '',
+      department: job.department || '',
       location: job.location || '',
+      locations: (job.secondaryLocations || []).map(l => l?.location || ''),
+      workplace: parseAshbyWorkplace(job),
       description: job.descriptionHtml || job.descriptionPlain || '',
       url: `https://jobs.ashbyhq.com/${slug}/${job.id}`,
       postedAt: job.publishedAt || null,
-      salary,
+      salary: parseAshbyCompensation(comp),
       metadata: {
         ashbyId: job.id,
         employmentType: job.employmentType || '',
         isRemote: job.isRemote || false,
-        team: job.teamName || '',
+        team: job.team || '',
+        // The rendered summaries keep what min/max drop: "Offers Equity",
+        // "Multiple Ranges", and per-location tiers labelled OTE.
+        compensationSummary: comp.compensationTierSummary || '',
+        compensationTiers: (comp.compensationTiers || []).map(tier => ({
+          title: tier.title || '',
+          summary: tier.tierSummary || '',
+          additionalInformation: tier.additionalInformation || '',
+        })),
       },
     }, 'ashby');
   });
 }
 
-async function fetchAshbyGraphQL(slug) {
-  const query = `{
-    jobBoard {
-      title
-      jobPostings {
-        id
-        title
-        locationName
-        employmentType
-        descriptionHtml
-        publishedDate
-        compensationTierSummary
-      }
-    }
-  }`;
-
-  const resp = await fetch(API_URL, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({
-      operationName: 'ApiJobBoardWithTeams',
-      variables: { organizationHostedJobsPageName: slug },
-      query,
-    }),
-  });
-
-  if (!resp.ok) return [];
-
-  const data = await resp.json();
-  const board = data.data?.jobBoard;
-  if (!board) return [];
-
-  const postings = board.jobPostings || [];
+const WORKPLACE_TYPES = { remote: 'remote', hybrid: 'hybrid', onsite: 'onsite' };
 
-  return postings.map(job => normalize({
-    companySlug: slug,
-    company: board.title || slug,
-    title: job.title || '',
-    department: '',
-    location: job.locationName || '',
-    description: job.descriptionHtml || '',
-    url: `https://jobs.ashbyhq.com/${slug}/${job.id}`,
-    postedAt: job.publishedDate || null,
-    salary: null,
-    metadata: {
-      ashbyId: job.id,
-      employmentType: job.employmentType || '',
-      compensationSummary: job.compensationTierSummary || '',
-    },
-  }, 'ashby'));
+/**
+ * `workplaceType` is 'Remote', 'Hybrid' or 'OnSite'. `isRemote` is the
+ * older flag and can only say remote, so it is the fallback when the type
+ * is absent. false means nothing: the role may be hybrid or onsite.
+ */
+function parseAshbyWorkplace(job) {
+  const type = WORKPLACE_TYPES[String(job.workplaceType || '').toLowerCase()];
+  if (type) return type;
+  return job.isRemote === true ? 'remote' : null;
 }
 
+const INTERVAL_PERIOD = { '1 YEAR': 'year', '1 MONTH': 'month', '1 HOUR': 'hour' };
+
+/**
+ * Read pay from Ashby's `compensation` object (issue #67).
+ *
+ * `summaryComponents` carries one structured entry per component type
+ * (Salary, Bonus, Commission, Equity); the Salary entry spans every tier.
+ * `scrapeableCompensationSalarySummary` and `compensationTierSummary` are
+ * the rendered strings. A board that publishes no pay still sends the
+ * object, with null summaries and empty arrays, so a miss here has to
+ * return null for the normalizer's text fallback to run.
+ */
 function parseAshbyCompensation(comp) {
-  if (!comp) return null;
-  // Ashby compensation can be a string or structured object
-  if (typeof comp === 'string') {
-    const match = comp.match(/\$?([\d,]+)\s*[-–]\s*\$?([\d,]+)/);
-    if (!match) return null;
+  const salary = (comp.summaryComponents || []).find(c => c.compensationType === 'Salary');
+  if (salary && (salary.minValue != null || salary.maxValue != null)) {
     return {
-      min: parseInt(match[1].replace(/,/g, '')),
-      max: parseInt(match[2].replace(/,/g, '')),
-      currency: 'USD',
+      min: salary.minValue ?? null,
+      max: salary.maxValue ?? null,
+      currency: salary.currencyCode || 'USD',
+      period: INTERVAL_PERIOD[salary.interval] || null,
+      source: 'ats',
     };
   }
-  if (comp.min && comp.max) {
-    return { min: comp.min, max: comp.max, currency: comp.currency || 'USD' };
-  }
-  return null;
+  // The summaries are still the ATS's own compensation field, so a range
+  // read out of one counts as source 'ats'.
+  const parsed = extractSalaryFromText(comp.scrapeableCompensationSalarySummary)
+    || extractSalaryFromText(comp.compensationTierSummary);
+  return parsed ? { ...parsed, source: 'ats' } : null;
 }
 
+/**
+ * Check if a company has an Ashby board. See probeResult for the outcomes.
+ */
 export async function hasAshby(slug) {
-  try {
-    const resp = await fetch(`${BOARD_URL}/${slug}`, { method: 'HEAD' });
-    return resp.ok;
-  } catch {
-    return false;
-  }
+  const resp = await atsFetch(`${BOARD_URL}/${slug}`, { method: 'HEAD' });
+  return probeResult(resp, `Ashby probe for ${slug}`);
 }

package/src/adapters/greenhouse.js

@@ -1,5 +1,6 @@
-import { normalize, stripHtml } from '../normalizer.js';
+import { normalize, decodeEntities } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { atsFetch, probeResult } from '../http.js';
 
 const BASE_URL = 'https://boards-api.greenhouse.io/v1/boards';
 
@@ -9,11 +10,13 @@ const BASE_URL = 'https://boards-api.greenhouse.io/v1/boards';
  * Docs: https://developers.greenhouse.io/job-board.html
  *
  * @param {string} slug - Company slug (e.g., 'stripe', 'notion')
+ * @param {object} [ctx] - { report }; report is called once with
+ *   { ats, org_name, org_url } when given
  * @returns {Promise<Array>} Normalized job objects
  */
-export async function fetchGreenhouse(slug) {
+export async function fetchGreenhouse(slug, ctx = {}) {
   const url = `${BASE_URL}/${slug}/jobs?content=true`;
-  const resp = await fetch(url);
+  const resp = await atsFetch(url);
 
   if (!resp.ok) {
     if (resp.status === 404) return []; // Company not found or no jobs
@@ -23,33 +26,63 @@ export async function fetchGreenhouse(slug) {
   const data = await resp.json();
   const jobs = data.jobs || [];
 
+  // The list response has no top-level name, but each row carries the
+  // board's company_name. Its only links are job-boards.greenhouse.io, so
+  // there is no company host to report (issue #58).
+  if (typeof ctx.report === 'function') {
+    ctx.report({
+      ats: 'greenhouse',
+      org_name: jobs.find(j => j.company_name)?.company_name || null,
+      org_url: null,
+    });
+  }
+
   return jobs.map(job => normalize({
     companySlug: slug,
     company: data.name || slug,
     title: job.title || '',
     department: job.departments?.[0]?.name || '',
     location: job.location?.name || '',
-    description: stripHtml(job.content || ''),
+    workplace: parseGreenhouseWorkplace(job.metadata),
+    // `content` arrives HTML-escaped (`&lt;p&gt;`). Decode that outer layer
+    // once so normalize() sees real tags; it strips and decodes the rest.
+    description: decodeEntities(job.content || ''),
     url: job.absolute_url || '',
-    postedAt: job.updated_at || null,
-    salary: null, // Greenhouse doesn't expose salary in public API
+    // updated_at is an edit time that many boards bulk-refresh, so it is not
+    // a posting date. first_published is. Fallback covers boards without it (#69).
+    postedAt: job.first_published || job.updated_at || null,
+    salary: null, // list endpoint has no structured pay; normalizer parses the pay transparency text
     metadata: {
       greenhouseId: job.id,
       internal_job_id: job.internal_job_id,
       departments: job.departments?.map(d => d.name) || [],
       offices: job.offices?.map(o => o.name) || [],
+      updatedAt: job.updated_at,
     },
   }, 'greenhouse'));
 }
 
 /**
- * Check if a company has a Greenhouse board.
+ * Greenhouse has no native workplace field. Boards that track it define a
+ * custom field ("Location Type", "Workplace Type") that arrives in the
+ * job's `metadata[]`, with `value` a string for single-select fields and
+ * an array for multi-select. Values seen: On-Site, Hybrid (Travel-Required),
+ * Remote. Anything else is no signal and the location string decides.
+ */
+function parseGreenhouseWorkplace(metadata) {
+  const field = (metadata || []).find(m => /location type|workplace type/i.test(m?.name || ''));
+  if (!field) return null;
+  const value = [].concat(field.value ?? []).join(' ').toLowerCase();
+  if (/remote/.test(value)) return 'remote';
+  if (/hybrid/.test(value)) return 'hybrid';
+  if (/on-?site/.test(value)) return 'onsite';
+  return null;
+}
+
+/**
+ * Check if a company has a Greenhouse board. See probeResult for the outcomes.
  */
 export async function hasGreenhouse(slug) {
-  try {
-    const resp = await fetch(`${BASE_URL}/${slug}`, { method: 'HEAD' });
-    return resp.ok;
-  } catch {
-    return false;
-  }
+  const resp = await atsFetch(`${BASE_URL}/${slug}`, { method: 'HEAD' });
+  return probeResult(resp, `Greenhouse probe for ${slug}`);
 }

package/src/adapters/lever.js

@@ -1,18 +1,26 @@
-import { normalize, stripHtml } from '../normalizer.js';
+import { normalize, extractSalaryFromText } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { atsFetch, probeResult } from '../http.js';
 
 const BASE_URL = 'https://api.lever.co/v0/postings';
 
+const PERIODS = { 'per-year-salary': 'year', 'per-month-salary': 'month', 'per-hour-wage': 'hour' };
+// Lever's workplaceType is one of these or 'unspecified'.
+const WORKPLACE_TYPES = new Set(['remote', 'hybrid', 'onsite']);
+
 /**
  * Fetch all jobs from a Lever job board.
  * Public API, no auth required.
+ * Docs: https://github.com/lever/postings-api
  *
  * @param {string} slug - Company slug (e.g., 'stripe', 'figma')
+ * @param {object} [ctx] - { report }; report is called once with
+ *   { ats, org_name, org_url } when given
  * @returns {Promise<Array>} Normalized job objects
  */
-export async function fetchLever(slug) {
+export async function fetchLever(slug, ctx = {}) {
   const url = `${BASE_URL}/${slug}?mode=json`;
-  const resp = await fetch(url);
+  const resp = await atsFetch(url);
 
   if (!resp.ok) {
     if (resp.status === 404) return [];
@@ -22,30 +30,78 @@ export async function fetchLever(slug) {
   const jobs = await resp.json();
   if (!Array.isArray(jobs)) return [];
 
-  return jobs.map(job => {
-    const salary = parseLeverSalary(job.categories?.commitment, job.text);
+  // The postings response is a bare array of jobs: no organization name
+  // anywhere, and every link is on jobs.lever.co. Both null (issue #58).
+  if (typeof ctx.report === 'function') {
+    ctx.report({ ats: 'lever', org_name: null, org_url: null });
+  }
+
+  return jobs.map(job => normalize({
+    companySlug: slug,
+    // Lever's API doesn't return the company name at the board or job level,
+    // so the slug is the honest fallback. `categories.team` is the team within
+    // the company ("Payments Platform"), not the company itself.
+    company: titleCaseSlug(slug),
+    title: job.text || '',
+    department: job.categories?.department || job.categories?.team || '',
+    location: job.categories?.location || '',
+    locations: job.categories?.allLocations || [],
+    workplace: WORKPLACE_TYPES.has(job.workplaceType) ? job.workplaceType : null,
+    description: buildDescription(job),
+    url: job.hostedUrl || '',
+    postedAt: job.createdAt ? new Date(job.createdAt).toISOString() : null,
+    salary: parseLeverSalary(job.salaryRange, job.text),
+    metadata: {
+      leverId: job.id,
+      team: job.categories?.team || '',
+      commitment: job.categories?.commitment || '', // Full-time, Part-time, etc.
+      workplaceType: job.workplaceType || '',
+      salaryDescription: job.salaryDescriptionPlain || '',
+    },
+  }, 'lever'));
+}
+
+/**
+ * Lever splits a posting across `description` (company intro plus overview),
+ * `lists` (one `{text, content}` per section: responsibilities, requirements,
+ * location details) and `additional` (benefits, EEO). Only the first used to
+ * reach the description, so requirements were invisible to filters and to
+ * the assistant (issue #64). Reassemble the whole posting as HTML and let
+ * normalize() render the headings and bullets.
+ */
+function buildDescription(job) {
+  const parts = [job.description || job.descriptionPlain || ''];
+  for (const list of job.lists || []) {
+    const heading = (list.text || '').trim();
+    parts.push((heading ? `<h3>${escapeHtml(heading)}</h3>` : '') + (list.content || ''));
+  }
+  parts.push(job.additional || job.additionalPlain || '');
+  return parts.filter(Boolean).join('\n');
+}
 
-    return normalize({
-      companySlug: slug,
-      // Lever's API doesn't return the company name at the board or job level,
-      // so the slug is the honest fallback. `categories.team` is the team within
-      // the company ("Payments Platform"), not the company itself.
-      company: titleCaseSlug(slug),
-      title: job.text || '',
-      department: job.categories?.department || job.categories?.team || '',
-      location: job.categories?.location || '',
-      description: stripHtml(job.descriptionPlain || job.description || ''),
-      url: job.hostedUrl || '',
-      postedAt: job.createdAt ? new Date(job.createdAt).toISOString() : null,
-      salary,
-      metadata: {
-        leverId: job.id,
-        team: job.categories?.team || '',
-        commitment: job.categories?.commitment || '', // Full-time, Part-time, etc.
-        workplaceType: job.workplaceType || '',
-      },
-    }, 'lever');
-  });
+// `lists[].text` is plain text ("Skills & Experience"). Escaped, normalize()
+// decodes it back; raw, a stray `<` would be stripped as a tag.
+function escapeHtml(s) {
+  return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
+/**
+ * Lever publishes `salaryRange: {min, max, currency, interval}` on boards
+ * that state pay. Boards that don't sometimes put the range in the title.
+ */
+function parseLeverSalary(range, title) {
+  const min = range?.min || null;
+  const max = range?.max || null;
+  if (min || max) {
+    return {
+      min,
+      max,
+      currency: range.currency || 'USD',
+      period: PERIODS[range.interval] || null,
+      source: 'ats',
+    };
+  }
+  return extractSalaryFromText(title || '');
 }
 
 function titleCaseSlug(slug) {
@@ -55,24 +111,10 @@ function titleCaseSlug(slug) {
   return slug.charAt(0).toUpperCase() + slug.slice(1);
 }
 
-function parseLeverSalary(commitment, title) {
-  // Lever doesn't have a salary field, but sometimes it's in the title
-  const match = (title || '').match(/\$[\d,]+\s*[-–]\s*\$[\d,]+/);
-  if (!match) return null;
-  const nums = match[0].match(/[\d,]+/g);
-  if (!nums || nums.length < 2) return null;
-  return {
-    min: parseInt(nums[0].replace(/,/g, '')),
-    max: parseInt(nums[1].replace(/,/g, '')),
-    currency: 'USD',
-  };
-}
-
+/**
+ * Check if a company has a Lever board. See probeResult for the outcomes.
+ */
 export async function hasLever(slug) {
-  try {
-    const resp = await fetch(`${BASE_URL}/${slug}?mode=json`, { method: 'HEAD' });
-    return resp.ok;
-  } catch {
-    return false;
-  }
+  const resp = await atsFetch(`${BASE_URL}/${slug}?mode=json`, { method: 'HEAD' });
+  return probeResult(resp, `Lever probe for ${slug}`);
 }