jd-intel 0.8.3 → 0.10.0

package/src/adapters/recruitee.js

@@ -1,21 +1,32 @@
-import { normalize, stripHtml } from '../normalizer.js';
+import { normalize } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { atsFetch, probeResult } from '../http.js';
+import { orgHost } from '../boards.js';
 
 /**
  * Fetch jobs from a Recruitee career site.
  * Public API, no auth required.
  * Docs: https://docs.recruitee.com/reference/offers
  *
- * Single GET returns every offer with the full HTML description
- * inline — no N+1 (unlike SmartRecruiters), no XML (unlike
- * TeamTailor/Personio). The simplest adapter shape in the toolkit.
+ * Single GET returns every offer inline — no N+1 (unlike SmartRecruiters),
+ * no XML (unlike TeamTailor/Personio). The simplest adapter shape in the
+ * toolkit.
+ *
+ * Each offer carries two HTML fields, `description` and `requirements`.
+ * Which one holds the role depends on the tenant's template (and sometimes
+ * the posting): some keep the duties in `description` and the candidate
+ * profile in `requirements`, others put a company intro in `description`
+ * and everything else in `requirements`. Neither alone is the posting, so
+ * both are joined before normalize() strips them (issue #65).
  *
  * @param {string} slug - Recruitee company subdomain (e.g., 'vandebron')
+ * @param {object} [ctx] - { report }; report is called once with
+ *   { ats, org_name, org_url } when given
  * @returns {Promise<Array>} Normalized job objects
  */
-export async function fetchRecruitee(slug) {
+export async function fetchRecruitee(slug, ctx = {}) {
   const url = `https://${slug}.recruitee.com/api/offers/`;
-  const resp = await fetch(url);
+  const resp = await atsFetch(url);
 
   if (!resp.ok) {
     if (resp.status === 404) return []; // No Recruitee site for this slug
@@ -25,18 +36,23 @@ export async function fetchRecruitee(slug) {
   const data = await resp.json();
   const offers = data.offers || [];
 
+  // The response is { offers } only, so the identity lives on the rows:
+  // company_name, and careers_url, which sits on the company's own careers
+  // domain when the site has one and on {slug}.recruitee.com otherwise.
+  if (typeof ctx.report === 'function') {
+    ctx.report({
+      ats: 'recruitee',
+      org_name: offers.find(o => o.company_name)?.company_name || null,
+      org_url: orgHost(offers.find(o => o.careers_url)?.careers_url),
+    });
+  }
+
   return offers.map(offer => {
     const place = [offer.city, offer.country].filter(Boolean).join(', ');
     let location = place;
     if (offer.remote) location = place ? `Remote - ${place}` : 'Remote';
 
-    let postedAt = null;
-    if (offer.created_at) {
-      // Recruitee returns "2026-05-13 07:38:11 UTC"; coerce to ISO.
-      const iso = offer.created_at.replace(' UTC', 'Z').replace(' ', 'T');
-      const d = new Date(iso);
-      if (!Number.isNaN(d.getTime())) postedAt = d.toISOString();
-    }
+    const createdAt = toIso(offer.created_at);
 
     return normalize({
       companySlug: slug,
@@ -44,27 +60,75 @@ export async function fetchRecruitee(slug) {
       title: offer.title || '',
       department: offer.department || '',
       location,
-      description: stripHtml(offer.description || ''),
+      locations: (offer.locations || []).map(l => [l.city, l.country].filter(Boolean).join(', ')),
+      workplace: parseRecruiteeWorkplace(offer),
+      description: [offer.description, offer.requirements].filter(Boolean).join('\n'),
       url: offer.careers_url || offer.careers_apply_url || '',
-      postedAt,
-      salary: null, // No structured salary; normalizer parses from text
+      // created_at can predate publication by years on long-lived offers,
+      // so it is not a posting date. published_at is.
+      postedAt: toIso(offer.published_at) || createdAt,
+      salary: parseRecruiteeSalary(offer.salary),
       metadata: {
         recruiteeId: offer.guid || offer.id,
         employmentType: offer.employment_type_code || '',
         category: offer.category_code || '',
+        createdAt,
       },
     }, 'recruitee');
   });
 }
 
 /**
- * Check if a company has a Recruitee career site.
+ * Recruitee returns "2026-05-13 07:38:11 UTC"; coerce to ISO.
+ */
+function toIso(ts) {
+  if (!ts) return null;
+  const d = new Date(ts.replace(' UTC', 'Z').replace(' ', 'T'));
+  return Number.isNaN(d.getTime()) ? null : d.toISOString();
+}
+
+/**
+ * Recruitee sends three booleans, not one enum. Hybrid wins when remote is
+ * also set, and on_site alone is onsite. All false is no signal.
+ */
+function parseRecruiteeWorkplace(offer) {
+  if (offer.hybrid) return 'hybrid';
+  if (offer.remote) return 'remote';
+  if (offer.on_site) return 'onsite';
+  return null;
+}
+
+const PERIODS = new Set(['year', 'month', 'hour']);
+
+/**
+ * Recruitee sends `salary` as `{min, max, period, currency}` with string
+ * amounts. Offers without pay still carry the object, either all-null or
+ * as a "0"/"0" placeholder, so anything without a positive side returns
+ * null and normalize() falls back to the posting text.
+ */
+function parseRecruiteeSalary(salary) {
+  if (!salary) return null;
+  const min = toAmount(salary.min);
+  const max = toAmount(salary.max);
+  if (min === null && max === null) return null;
+  return {
+    min,
+    max,
+    currency: (salary.currency || '').toUpperCase(),
+    period: PERIODS.has(salary.period) ? salary.period : null,
+    source: 'ats',
+  };
+}
+
+function toAmount(value) {
+  const n = parseFloat(value);
+  return Number.isFinite(n) && n > 0 ? n : null;
+}
+
+/**
+ * Check if a company has a Recruitee career site. See probeResult for the outcomes.
  */
 export async function hasRecruitee(slug) {
-  try {
-    const resp = await fetch(`https://${slug}.recruitee.com/api/offers/`);
-    return resp.ok;
-  } catch {
-    return false;
-  }
+  const resp = await atsFetch(`https://${slug}.recruitee.com/api/offers/`);
+  return probeResult(resp, `Recruitee probe for ${slug}`);
 }

package/src/adapters/smartrecruiters.js

@@ -1,33 +1,45 @@
-import { normalize, stripHtml } from '../normalizer.js';
+import { normalize } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { atsFetch, probeResult } from '../http.js';
+import { makeLocationMatcher } from '../filters.js';
 
 const BASE_URL = 'https://api.smartrecruiters.com/v1/companies';
 const PAGE_SIZE = 100;
+const MAX_DETAIL_FETCHES = 100;
 
 /**
- * Fetch all postings from a SmartRecruiters company.
+ * Fetch postings from a SmartRecruiters company.
  * Public API, no auth required.
  * Docs: https://developers.smartrecruiters.com/reference/postingsget-1
  *
  * Two-step flow (unavoidable N+1):
- *   - The postings LIST endpoint omits the job description entirely.
+ *   - The postings LIST endpoint omits the job description entirely,
+ *     and the structured `compensation` block with it.
  *   - jd-intel's contract is "full JD text", so we must fetch each
  *     posting's DETAIL endpoint to get jobAd.sections.
- * Large enterprise tenants with hundreds of openings will therefore be
- * slow against SmartRecruiters specifically. This is the API's shape,
- * not a bug here.
+ *
+ * The list does carry name, location and releasedDate, so the same
+ * pre-filter and detail budget Workday applies run here: list-evaluable
+ * filters narrow the candidates, then at most MAX_DETAIL_FETCHES of them
+ * are hydrated (see the budget note below). Without a filterContext the
+ * cap still holds, so a direct call on a 400-posting tenant reads 100.
  *
  * @param {string} slug - SmartRecruiters company identifier (e.g., 'Visa')
+ * @param {object} [ctx] - { filterContext, report }; report is called once
+ *   with { ats, listed, prefiltered, hydrated, capped, org_name, org_url }
+ *   when given
  * @returns {Promise<Array>} Normalized job objects
  */
-export async function fetchSmartrecruiters(slug) {
+export async function fetchSmartrecruiters(slug, ctx = {}) {
+  const fc = ctx.filterContext || {};
+
   // 1. Page through the postings list.
   const postings = [];
   let offset = 0;
 
   while (true) {
     const listUrl = `${BASE_URL}/${slug}/postings?limit=${PAGE_SIZE}&offset=${offset}`;
-    const resp = await fetch(listUrl);
+    const resp = await atsFetch(listUrl);
 
     if (!resp.ok) {
       if (resp.status === 404) return []; // Company not found
@@ -42,20 +54,89 @@ export async function fetchSmartrecruiters(slug) {
     if (content.length === 0 || offset >= (data.totalFound || 0)) break;
   }
 
-  // 2. Fetch detail per posting for the description.
-  const jobs = await Promise.all(postings.map(async (p) => {
+  // 2. Filter-aware candidate selection BEFORE the N+1 detail cost.
+  //    The list row carries name, location and releasedDate, and the
+  //    library re-applies every filter after this returns, so a keep here
+  //    is never final. The detail adds no location (unlike Workday's
+  //    additionalLocations), so a row with none follows the library's
+  //    rule now: out under includes, kept under excludes.
+  let candidates = postings;
+
+  if (fc.titleFilter) {
+    const re = new RegExp(fc.titleFilter, 'i');
+    candidates = candidates.filter(p => re.test(p.name || ''));
+  }
+  if (Array.isArray(fc.locationIncludes) && fc.locationIncludes.length > 0) {
+    const matchers = fc.locationIncludes.map(makeLocationMatcher);
+    candidates = candidates.filter(p => {
+      const loc = listLocation(p).location.toLowerCase();
+      return matchers.some(m => m(loc));
+    });
+  }
+  if (Array.isArray(fc.locationExcludes) && fc.locationExcludes.length > 0) {
+    const matchers = fc.locationExcludes.map(makeLocationMatcher);
+    candidates = candidates.filter(p => {
+      const loc = listLocation(p).location.toLowerCase();
+      return !loc || !matchers.some(m => m(loc));
+    });
+  }
+  if (typeof fc.postedWithinDays === 'number') {
+    // postedAt comes from releasedDate alone, so the library's rule can
+    // run here in full: a missing or unparseable date is out either way.
+    const cutoff = Date.now() - fc.postedWithinDays * 86400000;
+    candidates = candidates.filter(p => {
+      const released = new Date(p.releasedDate || '').getTime();
+      return Number.isFinite(released) && released >= cutoff;
+    });
+  }
+
+  // 3. Bound the detail-fetch set, Workday's reasoning verbatim: a
+  //    description `filter` is applied by the library AFTER this returns,
+  //    so that case keeps the full backstop instead of truncating to
+  //    `limit` (which could hydrate jobs that all fail the regex while
+  //    better matches go unscanned). The library pages with `offset`
+  //    after this returns, so the budget covers the page plus what
+  //    precedes it. Candidates keep list order.
+  const limit = typeof fc.limit === 'number' && fc.limit > 0 ? fc.limit : 100;
+  const skip = typeof fc.offset === 'number' && fc.offset > 0 ? fc.offset : 0;
+  const cap = fc.filter ? MAX_DETAIL_FETCHES : Math.min(skip + limit, MAX_DETAIL_FETCHES);
+  const hydrate = candidates.slice(0, cap);
+
+  // Every list row carries company { identifier, name }. Neither the list
+  // nor the detail has a company website, and postingUrl is always on
+  // jobs.smartrecruiters.com, so org_url stays null (issue #58).
+  if (typeof ctx.report === 'function') {
+    ctx.report({
+      ats: 'smartrecruiters',
+      listed: postings.length,
+      prefiltered: candidates.length,
+      hydrated: hydrate.length,
+      capped: hydrate.length < candidates.length,
+      org_name: postings.find(p => p.company?.name)?.company.name || null,
+      org_url: null,
+    });
+  }
+
+  // 4. Fetch detail per candidate for the description. atsFetch's per-host
+  //    queue keeps this fan-out to 4 requests at a time (a 412-posting
+  //    tenant measured 53s unbounded), so the cap also holds the detail
+  //    step to roughly 13s.
+  const jobs = await Promise.all(hydrate.map(async (p) => {
     let sections = {};
     let postingUrl = '';
+    let salary = null;
 
     try {
-      const detailResp = await fetch(`${BASE_URL}/${slug}/postings/${p.id}`);
+      const detailResp = await atsFetch(`${BASE_URL}/${slug}/postings/${p.id}`);
       if (detailResp.ok) {
         const detail = await detailResp.json();
         sections = detail.jobAd?.sections || {};
         postingUrl = detail.postingUrl || detail.applyUrl || '';
+        salary = parseCompensation(detail.compensation);
       }
     } catch {
-      // Detail fetch failed: fall back to list-only fields (no description).
+      // Detail fetch failed, retries included: fall back to list-only
+      // fields (no description). Reporting this is #85.
     }
 
     const description = [
@@ -64,12 +145,7 @@ export async function fetchSmartrecruiters(slug) {
       sections.additionalInformation?.text,
     ].filter(Boolean).join('\n\n');
 
-    const loc = p.location || {};
-    const place = loc.fullLocation
-      || [loc.city, loc.region, loc.country].filter(Boolean).join(', ');
-    let location = place;
-    if (loc.remote) location = `Remote - ${place}`.replace(/ - $/, ' ');
-    else if (loc.hybrid) location = `Hybrid - ${place}`.replace(/ - $/, ' ');
+    const { location, workplace } = listLocation(p);
 
     return normalize({
       companySlug: slug,
@@ -77,10 +153,11 @@ export async function fetchSmartrecruiters(slug) {
       title: p.name || '',
       department: p.department?.label || p.function?.label || '',
       location,
-      description: stripHtml(description),
+      workplace,
+      description,
       url: postingUrl,
       postedAt: p.releasedDate || null,
-      salary: null, // SmartRecruiters has no structured salary; normalizer parses text
+      salary, // null when the detail has no compensation; normalize() then parses text
       metadata: {
         smartRecruitersId: p.id,
         refNumber: p.refNumber || '',
@@ -95,20 +172,55 @@ export async function fetchSmartrecruiters(slug) {
 }
 
 /**
- * Check if a company exists on SmartRecruiters.
- * (HEAD isn't reliably supported on the postings endpoint, so use a
- * minimal GET.)
+ * The location string and workplace type a list row yields. Built once
+ * here so the pre-filter matches exactly what the normalized job carries,
+ * "Remote - " and "Hybrid - " prefixes included.
+ */
+function listLocation(p) {
+  const loc = p.location || {};
+  const place = loc.fullLocation
+    || [loc.city, loc.region, loc.country].filter(Boolean).join(', ');
+  if (loc.remote) return { location: `Remote - ${place}`.replace(/ - $/, ' '), workplace: 'remote' };
+  if (loc.hybrid) return { location: `Hybrid - ${place}`.replace(/ - $/, ' '), workplace: 'hybrid' };
+  return { location: place, workplace: null };
+}
+
+const PERIODS = { YEARLY: 'year', MONTHLY: 'month', HOURLY: 'hour' };
+
+/**
+ * Map the detail response's `compensation` to the shared salary shape.
+ *
+ * SmartRecruiters publishes `{min?, max?, currency, period}`, and both
+ * one-sided cases occur (a "max only" cap, a "from" floor), so each bound
+ * is passed through as null when absent rather than dropping the whole
+ * range. The period is kept as published: a MONTHLY figure is not
+ * annualized because tenants occasionally mislabel it (issue #70).
+ */
+function parseCompensation(comp) {
+  if (!comp || !comp.currency) return null;
+  const min = Number.isFinite(comp.min) ? comp.min : null;
+  const max = Number.isFinite(comp.max) ? comp.max : null;
+  if (min === null && max === null) return null;
+  return {
+    min,
+    max,
+    currency: comp.currency,
+    period: PERIODS[comp.period] ?? null,
+    source: 'ats',
+  };
+}
+
+/**
+ * Check if a company exists on SmartRecruiters. See probeResult for the
+ * outcomes. (HEAD isn't reliably supported on the postings endpoint, so
+ * use a minimal GET.)
  */
 export async function hasSmartrecruiters(slug) {
-  try {
-    const resp = await fetch(`${BASE_URL}/${slug}/postings?limit=1`);
-    if (!resp.ok) return false;
-    // SmartRecruiters returns 200 with an empty page (not 404) for unknown
-    // companies, so resp.ok alone false-positives on any slug. Confirm at
-    // least one real posting exists before claiming a match.
-    const data = await resp.json();
-    return (data.totalFound || 0) > 0 || (data.content || []).length > 0;
-  } catch {
-    return false;
-  }
+  const resp = await atsFetch(`${BASE_URL}/${slug}/postings?limit=1`);
+  if (!probeResult(resp, `SmartRecruiters probe for ${slug}`)) return false;
+  // SmartRecruiters returns 200 with an empty page (not 404) for unknown
+  // companies, so resp.ok alone false-positives on any slug. Confirm at
+  // least one real posting exists before claiming a match.
+  const data = await resp.json();
+  return (data.totalFound || 0) > 0 || (data.content || []).length > 0;
 }

package/src/adapters/teamtailor.js

@@ -1,5 +1,7 @@
-import { normalize, stripHtml } from '../normalizer.js';
+import { normalize, decodeEntities } from '../normalizer.js';
 import { atsErrorFromStatus } from '../errors.js';
+import { atsFetch } from '../http.js';
+import { orgHost } from '../boards.js';
 
 /**
  * Fetch jobs from a TeamTailor career site via its public RSS feed.
@@ -16,29 +18,40 @@ import { atsErrorFromStatus } from '../errors.js';
  * to a custom domain (e.g. jobs.tibber.com).
  *
  * RSS quirk: descriptions are HTML-entity-encoded inside the XML
- * (`<p>...`). We decode that outer layer to real HTML, then
- * hand it to stripHtml() which strips tags and resolves the inner
- * entities. Decode order matters — `&` resolves LAST so that
- * double-encoded sequences (`&`) collapse correctly.
+ * (`<p>...`). We decode that outer layer to real HTML with the
+ * shared decodeEntities() and hand the HTML to normalize(), which
+ * strips tags and resolves the inner entities. Decode order matters —
+ * `&` resolves LAST so double-encoded sequences (`&`)
+ * collapse by one layer per pass.
  *
  * @param {string} slug - TeamTailor career-site slug (e.g., 'tibber')
+ * @param {object} [ctx] - { report }; report is called once with
+ *   { ats, org_name, org_url } when given
  * @returns {Promise<Array>} Normalized job objects
  */
 // Most sites are {slug}.teamtailor.com, but some sit on a regional
-// segment, e.g. crunchbase.na.teamtailor.com. '' is the base host.
-const TT_REGIONS = ['', 'na', 'eu'];
+// segment, e.g. crunchbase.na.teamtailor.com. '' is the base host. There
+// is no reachable eu segment: {slug}.eu.teamtailor.com fails TLS for every
+// slug, known or not, because the wildcard certificate covers one label
+// only (live check 2026-09-27). Probing it was a guaranteed failure that
+// the has() contract would now report as an outage.
+const TT_REGIONS = ['', 'na'];
+
+// Feeds send `none`, `hybrid`, `fully` or `onsite`. `none` is no signal.
+const REMOTE_STATUS = { hybrid: 'hybrid', fully: 'remote', onsite: 'onsite' };
 
 /**
  * Resolve which TeamTailor host actually serves this slug's feed.
- * Returns the first 200 Response, throws on a non-404 error, or
- * returns null if no region has a feed.
+ * Returns the first 200 Response, throws on a non-404 error (atsFetch
+ * throws the 429, 5xx and network cases itself), or returns null if no
+ * region has a feed.
  */
 async function resolveFeed(slug, method = 'GET') {
   for (const region of TT_REGIONS) {
     const host = region
       ? `${slug}.${region}.teamtailor.com`
       : `${slug}.teamtailor.com`;
-    const resp = await fetch(`https://${host}/jobs.rss`, {
+    const resp = await atsFetch(`https://${host}/jobs.rss`, {
       method,
       redirect: 'follow',
     });
@@ -51,21 +64,36 @@ async function resolveFeed(slug, method = 'GET') {
   return null;
 }
 
-export async function fetchTeamtailor(slug) {
+export async function fetchTeamtailor(slug, ctx = {}) {
   const resp = await resolveFeed(slug, 'GET');
   if (!resp) return []; // No TeamTailor site in any known region
 
   const xml = await resp.text();
 
-  const company = (
-    xml.match(/<channel>[\s\S]*?<title>([\s\S]*?)<\/title>/)?.[1] || slug
-  ).trim();
+  const channelTitle = (xml.match(/<channel>[\s\S]*?<title>([\s\S]*?)<\/title>/)?.[1] || '').trim();
+  const company = channelTitle || slug;
 
   const items = [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map(m => m[1]);
 
+  // The channel title is the company as the site names itself. The channel
+  // <link> always sits on {slug}.teamtailor.com, but item links follow the
+  // site's custom domain when it has one (jobs.tibber.com on a feed served
+  // from tibber.teamtailor.com), so the first item's link is the host that
+  // can say something; the channel link is the fallback for an empty feed.
+  if (typeof ctx.report === 'function') {
+    const link = items[0]?.match(/<link>([\s\S]*?)<\/link>/)?.[1]
+      || xml.match(/<channel>[\s\S]*?<link>([\s\S]*?)<\/link>/)?.[1]
+      || '';
+    ctx.report({
+      ats: 'teamtailor',
+      org_name: decodeEntities(channelTitle) || null,
+      org_url: orgHost(link.trim()),
+    });
+  }
+
   return items.map(item => {
-    const pick = (tag) => {
-      const m = item.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}>`));
+    const pick = (tag, src = item) => {
+      const m = src.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}>`));
       return m ? m[1].trim() : '';
     };
 
@@ -78,6 +106,12 @@ export async function fetchTeamtailor(slug) {
     const country = decodeEntities(pick('tt:country'));
     const remoteStatus = decodeEntities(pick('remoteStatus'));
 
+    // One <tt:location> per office the posting is open in, read the same
+    // way as the primary above so the entries line up.
+    const locations = [...item.matchAll(/<tt:location>([\s\S]*?)<\/tt:location>/g)].map(m =>
+      [decodeEntities(pick('tt:city', m[1])), decodeEntities(pick('tt:country', m[1]))].filter(Boolean).join(', ')
+    );
+
     let location = [city, country].filter(Boolean).join(', ');
     if (/remote/i.test(remoteStatus)) {
       location = location ? `Remote - ${location}` : 'Remote';
@@ -95,7 +129,9 @@ export async function fetchTeamtailor(slug) {
       title,
       department,
       location,
-      description: stripHtml(decodeEntities(pick('description'))),
+      locations,
+      workplace: REMOTE_STATUS[remoteStatus.toLowerCase()] || null,
+      description: decodeEntities(pick('description')),
       url: link,
       postedAt,
       salary: null, // No structured salary; normalizer parses from text
@@ -108,28 +144,10 @@ export async function fetchTeamtailor(slug) {
 }
 
 /**
- * Decode the RSS entity/CDATA layer to real HTML.
- * `&amp;` is intentionally resolved LAST.
- */
-function decodeEntities(s) {
-  if (!s) return '';
-  return s
-    .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/&apos;/g, "'")
-    .replace(/&amp;/g, '&');
-}
-
-/**
- * Check if a company has a TeamTailor career site.
+ * Check if a company has a TeamTailor career site: true when a regional
+ * host serves the feed, false when every host answers 404, and the
+ * AtsError from resolveFeed for anything else.
  */
 export async function hasTeamtailor(slug) {
-  try {
-    return (await resolveFeed(slug, 'HEAD')) !== null;
-  } catch {
-    return false;
-  }
+  return (await resolveFeed(slug, 'HEAD')) !== null;
 }