crawlforge-extractors 1.5.0 → 1.5.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.5.0",
+  "version": "1.5.2",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/connectors/ats.js

@@ -20,6 +20,7 @@
  */
 
 import { load } from 'cheerio';
+import { safeHref } from '../urls.js';
 
 // ── The common job shape ─────────────────────────────────────────────────────
 
@@ -269,7 +270,7 @@ export const ATS_TEMPLATES = [
       const items = payload.jobs.map(j => job({
         id: id(j.id),
         title: str(j.title),
-        url: str(j.absolute_url),
+        url: safeHref(str(j.absolute_url)),
         location: str(j.location?.name),
         // departments and offices ship only with content=true, so a summary
         // record reports null here rather than a department guessed from the
@@ -359,7 +360,7 @@ export const ATS_TEMPLATES = [
           id: id(p.id),
           // Lever calls the job title "text".
           title: str(p.text),
-          url: str(p.hostedUrl),
+          url: safeHref(str(p.hostedUrl)),
           location: str(categories.location),
           department: str(categories.department),
           team: str(categories.team),
@@ -377,7 +378,7 @@ export const ATS_TEMPLATES = [
             // cities lists all three here.
             all_locations: (categories.allLocations || []).map(str).filter(Boolean),
             salary_range: p.salaryRange || null,
-            apply_url: str(p.applyUrl)
+            apply_url: safeHref(str(p.applyUrl))
           }
         });
       });
@@ -432,7 +433,7 @@ export const ATS_TEMPLATES = [
       const items = payload.jobs.map(j => job({
         id: id(j.id),
         title: str(j.title),
-        url: str(j.jobUrl),
+        url: safeHref(str(j.jobUrl)),
         location: str(j.location),
         department: str(j.department),
         team: str(j.team),
@@ -452,7 +453,7 @@ export const ATS_TEMPLATES = [
           // Names only. The full entries carry a postal address per country,
           // which is an office directory, not part of a job listing.
           secondary_locations: (j.secondaryLocations || []).map(l => str(l.location)).filter(Boolean),
-          apply_url: str(j.applyUrl)
+          apply_url: safeHref(str(j.applyUrl))
         }
       }));
 
@@ -523,7 +524,7 @@ export const ATS_TEMPLATES = [
         // separate numeric id in this payload.
         id: id(j.shortcode),
         title: str(j.title),
-        url: str(j.url),
+        url: safeHref(str(j.url)),
         location: joinLocation(j.city, j.state, j.country),
         department: str(j.department),
         // Workable has no team level.
@@ -540,7 +541,7 @@ export const ATS_TEMPLATES = [
           shortcode: str(j.shortcode),
           code: str(j.code),
           function: str(j.function),
-          apply_url: str(j.application_url)
+          apply_url: safeHref(str(j.application_url))
         }
       }));
 
@@ -602,7 +603,7 @@ export const ATS_TEMPLATES = [
       const items = payload.offers.map(o => job({
         id: id(o.id),
         title: str(o.title),
-        url: str(o.careers_url),
+        url: safeHref(str(o.careers_url)),
         location: str(o.location),
         department: str(o.department),
         team: null,
@@ -623,7 +624,7 @@ export const ATS_TEMPLATES = [
           slug: str(o.slug),
           salary: o.salary || null,
           tags: (o.tags || []).map(str).filter(Boolean),
-          apply_url: str(o.careers_apply_url)
+          apply_url: safeHref(str(o.careers_apply_url))
         }
       }));
 
@@ -652,7 +653,10 @@ export const ATS_TEMPLATES = [
       'Read a company\'s published Teamtailor jobs from the careers site\'s documented RSS feed ' +
       'rather than the rendered page: title, department, locations, remote status and plain-text ' +
       'description for every open role. The feed returns the first 100 jobs unless per_page says otherwise.',
-    targetPattern: /teamtailor\.com\/jobs(\.rss)?(\?|$)/i,
+    // The careers-site root counts as a target: it is what a user pastes, and
+    // resolveUrl turns it into <host>/jobs.rss. A deeper path (/jobs/internal/,
+    // which robots disallows anyway) still does not match.
+    targetPattern: /teamtailor\.com\/?(?:jobs(\.rss)?\/?)?(\?|#|$)/i,
 
     /** `company` is the subdomain in https://<company>.teamtailor.com. */
     listUrl(params = {}) {
@@ -671,7 +675,13 @@ export const ATS_TEMPLATES = [
     resolveUrl(url) {
       const parsed = new URL(url);
       if (parsed.pathname.endsWith('.rss')) return url;
-      parsed.pathname = `${parsed.pathname.replace(/\/$/, '')}.rss`;
+      // The feed is the JOBS page with ".rss" appended, so a careers-site root
+      // has to gain the jobs path first. Stripping the trailing slash off "/"
+      // leaves "", which built "<host>/.rss" — a URL Teamtailor answers with
+      // 403 on every tenant tested. The bare root is what a user actually
+      // pastes, so it has to resolve to the documented "<host>/jobs.rss".
+      const path = parsed.pathname.replace(/\/+$/, '');
+      parsed.pathname = `${path || '/jobs'}.rss`;
       return parsed.toString();
     },
 
@@ -681,9 +691,14 @@ export const ATS_TEMPLATES = [
       const $ = load(body, { xmlMode: true });
       const items = $('channel > item');
 
-      if (!$('rss').length || !items.length) {
+      // Only a response that is not a feed is an error. A valid feed with no
+      // <item> is a company with nothing open right now — normative.teamtailor.com
+      // serves exactly that — and every sibling ATS connector reports an empty
+      // board as count: 0 rather than throwing. Conflating the two turned
+      // "nobody is hiring" into "the tool is broken".
+      if (!$('rss').length) {
         throw new Error(
-          `No Teamtailor job feed at ${url}: ${$('rss').length ? 'the feed has no items' : 'the response is not an RSS feed'}. ` +
+          `No Teamtailor job feed at ${url}: the response is not an RSS feed. ` +
           'The feed is the careers site jobs page with ".rss" appended, e.g. ' +
           'https://<company>.teamtailor.com/jobs.rss.'
         );
@@ -701,7 +716,7 @@ export const ATS_TEMPLATES = [
           // is the one their support pages call the "Job ID".
           id: id(field('guid')),
           title: field('title'),
-          url: field('link'),
+          url: safeHref(field('link')),
           location: locations.length ? locations.join(', ') : null,
           department: field('tt\\:department'),
           team: null,

package/src/embeddedState.js

@@ -147,8 +147,14 @@ function parseFlightRows(stream) {
     const textRow = payload.match(/^T([0-9a-f]+),/i);
     if (textRow) {
       const blobStart = payloadStart + textRow[0].length;
-      const text = Buffer.from(stream.slice(blobStart), 'utf8')
-        .subarray(0, parseInt(textRow[1], 16))
+      const byteLen = parseInt(textRow[1], 16);
+      // Decode only up to byteLen bytes. Slicing the whole remaining stream on
+      // every text row makes this O(N) per row -> O(N^2) for an
+      // attacker-controlled stream of many small text rows. byteLen bytes span
+      // at most byteLen characters, so bounding the slice to that many chars
+      // keeps the work linear and yields the identical decoded blob.
+      const text = Buffer.from(stream.slice(blobStart, blobStart + byteLen), 'utf8')
+        .subarray(0, byteLen)
         .toString('utf8');
       rows[id] = text;
       cursor = blobStart + text.length;

package/src/templates.js

@@ -45,6 +45,7 @@ import { load } from 'cheerio';
 // have made one file the whole package.
 import { ATS_TEMPLATES } from './connectors/ats.js';
 import { GOV_TEMPLATES } from './connectors/gov.js';
+import { safeHref } from './urls.js';
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
@@ -608,7 +609,7 @@ export const TEMPLATES = [
         last_updated: attr($, 'relative-time', 'datetime'),
         // When the sidebar payload is present it is authoritative: an empty
         // website means "no homepage", not "go scrape some external link".
-        homepage: about ? about.website || null : attr($, 'a[href][rel="noopener noreferrer"]', 'href'),
+        homepage: safeHref(about ? about.website : attr($, 'a[href][rel="noopener noreferrer"]', 'href')),
         open_issues: githubCounter($, '#issues-repo-tab-count') || text($, '.Counter[aria-label*="issue"]')
       };
     }
@@ -623,7 +624,7 @@ export const TEMPLATES = [
       return {
         title: attr($, 'meta[name="title"]', 'content') || attr($, 'meta[property="og:title"]', 'content'),
         channel: attr($, 'link[itemprop="name"]', 'content') || text($, '#channel-name'),
-        channel_url: attr($, 'span[itemprop="author"] link[itemprop="url"]', 'href'),
+        channel_url: safeHref(attr($, 'span[itemprop="author"] link[itemprop="url"]', 'href')),
         views: youtubeInteractionCount($, 'WatchAction'),
         likes: youtubeInteractionCount($, 'LikeAction'),
         published: attr($, 'meta[itemprop="uploadDate"]', 'content') || attr($, 'meta[itemprop="datePublished"]', 'content'),
@@ -687,7 +688,7 @@ export const TEMPLATES = [
         body: post.selftext || null,
         // A link post carries its external URL here; a self post carries its
         // own permalink, which `url` already reports.
-        link_url: post.is_self ? null : (post.url || null),
+        link_url: post.is_self ? null : safeHref(post.url),
         url: post.permalink ? `https://www.reddit.com${post.permalink}` : null,
         flair: post.link_flair_text ?? null,
         over_18: Boolean(post.over_18),
@@ -713,7 +714,7 @@ export const TEMPLATES = [
         stories.push({
           id: $row.attr('id'),
           title: $titleLink.text().trim(),
-          url: $titleLink.attr('href'),
+          url: safeHref($titleLink.attr('href')),
           site: $row.find('.sitebit a').text().trim() || null,
           score: $score.text().replace(' points', '').trim() || null,
           author: $subtext.find('.hnuser').text().trim() || null,
@@ -738,10 +739,10 @@ export const TEMPLATES = [
         name: attr($, 'meta[property="og:title"]', 'content'),
         tagline: attr($, 'meta[property="og:description"]', 'content'),
         image: attr($, 'meta[property="og:image"]', 'content'),
-        url: attr($, 'meta[property="og:url"]', 'content'),
+        url: safeHref(attr($, 'meta[property="og:url"]', 'content')),
         votes: text($, '[data-test="vote-button"] span') || text($, 'button[data-vote-button]'),
         topics: list($, 'a[href*="/topics/"]'),
-        website: attr($, 'a[data-test="product-link"]', 'href') || attr($, 'a[href][rel="noopener"][target="_blank"]', 'href')
+        website: safeHref(attr($, 'a[data-test="product-link"]', 'href') || attr($, 'a[href][rel="noopener"][target="_blank"]', 'href'))
       };
     }
   },
@@ -869,8 +870,8 @@ export const TEMPLATES = [
         version: latest,
         description: release.description || doc.description || null,
         license: npmLicense(release.license ?? doc.license),
-        homepage: release.homepage || doc.homepage || null,
-        repository: npmRepositoryUrl(release.repository || doc.repository),
+        homepage: safeHref(release.homepage || doc.homepage),
+        repository: safeHref(npmRepositoryUrl(release.repository || doc.repository)),
         bugs: npmBugsUrl(release.bugs || doc.bugs),
         keywords: release.keywords || doc.keywords || [],
         maintainers: (doc.maintainers || [])

package/src/urls.js

@@ -0,0 +1,41 @@
+/**
+ * URL scheme filtering for values that come from scraped page content or a
+ * third-party payload — both untrusted.
+ *
+ * Such a URL flows into two places that will act on it: a React `href` (React
+ * does not sanitise href attributes) and an LLM client that may follow links.
+ * A `javascript:` or `data:` value there becomes a live XSS vector or an
+ * actionable prompt-injection URL. safeHref keeps only what is safe to hand on.
+ */
+
+// ASCII control characters (codepoints 0-31 and 127). A browser ignores these
+// inside a URL, so they can disguise a scheme, e.g. "java\tscript:alert(1)".
+// Built from char codes so the source stays plain ASCII (no literal controls).
+const CONTROL_CHARS = (() => {
+  let chars = '';
+  for (let c = 0; c <= 31; c++) chars += String.fromCharCode(c);
+  chars += String.fromCharCode(127);
+  return new RegExp('[' + chars + ']', 'g');
+})();
+
+/**
+ * Return the URL unchanged when it is an http(s) absolute URL or a scheme-less
+ * value (a relative path or a protocol-relative `//host` URL — neither carries a
+ * scheme to abuse), and null for everything else, including `javascript:`,
+ * `data:`, `vbscript:`, `file:`, `mailto:` and `tel:`.
+ *
+ * Control characters are stripped before the scheme is read, so a value like
+ * `java\nscript:alert(1)` — which a browser would execute after ignoring the
+ * newline — cannot masquerade as scheme-less and slip through.
+ *
+ * @param {unknown} url
+ * @returns {string|null}
+ */
+export function safeHref(url) {
+  if (typeof url !== 'string') return null;
+  const cleaned = url.replace(CONTROL_CHARS, '').trim();
+  if (!cleaned) return null;
+  // A leading scheme is a letter followed by letters/digits/+/-/. and a colon.
+  if (!/^[a-z][a-z0-9+.-]*:/i.test(cleaned)) return cleaned; // relative / protocol-relative
+  return /^https?:\/\//i.test(cleaned) ? cleaned : null; // absolute: http(s) only
+}