crawlforge-extractors 1.5.1 → 1.5.3

package/README.md

@@ -40,6 +40,8 @@ const template = registry.get('shopify-product');
 const url = 'https://shop.example.com/products/some-handle';
 const fetchUrl = template.resolveUrl ? template.resolveUrl(url) : url;
 
+// Apply your SSRF policy to fetchUrl (the RETURNED url) and to any redirect the
+// fetch follows — not just to `url`. resolveUrl/listUrl may rewrite the target.
 const body = await (await fetch(fetchUrl)).text();
 const result = await registry.run('shopify-product', body, url, fetchUrl);
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.5.1",
+  "version": "1.5.3",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/body.js

@@ -70,9 +70,18 @@ export async function readBody(response, options = {}) {
   }
 
   // Only the byte-count guard needs a stream. Responses that are already
-  // buffered (and test doubles) are read as-is so callers still get their text.
+  // buffered (and test doubles) have no reader to meter, but a server can omit
+  // or lie about Content-Length, so enforce the cap on the read result too.
   if (!response.body || typeof response.body.getReader !== 'function') {
-    return response.text();
+    const text = await response.text();
+    const size = Buffer.byteLength(text, 'utf8');
+    if (size > maxBytes) {
+      throw new BodyTooLargeError(
+        `Response body too large: ${size} bytes exceeds limit of ${maxBytes} bytes`,
+        { limit: maxBytes, size }
+      );
+    }
+    return text;
   }
 
   const reader = response.body.getReader();

package/src/connectors/ats.js

@@ -20,6 +20,7 @@
  */
 
 import { load } from 'cheerio';
+import { safeHref } from '../urls.js';
 
 // ── The common job shape ─────────────────────────────────────────────────────
 
@@ -269,7 +270,7 @@ export const ATS_TEMPLATES = [
       const items = payload.jobs.map(j => job({
         id: id(j.id),
         title: str(j.title),
-        url: str(j.absolute_url),
+        url: safeHref(str(j.absolute_url)),
         location: str(j.location?.name),
         // departments and offices ship only with content=true, so a summary
         // record reports null here rather than a department guessed from the
@@ -359,7 +360,7 @@ export const ATS_TEMPLATES = [
           id: id(p.id),
           // Lever calls the job title "text".
           title: str(p.text),
-          url: str(p.hostedUrl),
+          url: safeHref(str(p.hostedUrl)),
           location: str(categories.location),
           department: str(categories.department),
           team: str(categories.team),
@@ -377,7 +378,7 @@ export const ATS_TEMPLATES = [
             // cities lists all three here.
             all_locations: (categories.allLocations || []).map(str).filter(Boolean),
             salary_range: p.salaryRange || null,
-            apply_url: str(p.applyUrl)
+            apply_url: safeHref(str(p.applyUrl))
           }
         });
       });
@@ -432,7 +433,7 @@ export const ATS_TEMPLATES = [
       const items = payload.jobs.map(j => job({
         id: id(j.id),
         title: str(j.title),
-        url: str(j.jobUrl),
+        url: safeHref(str(j.jobUrl)),
         location: str(j.location),
         department: str(j.department),
         team: str(j.team),
@@ -452,7 +453,7 @@ export const ATS_TEMPLATES = [
           // Names only. The full entries carry a postal address per country,
           // which is an office directory, not part of a job listing.
           secondary_locations: (j.secondaryLocations || []).map(l => str(l.location)).filter(Boolean),
-          apply_url: str(j.applyUrl)
+          apply_url: safeHref(str(j.applyUrl))
         }
       }));
 
@@ -523,7 +524,7 @@ export const ATS_TEMPLATES = [
         // separate numeric id in this payload.
         id: id(j.shortcode),
         title: str(j.title),
-        url: str(j.url),
+        url: safeHref(str(j.url)),
         location: joinLocation(j.city, j.state, j.country),
         department: str(j.department),
         // Workable has no team level.
@@ -540,7 +541,7 @@ export const ATS_TEMPLATES = [
           shortcode: str(j.shortcode),
           code: str(j.code),
           function: str(j.function),
-          apply_url: str(j.application_url)
+          apply_url: safeHref(str(j.application_url))
         }
       }));
 
@@ -602,7 +603,7 @@ export const ATS_TEMPLATES = [
       const items = payload.offers.map(o => job({
         id: id(o.id),
         title: str(o.title),
-        url: str(o.careers_url),
+        url: safeHref(str(o.careers_url)),
         location: str(o.location),
         department: str(o.department),
         team: null,
@@ -623,7 +624,7 @@ export const ATS_TEMPLATES = [
           slug: str(o.slug),
           salary: o.salary || null,
           tags: (o.tags || []).map(str).filter(Boolean),
-          apply_url: str(o.careers_apply_url)
+          apply_url: safeHref(str(o.careers_apply_url))
         }
       }));
 
@@ -715,7 +716,7 @@ export const ATS_TEMPLATES = [
           // is the one their support pages call the "Job ID".
           id: id(field('guid')),
           title: field('title'),
-          url: field('link'),
+          url: safeHref(field('link')),
           location: locations.length ? locations.join(', ') : null,
           department: field('tt\\:department'),
           team: null,

package/src/embeddedState.js

@@ -147,8 +147,14 @@ function parseFlightRows(stream) {
     const textRow = payload.match(/^T([0-9a-f]+),/i);
     if (textRow) {
       const blobStart = payloadStart + textRow[0].length;
-      const text = Buffer.from(stream.slice(blobStart), 'utf8')
-        .subarray(0, parseInt(textRow[1], 16))
+      const byteLen = parseInt(textRow[1], 16);
+      // Decode only up to byteLen bytes. Slicing the whole remaining stream on
+      // every text row makes this O(N) per row -> O(N^2) for an
+      // attacker-controlled stream of many small text rows. byteLen bytes span
+      // at most byteLen characters, so bounding the slice to that many chars
+      // keeps the work linear and yields the identical decoded blob.
+      const text = Buffer.from(stream.slice(blobStart, blobStart + byteLen), 'utf8')
+        .subarray(0, byteLen)
         .toString('utf8');
       rows[id] = text;
       cursor = blobStart + text.length;

package/src/jsonPath.js

@@ -66,7 +66,7 @@ export function selectJsonPath(root, path) {
   for (const segment of segments) {
     const container = current !== null && typeof current === 'object';
     const key = Array.isArray(current) ? Number(segment) : segment;
-    if (!container || !(key in current)) {
+    if (!container || !Object.hasOwn(current, key)) {
       const at = walked.length === 0 ? 'the result' : `"${walked.join('.')}"`;
       throw new Error(
         `Path "${path}" not found: ${at} has no "${segment}" (${describeOptions(current)})`

package/src/templates.js

@@ -14,7 +14,9 @@
  *
  * Templates do NOT make network calls. The caller fetches the page and passes
  * the body in; that keeps SSRF policy, timeouts and billing with the surface
- * that owns them.
+ * that owns them. resolveUrl/listUrl rewrite or build the target, so the caller
+ * must apply its SSRF policy to the URL they RETURN (and to every redirect the
+ * fetch follows), not just to the URL the caller started with.
  *
  * Two optional hooks let a template read a machine-readable endpoint instead of
  * scraping the rendered page, without taking the fetch into its own hands:
@@ -45,6 +47,7 @@ import { load } from 'cheerio';
 // have made one file the whole package.
 import { ATS_TEMPLATES } from './connectors/ats.js';
 import { GOV_TEMPLATES } from './connectors/gov.js';
+import { safeHref } from './urls.js';
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
@@ -608,7 +611,7 @@ export const TEMPLATES = [
         last_updated: attr($, 'relative-time', 'datetime'),
         // When the sidebar payload is present it is authoritative: an empty
         // website means "no homepage", not "go scrape some external link".
-        homepage: about ? about.website || null : attr($, 'a[href][rel="noopener noreferrer"]', 'href'),
+        homepage: safeHref(about ? about.website : attr($, 'a[href][rel="noopener noreferrer"]', 'href')),
         open_issues: githubCounter($, '#issues-repo-tab-count') || text($, '.Counter[aria-label*="issue"]')
       };
     }
@@ -623,7 +626,7 @@ export const TEMPLATES = [
       return {
         title: attr($, 'meta[name="title"]', 'content') || attr($, 'meta[property="og:title"]', 'content'),
         channel: attr($, 'link[itemprop="name"]', 'content') || text($, '#channel-name'),
-        channel_url: attr($, 'span[itemprop="author"] link[itemprop="url"]', 'href'),
+        channel_url: safeHref(attr($, 'span[itemprop="author"] link[itemprop="url"]', 'href')),
         views: youtubeInteractionCount($, 'WatchAction'),
         likes: youtubeInteractionCount($, 'LikeAction'),
         published: attr($, 'meta[itemprop="uploadDate"]', 'content') || attr($, 'meta[itemprop="datePublished"]', 'content'),
@@ -687,7 +690,7 @@ export const TEMPLATES = [
         body: post.selftext || null,
         // A link post carries its external URL here; a self post carries its
         // own permalink, which `url` already reports.
-        link_url: post.is_self ? null : (post.url || null),
+        link_url: post.is_self ? null : safeHref(post.url),
         url: post.permalink ? `https://www.reddit.com${post.permalink}` : null,
         flair: post.link_flair_text ?? null,
         over_18: Boolean(post.over_18),
@@ -713,7 +716,7 @@ export const TEMPLATES = [
         stories.push({
           id: $row.attr('id'),
           title: $titleLink.text().trim(),
-          url: $titleLink.attr('href'),
+          url: safeHref($titleLink.attr('href')),
           site: $row.find('.sitebit a').text().trim() || null,
           score: $score.text().replace(' points', '').trim() || null,
           author: $subtext.find('.hnuser').text().trim() || null,
@@ -738,10 +741,10 @@ export const TEMPLATES = [
         name: attr($, 'meta[property="og:title"]', 'content'),
         tagline: attr($, 'meta[property="og:description"]', 'content'),
         image: attr($, 'meta[property="og:image"]', 'content'),
-        url: attr($, 'meta[property="og:url"]', 'content'),
+        url: safeHref(attr($, 'meta[property="og:url"]', 'content')),
         votes: text($, '[data-test="vote-button"] span') || text($, 'button[data-vote-button]'),
         topics: list($, 'a[href*="/topics/"]'),
-        website: attr($, 'a[data-test="product-link"]', 'href') || attr($, 'a[href][rel="noopener"][target="_blank"]', 'href')
+        website: safeHref(attr($, 'a[data-test="product-link"]', 'href') || attr($, 'a[href][rel="noopener"][target="_blank"]', 'href'))
       };
     }
   },
@@ -869,8 +872,8 @@ export const TEMPLATES = [
         version: latest,
         description: release.description || doc.description || null,
         license: npmLicense(release.license ?? doc.license),
-        homepage: release.homepage || doc.homepage || null,
-        repository: npmRepositoryUrl(release.repository || doc.repository),
+        homepage: safeHref(release.homepage || doc.homepage),
+        repository: safeHref(npmRepositoryUrl(release.repository || doc.repository)),
         bugs: npmBugsUrl(release.bugs || doc.bugs),
         keywords: release.keywords || doc.keywords || [],
         maintainers: (doc.maintainers || [])

package/src/urls.js

@@ -0,0 +1,41 @@
+/**
+ * URL scheme filtering for values that come from scraped page content or a
+ * third-party payload — both untrusted.
+ *
+ * Such a URL flows into two places that will act on it: a React `href` (React
+ * does not sanitise href attributes) and an LLM client that may follow links.
+ * A `javascript:` or `data:` value there becomes a live XSS vector or an
+ * actionable prompt-injection URL. safeHref keeps only what is safe to hand on.
+ */
+
+// ASCII control characters (codepoints 0-31 and 127). A browser ignores these
+// inside a URL, so they can disguise a scheme, e.g. "java\tscript:alert(1)".
+// Built from char codes so the source stays plain ASCII (no literal controls).
+const CONTROL_CHARS = (() => {
+  let chars = '';
+  for (let c = 0; c <= 31; c++) chars += String.fromCharCode(c);
+  chars += String.fromCharCode(127);
+  return new RegExp('[' + chars + ']', 'g');
+})();
+
+/**
+ * Return the URL unchanged when it is an http(s) absolute URL or a scheme-less
+ * value (a relative path or a protocol-relative `//host` URL — neither carries a
+ * scheme to abuse), and null for everything else, including `javascript:`,
+ * `data:`, `vbscript:`, `file:`, `mailto:` and `tel:`.
+ *
+ * Control characters are stripped before the scheme is read, so a value like
+ * `java\nscript:alert(1)` — which a browser would execute after ignoring the
+ * newline — cannot masquerade as scheme-less and slip through.
+ *
+ * @param {unknown} url
+ * @returns {string|null}
+ */
+export function safeHref(url) {
+  if (typeof url !== 'string') return null;
+  const cleaned = url.replace(CONTROL_CHARS, '').trim();
+  if (!cleaned) return null;
+  // A leading scheme is a letter followed by letters/digits/+/-/. and a colon.
+  if (!/^[a-z][a-z0-9+.-]*:/i.test(cleaned)) return cleaned; // relative / protocol-relative
+  return /^https?:\/\//i.test(cleaned) ? cleaned : null; // absolute: http(s) only
+}