crawlforge-extractors 1.5.2 → 1.6.0

package/README.md

@@ -40,6 +40,8 @@ const template = registry.get('shopify-product');
 const url = 'https://shop.example.com/products/some-handle';
 const fetchUrl = template.resolveUrl ? template.resolveUrl(url) : url;
 
+// Apply your SSRF policy to fetchUrl (the RETURNED url) and to any redirect the
+// fetch follows — not just to `url`. resolveUrl/listUrl may rewrite the target.
 const body = await (await fetch(fetchUrl)).text();
 const result = await registry.run('shopify-product', body, url, fetchUrl);
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.5.2",
+  "version": "1.6.0",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/body.js

@@ -70,9 +70,18 @@ export async function readBody(response, options = {}) {
   }
 
   // Only the byte-count guard needs a stream. Responses that are already
-  // buffered (and test doubles) are read as-is so callers still get their text.
+  // buffered (and test doubles) have no reader to meter, but a server can omit
+  // or lie about Content-Length, so enforce the cap on the read result too.
   if (!response.body || typeof response.body.getReader !== 'function') {
-    return response.text();
+    const text = await response.text();
+    const size = Buffer.byteLength(text, 'utf8');
+    if (size > maxBytes) {
+      throw new BodyTooLargeError(
+        `Response body too large: ${size} bytes exceeds limit of ${maxBytes} bytes`,
+        { limit: maxBytes, size }
+      );
+    }
+    return text;
   }
 
   const reader = response.body.getReader();

package/src/connectors/ats.js

@@ -398,8 +398,10 @@ export const ATS_TEMPLATES = [
     name: 'Ashby Job Board',
     description:
       'Read a company\'s whole Ashby job board from the Public Job Posting API rather than the ' +
-      'rendered careers page: every listed job with its department, team, employment type, ' +
-      'workplace type and plain-text description in one request.',
+      'rendered careers page: every listed job with its department, team, employment type and ' +
+      'workplace type in one request. Descriptions are opt-in — pass descriptions: true — ' +
+      'because they are most of the payload: OpenAI\'s 767-job board is 5.9 MB with them and ' +
+      'a fraction of that without.',
     targetPattern: /(jobs|api)\.ashbyhq\.com\//i,
 
     /** `company` is Ashby's jobs page name — the segment in https://jobs.ashbyhq.com/<name>. */
@@ -418,7 +420,7 @@ export const ATS_TEMPLATES = [
       return company ? ashbyUrl({ company }) : url;
     },
 
-    extractList(body, url) {
+    extractList(body, url, params = {}) {
       const payload = parseJson(body, () =>
         notJson('Ashby job board', url, 'the Ashby Public Job Posting API')
       );
@@ -446,7 +448,11 @@ export const ATS_TEMPLATES = [
         remote: isRemote(j.workplaceType),
         published_at: isoDate(j.publishedAt),
         updated_at: null,
-        description: str(j.descriptionPlain),
+        // The API has no query switch for this, so the trim happens here:
+        // descriptions are most of the payload (OpenAI's 767-job board is
+        // 5.9 MB with them, 2026-09-01), matching Greenhouse content:true
+        // and Workable details:true.
+        description: params?.descriptions === true ? str(j.descriptionPlain) : null,
         source: 'ashby-jobs',
         raw_extra: {
           workplace_type: str(j.workplaceType),
@@ -457,7 +463,15 @@ export const ATS_TEMPLATES = [
         }
       }));
 
-      return listResult(items, { api_version: str(payload.apiVersion) });
+      // The API carries no organization name anywhere in its payload; the
+      // board slug — from the caller's params, or the path segment of the
+      // API URL — is the one identity the response itself confirms (every
+      // jobUrl embeds it), and beats reporting null.
+      const company =
+        str(params?.company) ||
+        (url ? str(new URL(url).pathname.split('/').filter(Boolean).pop()) : null);
+
+      return listResult(items, { company, api_version: str(payload.apiVersion) });
     }
   },
 

package/src/embeddedState.js

@@ -35,6 +35,14 @@ const HTML_COMMENT_RE = /<!--[\s\S]*?-->/g;
 // self.__next_f.push([1,"<chunk>"]) — Next.js App Router RSC flight chunks.
 const NEXT_F_PUSH_RE = /self\.__next_f\.push\(\s*\[\s*1\s*,\s*/g;
 
+// (window[Symbol.for("ApolloSSRDataTransport")] ??= []).push({...}) — Apollo
+// Client's streaming-SSR rehydration payload. On an App Router page the RSC
+// flight stream can be a near-empty shell while the page's real data — the
+// GraphQL results its UI renders from — rides in these pushes instead
+// (producthunt.com product pages, observed 2026-09-01).
+const APOLLO_TRANSPORT_RE =
+  /Symbol\.for\(\s*["']ApolloSSRDataTransport["']\s*\)\s*\]\s*\?\?=\s*\[\s*\]\s*\)\s*\.push\(\s*/g;
+
 /**
  * Read an HTML attribute out of a raw tag's attribute string.
  * @param {string} attrs
@@ -191,6 +199,35 @@ function readFlightStream(html) {
   return { chunks: parts.length, stream: parts.join('') };
 }
 
+/**
+ * Collect every ApolloSSRDataTransport push in document order and parse it.
+ *
+ * The pushed literal is JSON except for bare `undefined` values, which Apollo
+ * emits for fields a query is still streaming (`"data":undefined`). Those are
+ * healed to null before parsing: the lookbehind/lookahead pins `undefined` to
+ * value position, so the word inside a string literal is never touched. A push
+ * that still fails to parse is skipped rather than failing the page.
+ *
+ * @param {string} html raw HTML
+ * @returns {object[]} parsed push arguments, in document order
+ */
+export function extractApolloTransport(html) {
+  const pushes = [];
+  APOLLO_TRANSPORT_RE.lastIndex = 0;
+  let match;
+  while ((match = APOLLO_TRANSPORT_RE.exec(html)) !== null) {
+    const literal = readBracketedLiteral(html, APOLLO_TRANSPORT_RE.lastIndex);
+    if (!literal) continue;
+    APOLLO_TRANSPORT_RE.lastIndex += literal.length;
+    try {
+      pushes.push(JSON.parse(literal.replace(/(?<=[:,[])\s*undefined\s*(?=[,}\]])/g, 'null')));
+    } catch {
+      // Not JSON we can heal — skip this push, keep the rest.
+    }
+  }
+  return pushes;
+}
+
 const serializedBytes = (value) => Buffer.byteLength(JSON.stringify(value) ?? '');
 
 /**
@@ -256,6 +293,17 @@ export function extractEmbeddedState(rawHtml) {
     });
   }
 
+  const apolloPushes = extractApolloTransport(html);
+  if (apolloPushes.length > 0) {
+    data.apollo_ssr_transport = apolloPushes;
+    found.push({
+      name: 'apollo_ssr_transport',
+      variable: 'window[Symbol.for("ApolloSSRDataTransport")]',
+      bytes: serializedBytes(apolloPushes),
+      note: `${apolloPushes.length} streaming-SSR push(es), in document order`
+    });
+  }
+
   for (const { name, variable } of STATE_VARIABLES) {
     const assignment = html.match(
       new RegExp(`(?:window|self|globalThis)?\\.?\\b${variable}\\s*=\\s*`)

package/src/jsonPath.js

@@ -66,7 +66,7 @@ export function selectJsonPath(root, path) {
   for (const segment of segments) {
     const container = current !== null && typeof current === 'object';
     const key = Array.isArray(current) ? Number(segment) : segment;
-    if (!container || !(key in current)) {
+    if (!container || !Object.hasOwn(current, key)) {
       const at = walked.length === 0 ? 'the result' : `"${walked.join('.')}"`;
       throw new Error(
         `Path "${path}" not found: ${at} has no "${segment}" (${describeOptions(current)})`

package/src/templates.js

@@ -14,7 +14,9 @@
  *
  * Templates do NOT make network calls. The caller fetches the page and passes
  * the body in; that keeps SSRF policy, timeouts and billing with the surface
- * that owns them.
+ * that owns them. resolveUrl/listUrl rewrite or build the target, so the caller
+ * must apply its SSRF policy to the URL they RETURN (and to every redirect the
+ * fetch follows), not just to the URL the caller started with.
  *
  * Two optional hooks let a template read a machine-readable endpoint instead of
  * scraping the rendered page, without taking the fetch into its own hands:
@@ -45,6 +47,7 @@ import { load } from 'cheerio';
 // have made one file the whole package.
 import { ATS_TEMPLATES } from './connectors/ats.js';
 import { GOV_TEMPLATES } from './connectors/gov.js';
+import { extractApolloTransport } from './embeddedState.js';
 import { safeHref } from './urls.js';
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
@@ -541,6 +544,16 @@ export const TEMPLATES = [
     description: 'Scrape an Amazon product page for title, price, rating, reviews, ASIN, and description.',
     targetPattern: /amazon\.(com|co\.uk|de|fr|jp|ca|com\.au)/i,
     extract($) {
+      // Amazon serves its robot check as HTTP 200: a "Continue shopping" page
+      // whose only form posts to /errors/validateCaptcha. Every selector below
+      // misses on it, which used to come back as a silent all-null record
+      // (amazon.es, observed 2026-09-01) — name the block instead.
+      if ($('form[action*="validateCaptcha"]').length > 0) {
+        throw new Error(
+          'Amazon answered with a captcha interstitial (an HTTP 200 robot check), not the product page. ' +
+          'The product data is not in this response — retry later or from a different IP.'
+        );
+      }
       const bullets = $('#feature-bullets ul li span.a-list-item')
         .map((_, el) => tidy($(el).text()))
         .get()
@@ -732,17 +745,50 @@ export const TEMPLATES = [
   {
     id: 'producthunt-launch',
     name: 'Product Hunt Launch',
-    description: 'Scrape a Product Hunt product page for name, tagline, vote count, topics, and maker details.',
-    targetPattern: /producthunt\.com\/posts\//i,
-    extract($) {
+    description:
+      'Scrape a Product Hunt product page for name, tagline, description, categories, website, ' +
+      'and follower/review counts. Product Hunt folded /posts/* launch pages into /products/* ' +
+      'product hubs, which carry no product-level vote count — followers and reviews are the ' +
+      "page's engagement numbers now.",
+    targetPattern: /producthunt\.com\/(posts|products)\//i,
+    extractRaw(body, url) {
+      // The RSC flight stream on these pages is a near-empty shell; the data
+      // the UI renders from — the GraphQL Product record — rides in Apollo's
+      // streaming-SSR transport pushes instead.
+      const products = [];
+      const walk = (node) => {
+        if (!node || typeof node !== 'object') return;
+        if (node.__typename === 'Product') products.push(node);
+        for (const value of Object.values(node)) walk(value);
+      };
+      for (const push of extractApolloTransport(body)) walk(push);
+      // The transport carries several Product objects (latestLaunch.product,
+      // forum subjects); the page's own is the one with the page-level fields.
+      const product =
+        products.find(p => 'followersCount' in p) || products.find(p => 'websiteUrl' in p) || null;
+
+      const $ = load(body);
+      const metaName = attr($, 'meta[property="og:title"]', 'content');
+      const categories = Array.isArray(product?.categories)
+        ? product.categories.map(c => c?.name).filter(Boolean)
+        : null;
+      // The DOM fallback's href carries PH's ?ref=producthunt tracking param.
+      const domWebsite = (attr($, 'a[data-test="visit-website-button"]', 'href') || '')
+        .replace(/([?&])ref=producthunt(?=&|$)/, '$1')
+        .replace(/[?&]$/, '') || null;
+
       return {
-        name: attr($, 'meta[property="og:title"]', 'content'),
-        tagline: attr($, 'meta[property="og:description"]', 'content'),
+        name: product?.name || (metaName ? metaName.replace(/\s*\|\s*Product Hunt\s*$/i, '') : null),
+        tagline: product?.tagline || attr($, 'meta[property="og:description"]', 'content'),
+        description: product?.description ?? null,
         image: attr($, 'meta[property="og:image"]', 'content'),
-        url: safeHref(attr($, 'meta[property="og:url"]', 'content')),
-        votes: text($, '[data-test="vote-button"] span') || text($, 'button[data-vote-button]'),
-        topics: list($, 'a[href*="/topics/"]'),
-        website: safeHref(attr($, 'a[data-test="product-link"]', 'href') || attr($, 'a[href][rel="noopener"][target="_blank"]', 'href'))
+        url: safeHref(attr($, 'meta[property="og:url"]', 'content')) || url,
+        website: safeHref(product?.websiteUrl || domWebsite),
+        // null = the data layer was missing, [] = present with no categories.
+        topics: categories,
+        followers: product?.followersCount ?? null,
+        reviews_count: product?.reviewsCount ?? null,
+        reviews_rating: product?.reviewsRating ?? null
       };
     }
   },
@@ -1028,6 +1074,21 @@ export class TemplateRegistry {
       ? template.extractRaw(body, url)
       : template.extract(load(body));
 
+    // A record with literally every field empty is not a page with no data —
+    // it is a page the template's selectors all missed: an interstitial
+    // (captcha, consent wall, bot check) served as HTTP 200, or a layout
+    // change. Reporting it as success is how amazon.es's captcha page came
+    // back as a clean all-null product (2026-09-01). Fail loudly instead.
+    const values = data && typeof data === 'object' && !Array.isArray(data) ? Object.values(data) : [];
+    const isEmpty = (v) => v == null || v === '' || (Array.isArray(v) && v.length === 0);
+    if (values.length > 0 && values.every(isEmpty)) {
+      throw new Error(
+        `Template "${id}" matched the page but extracted no data — every field came back empty. ` +
+        'The server likely answered with an interstitial (captcha, consent or bot wall) or the site changed ' +
+        'its layout. This is an extraction failure, not a page with nothing on it.'
+      );
+    }
+
     return {
       template: id,
       template_name: template.name,
@@ -1067,7 +1128,10 @@ export class TemplateRegistry {
       template_name: template.name,
       ...(url !== undefined ? { url } : {}),
       ...(params !== undefined ? { params } : {}),
-      data: template.extractList(body, url),
+      // params ride along for connectors whose upstream API has no query
+      // switch for an option (Ashby's descriptions opt-in trims at extract
+      // time; Greenhouse/Workable put theirs in the URL instead).
+      data: template.extractList(body, url, params),
       extractedAt: new Date().toISOString()
     };
   }