crawlforge-extractors 1.6.4 → 1.7.0

package/README.md

@@ -167,6 +167,47 @@ A source that is present but is not JSON — Nuxt 2's IIFE wrapper, Nuxt 3's
 unquoted-key object literal — is reported in `warnings` unparsed. Nothing here
 calls `eval`.
 
+### A Shopify product from the page's JSON-LD
+
+`shopifyProductFromJsonLd` reads the record `shopify-product` returns from the
+product page's own schema.org JSON-LD — a `Product`, a `ProductGroup` with one
+`hasVariant` `Product` per size, or an `AggregateOffer` — for stores that
+refuse `/products/<handle>.json` (gymshark.com answers it with 403 while the
+page itself is public).
+
+```js
+import { shopifyProductFromJsonLd } from 'crawlforge-extractors';
+
+const { found, data, reason } = shopifyProductFromJsonLd(html, response.url);
+// data.source === 'json-ld'; data.price, data.variants[].options, …
+```
+
+JSON-LD carries no per-variant stock count, compare-at price or option names,
+so those fields are null. Pass the page URL after redirects: a `/collections/`
+URL is reported in `reason` as a retired handle.
+
+### Recognising a blocked page
+
+`documentVerdict` says what a fetched document is: the page, a bot-wall
+interstitial (Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel), an
+HTTP error page, an empty shell, or a short error-titled placeholder. A wall
+arrives as HTTP 200 with a title and prose of its own, so a fetch that only
+checks the status reports it as a success — producthunt.com came back
+`success: true, title: "Just a moment..."` for three regression rounds.
+
+```js
+import { documentVerdict } from 'crawlforge-extractors';
+
+const verdict = documentVerdict(
+  { url: response.url, status: response.status, title, text, html },
+  { fetcher: 'a plain fetch', rendered: false, contentReturned: false }
+);
+// { success: false, status: 200, blocked: { vendor: 'cloudflare', evidence: 'title "Just a moment..."' }, error: '…' }
+```
+
+`detectChallengePage` is the vendor check alone. Both are pure: the caller
+keeps or drops the content, and decides what to try next.
+
 ## Templates
 
 **Pages and products.** `shopify-product` · `shopify-collection` ·

package/index.d.ts

@@ -227,4 +227,100 @@ export declare function parseJsonPath(path: string): string[];
  */
 export declare function selectJsonPath(root: unknown, path: string): unknown;
 
+/** One variant as read from a product page's JSON-LD; stock counts and compare-at prices are not in JSON-LD. */
+export interface ShopifyJsonLdVariant {
+  id: string | null;
+  title: string | null;
+  price: string | null;
+  compare_at_price: null;
+  sku: string | null;
+  available: boolean | null;
+  inventory_quantity: null;
+  options: string[];
+}
+
+/** The shopify-product record shape, read from schema.org JSON-LD instead of /products/<handle>.json. */
+export interface ShopifyJsonLdProduct {
+  title: string | null;
+  vendor: string | null;
+  product_type: string | null;
+  handle: string | null;
+  product_id: string | null;
+  price: string | null;
+  compare_at_price: null;
+  on_sale: null;
+  currency: string | null;
+  price_min: string | null;
+  price_max: string | null;
+  available: boolean | null;
+  variants: ShopifyJsonLdVariant[];
+  options: string[];
+  description: string | null;
+  images: string[];
+  url: string;
+  source: 'json-ld';
+}
+
+/**
+ * A Shopify product from the product page's own schema.org JSON-LD (Product,
+ * ProductGroup with hasVariant, AggregateOffer), for stores that refuse
+ * /products/<handle>.json. `url` is the page URL after redirects; a
+ * /collections/ URL is named in `reason` as a retired handle.
+ */
+export declare function shopifyProductFromJsonLd(
+  html: string,
+  url: string
+): { found: true; data: ShopifyJsonLdProduct } | { found: false; reason: string };
+
 export default TemplateRegistry;
+
+/** The bot-defence vendor whose interstitial a document is, and what gave it away. */
+export interface ChallengeVerdict {
+  vendor: 'cloudflare' | 'amazon' | 'datadome' | 'perimeterx' | 'akamai' | 'vercel';
+  evidence: string;
+}
+
+/**
+ * Recognise a bot-wall interstitial served as a page (HTTP 200, a title, some
+ * prose and a challenge script). A title match is definitive; a script or
+ * form marker is definitive only on a short page, because a real page can
+ * legitimately embed a Turnstile widget.
+ */
+export declare function detectChallengePage(page: {
+  title?: string;
+  html?: string;
+  text?: string;
+}): ChallengeVerdict | null;
+
+export interface DocumentVerdict {
+  success: boolean;
+  /** The navigation's HTTP status when the caller had one, else null. */
+  status: number | null;
+  /** Present on every failure: what the document is and what to do. */
+  error?: string;
+  /** Present when the failure is a challenge wall. */
+  blocked?: ChallengeVerdict;
+}
+
+/**
+ * What a fetched document is: the page, a challenge wall, an HTTP error page,
+ * an empty shell, or a short error-titled placeholder. The defaults describe
+ * a stealth-browser caller: a browser `rendered` the document, `fetcher`
+ * names it in the messages, `waitedMs` is the extra render wait it gave an
+ * empty document, and the failure result still carries the content
+ * (`contentReturned`). A plain fetch passes `rendered: false` and
+ * `contentReturned: false`.
+ */
+export declare function documentVerdict(
+  scraped: { url?: string; title?: string; text?: string; html?: string; status?: number | null },
+  options?: {
+    waitedMs?: number;
+    allowEmpty?: boolean;
+    fetcher?: string;
+    rendered?: boolean;
+    contentReturned?: boolean;
+  }
+): DocumentVerdict;
+
+/** A document with this much text or less and an error title is a placeholder. */
+export declare const SOFT_ERROR_MAX_CHARS: number;

package/index.js

@@ -29,3 +29,7 @@ export { structureSignature, structuralSimilarity } from './src/structure.js';
 export { extractEmbeddedState } from './src/embeddedState.js';
 
 export { parseJsonPath, selectJsonPath } from './src/jsonPath.js';
+
+export { shopifyProductFromJsonLd } from './src/shopifyJsonLd.js';
+
+export { detectChallengePage, documentVerdict, SOFT_ERROR_MAX_CHARS } from './src/blockedPage.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.6.4",
+  "version": "1.7.0",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/blockedPage.js

@@ -0,0 +1,168 @@
+/**
+ * blockedPage.js — decide whether a fetched document is the page or a wall.
+ *
+ * Cloudflare, Amazon, DataDome, PerimeterX, Akamai and Vercel all answer a
+ * blocked request with HTTP 200 and a page of their own: a title, some prose
+ * and a challenge script. Reported as a successful scrape, that page hides
+ * the block — producthunt.com came back "success:true, title: Just a
+ * moment..." for three regression rounds (R10 Q1 → R15, 2026-09-04). The
+ * MCP server's stealth path learned to name these in 5.6.2 and to name HTTP
+ * error pages and short error-titled placeholders in 5.6.9; the plain
+ * `scrape` path on both surfaces never looked. The tables live here so the
+ * two surfaces reach one verdict.
+ *
+ * A title match is definitive; a script or form marker is definitive only
+ * on a short page, because a real page can legitimately embed a Turnstile
+ * widget.
+ */
+
+const SHORT_PAGE_CHARS = 4000;
+
+const CHALLENGES = [
+  {
+    vendor: 'cloudflare',
+    title: /^just a moment/i,
+    markers: /challenges\.cloudflare\.com|cf-chl-|_cf_chl_opt|cf_chl_rc_|window\._cf_chl/i,
+    evidence: 'a Cloudflare challenge script'
+  },
+  {
+    vendor: 'amazon',
+    // The robot check is the only Amazon page whose form posts to validateCaptcha.
+    definitive: /action="[^"]*validateCaptcha/i,
+    evidence: 'the validateCaptcha form'
+  },
+  {
+    vendor: 'datadome',
+    markers: /captcha-delivery\.com\/captcha|geo\.captcha-delivery\.com|dd\.captcha/i,
+    evidence: 'a DataDome captcha frame'
+  },
+  {
+    vendor: 'perimeterx',
+    markers: /px-captcha|_pxCaptcha|human-challenge/i,
+    evidence: 'a PerimeterX / HUMAN challenge element'
+  },
+  {
+    vendor: 'akamai',
+    title: /^access denied/i,
+    markers: /errors\.edgesuite\.net/i,
+    evidence: 'an Akamai access-denied page'
+  },
+  {
+    // Vercel's Attack Challenge Mode answers with HTTP 429, an
+    // x-vercel-mitigated: challenge header and a JavaScript interstitial
+    // titled "Vercel Security Checkpoint" (lesswrong.com, hashicorp.com,
+    // bombas.com, R17 2026-09-04). Chromium solves it and reloads; camoufox
+    // was left on the interstitial, which then read as a successful scrape.
+    vendor: 'vercel',
+    title: /^vercel security checkpoint/i,
+    markers: /vercel\.link\/security-checkpoint|_vercel\/challenge|x-vercel-challenge-token/i,
+    evidence: 'a Vercel Security Checkpoint page'
+  }
+];
+
+/**
+ * @param {{ title?: string, html?: string, text?: string }} page
+ * @returns {{ vendor: string, evidence: string } | null}
+ */
+export function detectChallengePage({ title = '', html = '', text = '' } = {}) {
+  const cleanTitle = (title || '').trim();
+  const visible = (text || '').replace(/\s+/g, ' ').trim();
+  const shortPage = visible.length < SHORT_PAGE_CHARS;
+  for (const challenge of CHALLENGES) {
+    if (challenge.title && challenge.title.test(cleanTitle)) {
+      return { vendor: challenge.vendor, evidence: `title "${cleanTitle}"` };
+    }
+    if (challenge.definitive && challenge.definitive.test(html)) {
+      return { vendor: challenge.vendor, evidence: challenge.evidence };
+    }
+    if (shortPage && challenge.markers && challenge.markers.test(html)) {
+      return { vendor: challenge.vendor, evidence: `${challenge.evidence} on a ${visible.length}-character page` };
+    }
+  }
+  return null;
+}
+
+// A document this short with one of these titles is an error placeholder,
+// not a page. Real pages with these words in a longer title (a news story
+// about an outage) carry far more text than the cap.
+const ERROR_TITLE = /^(?:(?:\d{3}\s*[-–—|:]\s*)?(?:error(?: page)?|access denied|forbidden|(?:page )?not found|service unavailable|internal server error|bad gateway|something went wrong|oops!?[^\n]{0,60}))$/i;
+export const SOFT_ERROR_MAX_CHARS = 1500;
+
+/**
+ * What a fetched document is: the page, a challenge wall, an HTTP error
+ * page, an empty shell, or a short error-titled placeholder. The content is
+ * for the caller to keep or drop — this only says what it is.
+ *
+ * The defaults describe the MCP server's stealth path, the original caller:
+ * a browser `rendered` the document, `fetcher` names it in the messages,
+ * `waitedMs` is the extra render wait it gave an empty document, and the
+ * failure result still carries the content (`contentReturned`). A plain
+ * fetch passes `rendered: false` (its empty shell or placeholder cannot be
+ * waited out — only a browser paints it) and `contentReturned: false` (it
+ * drops the document on a failure).
+ *
+ * @param {{ url?: string, title?: string, text?: string, html?: string, status?: number|null }} scraped
+ * @param {{ waitedMs?: number, allowEmpty?: boolean, fetcher?: string, rendered?: boolean, contentReturned?: boolean }} [options]
+ * @returns {{ success: boolean, status: number|null, error?: string, blocked?: { vendor: string, evidence: string } }}
+ */
+export function documentVerdict(scraped, { waitedMs = 0, allowEmpty = false, fetcher = 'the stealth browser', rendered = true, contentReturned = true } = {}) {
+  const status = Number.isInteger(scraped?.status) ? scraped.status : null;
+  const url = scraped?.url || '';
+  const title = String(scraped?.title || '').trim();
+  const text = String(scraped?.text || '').trim();
+
+  const challenge = detectChallengePage(scraped || {});
+  if (challenge) {
+    return {
+      success: false,
+      status,
+      blocked: challenge,
+      error: `${challenge.vendor} served a challenge page instead of the content (${challenge.evidence}); ${fetcher} did not pass it.`
+    };
+  }
+
+  if (status !== null && status >= 400) {
+    const why = status === 403
+      ? 'A 403 with no challenge vendor on the page is an IP-reputation or WAF block; the site will not serve this network.'
+      : status === 404
+        ? 'The site says the URL does not exist — check the path.'
+        : status === 429
+          ? 'The site is rate-limiting this network; wait before retrying.'
+          : 'Retry later; the server, not the page, failed.';
+    return {
+      success: false,
+      status,
+      error: `HTTP ${status}: ${url} answered with an error page${title ? ` titled "${title}"` : ''}, not the resource${contentReturned ? '; the content returned is that page' : ''}. ${why}`
+    };
+  }
+
+  if (!title && !text) {
+    if (allowEmpty) return { success: true, status };
+    const reached = fetcher.charAt(0).toUpperCase() + fetcher.slice(1);
+    const bytes = (scraped?.html || '').length;
+    return {
+      success: false,
+      status,
+      error: rendered
+        ? `${reached} reached ${url} but the document rendered no title and no text` +
+          ` after ${waitedMs}ms of extra wait (${bytes} bytes of HTML).` +
+          ' A JavaScript-rendered page needs a longer wait_for; an empty document means the server sent nothing to render.'
+        : `${reached} reached ${url} but the document has no title and no text (${bytes} bytes of HTML).` +
+          ' The page is rendered by JavaScript or the server sent an empty shell; only a browser renders it.'
+    };
+  }
+
+  if (text.length < SOFT_ERROR_MAX_CHARS && ERROR_TITLE.test(title)) {
+    return {
+      success: false,
+      status,
+      error:
+        `${url} rendered an error page titled "${title}" (${text.length} characters of text) instead of the resource` +
+        (rendered
+          ? ' — a soft block or an application error. Retry later, or with a longer wait_for if the site paints content after a placeholder.'
+          : ' — a soft block or an application error. Retry later; if the site paints content after a placeholder, only a browser renders it.')
+    };
+  }
+
+  return { success: true, status };
+}

package/src/shopifyJsonLd.js

@@ -0,0 +1,178 @@
+/**
+ * A Shopify product read from the storefront page's own schema.org JSON-LD.
+ *
+ * shopify-product reads /products/<handle>.json, and some stores refuse that
+ * endpoint while the product page stays public: gymshark.com answers it with
+ * 403 and ships a ProductGroup with one priced Product per size in the page
+ * head (captured 2026-09-04); allbirds.com redirected a retired handle to a
+ * collection page, leaving a bare 404 behind the .json URL. Both surfaces
+ * (MCP server and REST API) fall back to this reader when the endpoint
+ * answers 401, 403, 404 or 410. The fetch itself stays with the caller.
+ *
+ * JSON-LD carries less than the endpoint: no per-variant inventory count, no
+ * compare-at price, no option names beyond what a variant states (size,
+ * color). Those fields are null or empty here, and `source: 'json-ld'` marks
+ * the record so a caller can tell the two readings apart.
+ */
+
+import { load } from 'cheerio';
+import { safeHref } from './urls.js';
+
+function asArray(value) {
+  if (value == null) return [];
+  return Array.isArray(value) ? value : [value];
+}
+
+function flattenLd(node, out = []) {
+  if (!node || typeof node !== 'object') return out;
+  if (Array.isArray(node)) { node.forEach((n) => flattenLd(n, out)); return out; }
+  out.push(node);
+  if (node['@graph']) flattenLd(node['@graph'], out);
+  return out;
+}
+
+const typed = (re) => (node) => asArray(node?.['@type']).some((t) => re.test(String(t)));
+const isProduct = typed(/^product$/i);
+const isProductGroup = typed(/^productgroup$/i);
+const isAggregateOffer = typed(/^aggregateoffer$/i);
+
+function toNumber(value) {
+  if (value == null || value === '') return null;
+  const n = Number(String(value).replace(/[^\d.,-]/g, '').replace(/,(?=\d{3}\b)/g, '').replace(',', '.'));
+  return Number.isFinite(n) ? n : null;
+}
+
+function money(n) {
+  return n == null ? null : n.toFixed(2);
+}
+
+/** Gymshark writes its name as "Arrival 5&quot; Shorts" inside the JSON. */
+function decodeEntities(value) {
+  if (typeof value !== 'string') return null;
+  const text = /&(?:[a-z]+|#\d+|#x[0-9a-f]+);/i.test(value) ? load(`<x>${value}</x>`)('x').text() : value;
+  return text.replace(/\s+/g, ' ').trim() || null;
+}
+
+function availabilityOf(offer) {
+  const a = String(offer?.availability || '');
+  if (!a) return null;
+  if (/InStock|PreOrder|BackOrder|LimitedAvailability|OnlineOnly/i.test(a)) return true;
+  if (/OutOfStock|SoldOut|Discontinued/i.test(a)) return false;
+  return null;
+}
+
+/** The offers of a node, an AggregateOffer's own list included. */
+function offersOf(node) {
+  return asArray(node?.offers)
+    .flatMap((o) => (o && isAggregateOffer(o) && o.offers ? asArray(o.offers) : [o]))
+    .filter(Boolean);
+}
+
+function offerPrice(offer) {
+  return toNumber(offer.price ?? offer.lowPrice);
+}
+
+function imageUrls(image) {
+  return asArray(image)
+    .map((i) => (i && typeof i === 'object' ? i.url || i.contentUrl : i))
+    .map((u) => (typeof u === 'string' ? safeHref(u) : null))
+    .filter(Boolean);
+}
+
+/**
+ * @param {string} html - the product page
+ * @param {string} url - the URL that page was read from (after redirects)
+ * @returns {{ found: true, data: object } | { found: false, reason: string }}
+ */
+export function shopifyProductFromJsonLd(html, url) {
+  const $ = load(html || '');
+  const nodes = [];
+  $('script[type="application/ld+json"]').each((_, el) => {
+    const raw = $(el).html();
+    if (!raw) return;
+    try { flattenLd(JSON.parse(raw), nodes); } catch { /* one bad block does not lose the rest */ }
+  });
+  const product = nodes.find(isProductGroup) || nodes.find(isProduct);
+  if (!product) {
+    return {
+      found: false,
+      reason: `${url} carries no schema.org Product JSON-LD` +
+        (/\/collections\//.test(url)
+          ? ' — the product URL redirected to a collection page, so the product handle no longer exists'
+          : '')
+    };
+  }
+
+  // A ProductGroup (or a Product with hasVariant) prices each variant on the
+  // variant; a plain Product prices its offers directly.
+  const variantNodes = asArray(product.hasVariant).filter(isProduct);
+  const variants = variantNodes.length > 0
+    ? variantNodes.map((v) => {
+      const offers = offersOf(v);
+      const prices = offers.map(offerPrice).filter((n) => n != null);
+      const availabilities = offers.map(availabilityOf).filter((a) => a !== null);
+      const options = ['size', 'color', 'material', 'pattern'].map((k) => decodeEntities(v[k])).filter(Boolean);
+      return {
+        // mpn is the per-size id on gymshark, where sku is shared by the group;
+        // Dawn-theme stores put the per-variant id in sku and have no mpn.
+        id: v.mpn || v.sku || v.productID || null,
+        title: decodeEntities(v.name) || options.join(' / ') || v.sku || null,
+        price: money(prices.length ? Math.min(...prices) : null),
+        compare_at_price: null,
+        sku: v.sku || null,
+        available: availabilities.length ? availabilities.some(Boolean) : null,
+        inventory_quantity: null,
+        options
+      };
+    })
+    : offersOf(product).map((o) => ({
+      id: o.sku || o.mpn || null,
+      title: decodeEntities(o.name) || o.sku || null,
+      price: money(offerPrice(o)),
+      compare_at_price: null,
+      sku: o.sku || null,
+      available: availabilityOf(o),
+      inventory_quantity: null,
+      options: []
+    }));
+
+  const allOffers = variantNodes.length > 0 ? variantNodes.flatMap(offersOf) : offersOf(product);
+  const prices = allOffers.map(offerPrice).filter((n) => n != null);
+  const highs = allOffers.map((o) => toNumber(o.highPrice)).filter((n) => n != null);
+  const priceMin = prices.length ? Math.min(...prices) : null;
+  const priceMax = prices.length ? Math.max(...prices, ...highs) : (highs.length ? Math.max(...highs) : null);
+  const currency = allOffers.map((o) => o.priceCurrency).find(Boolean) || null;
+  const availabilities = allOffers.map(availabilityOf).filter((v) => v !== null);
+  const available = availabilities.length ? availabilities.some(Boolean) : null;
+
+  let handle = null;
+  try { handle = (new URL(url).pathname.match(/\/products\/([^/?#]+)/) || [])[1] || null; } catch { /* keep null */ }
+
+  const brand = product.brand && typeof product.brand === 'object' ? product.brand.name : product.brand;
+  const options = asArray(product.variesBy).map((v) => String(v).split('/').pop().toLowerCase()).filter(Boolean);
+  const productUrl = typeof product.url === 'string' ? safeHref(product.url) : null;
+
+  return {
+    found: true,
+    data: {
+      title: decodeEntities(product.name),
+      vendor: decodeEntities(brand),
+      product_type: decodeEntities(product.category),
+      handle,
+      product_id: product.productGroupID || product.productID || product.sku || null,
+      price: money(priceMin),
+      compare_at_price: null,
+      on_sale: null,
+      currency,
+      price_min: money(priceMin),
+      price_max: money(priceMax),
+      available,
+      variants,
+      options,
+      description: decodeEntities(product.description),
+      images: imageUrls(product.image),
+      url: productUrl || url,
+      source: 'json-ld'
+    }
+  };
+}