crawlforge-extractors 1.6.3 → 1.6.5

package/README.md

@@ -167,6 +167,25 @@ A source that is present but is not JSON — Nuxt 2's IIFE wrapper, Nuxt 3's
 unquoted-key object literal — is reported in `warnings` unparsed. Nothing here
 calls `eval`.
 
+### A Shopify product from the page's JSON-LD
+
+`shopifyProductFromJsonLd` reads the record `shopify-product` returns from the
+product page's own schema.org JSON-LD — a `Product`, a `ProductGroup` with one
+`hasVariant` `Product` per size, or an `AggregateOffer` — for stores that
+refuse `/products/<handle>.json` (gymshark.com answers it with 403 while the
+page itself is public).
+
+```js
+import { shopifyProductFromJsonLd } from 'crawlforge-extractors';
+
+const { found, data, reason } = shopifyProductFromJsonLd(html, response.url);
+// data.source === 'json-ld'; data.price, data.variants[].options, …
+```
+
+JSON-LD carries no per-variant stock count, compare-at price or option names,
+so those fields are null. Pass the page URL after redirects: a `/collections/`
+URL is reported in `reason` as a retired handle.
+
 ## Templates
 
 **Pages and products.** `shopify-product` · `shopify-collection` ·

package/index.d.ts

@@ -227,4 +227,49 @@ export declare function parseJsonPath(path: string): string[];
  */
 export declare function selectJsonPath(root: unknown, path: string): unknown;
 
+/** One variant as read from a product page's JSON-LD; stock counts and compare-at prices are not in JSON-LD. */
+export interface ShopifyJsonLdVariant {
+  id: string | null;
+  title: string | null;
+  price: string | null;
+  compare_at_price: null;
+  sku: string | null;
+  available: boolean | null;
+  inventory_quantity: null;
+  options: string[];
+}
+
+/** The shopify-product record shape, read from schema.org JSON-LD instead of /products/<handle>.json. */
+export interface ShopifyJsonLdProduct {
+  title: string | null;
+  vendor: string | null;
+  product_type: string | null;
+  handle: string | null;
+  product_id: string | null;
+  price: string | null;
+  compare_at_price: null;
+  on_sale: null;
+  currency: string | null;
+  price_min: string | null;
+  price_max: string | null;
+  available: boolean | null;
+  variants: ShopifyJsonLdVariant[];
+  options: string[];
+  description: string | null;
+  images: string[];
+  url: string;
+  source: 'json-ld';
+}
+
+/**
+ * A Shopify product from the product page's own schema.org JSON-LD (Product,
+ * ProductGroup with hasVariant, AggregateOffer), for stores that refuse
+ * /products/<handle>.json. `url` is the page URL after redirects; a
+ * /collections/ URL is named in `reason` as a retired handle.
+ */
+export declare function shopifyProductFromJsonLd(
+  html: string,
+  url: string
+): { found: true; data: ShopifyJsonLdProduct } | { found: false; reason: string };
+
 export default TemplateRegistry;

package/index.js

@@ -29,3 +29,5 @@ export { structureSignature, structuralSimilarity } from './src/structure.js';
 export { extractEmbeddedState } from './src/embeddedState.js';
 
 export { parseJsonPath, selectJsonPath } from './src/jsonPath.js';
+
+export { shopifyProductFromJsonLd } from './src/shopifyJsonLd.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.6.3",
+  "version": "1.6.5",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/body.js

@@ -26,6 +26,8 @@ export class BodyTooLargeError extends Error {
  * @param {Uint8Array} bytes
  * @returns {string}
  */
+export const META_CHARSET_SNIFF_BYTES = 8192;
+
 export function detectCharset(response, bytes) {
   const contentType = response.headers?.get?.('content-type') || '';
   const headerMatch = /charset=["']?([\w-]+)/i.exec(contentType);
@@ -33,10 +35,14 @@ export function detectCharset(response, bytes) {
     return headerMatch[1].trim().toLowerCase();
   }
 
-  // <meta charset> tags must appear within the first 1024 bytes per the
-  // HTML5 spec's prescan algorithm; ASCII-range bytes decode identically
-  // under latin1 regardless of the document's real encoding.
-  const sniffLength = Math.min(bytes.byteLength, 1024);
+  // The HTML5 prescan algorithm requires a <meta charset> within the first
+  // 1024 bytes, but real pages break the rule: vector.co.jp/magazine/softnews
+  // (Shift_JIS, no charset in the Content-Type header) declares it at byte
+  // 1293, behind a comment block, and every browser still decodes it
+  // correctly. Sniff a full 8 KB, which covers every <head> seen in the
+  // wild without decoding the whole body twice. ASCII-range bytes decode
+  // identically under latin1 regardless of the document's real encoding.
+  const sniffLength = Math.min(bytes.byteLength, META_CHARSET_SNIFF_BYTES);
   const sniffText = new TextDecoder('latin1').decode(bytes.subarray(0, sniffLength));
   const metaMatch =
     /<meta[^>]+charset=["']?([\w-]+)/i.exec(sniffText) ||

package/src/embeddedState.js

@@ -23,6 +23,10 @@ const STATE_VARIABLES = [
   { name: 'nuxt', variable: '__NUXT__' },
   { name: 'apollo_state', variable: '__APOLLO_STATE__' },
   { name: 'initial_state', variable: '__INITIAL_STATE__' },
+  // tumblr.com spells it with three underscores and assigns it in bracket
+  // notation: window['___INITIAL_STATE___'] = {...} (R17, 2026-09-04). Same
+  // key as the two-underscore form; the first one found on a page wins.
+  { name: 'initial_state', variable: '___INITIAL_STATE___' },
   { name: 'preloaded_state', variable: '__PRELOADED_STATE__' },
   // nytimes.com ships its whole front page as window.__preloadedData; with
   // only the four names above, a 1.1 MB page surfaced nothing but its
@@ -334,8 +338,13 @@ export function extractEmbeddedState(rawHtml) {
   }
 
   for (const { name, variable } of STATE_VARIABLES) {
+    if (data[name] !== undefined) continue;
+    // Dot or bracket notation on window/self/globalThis, or a bare/var
+    // assignment; \b keeps MY__INITIAL_STATE__ from matching __INITIAL_STATE__.
     const assignment = html.match(
-      new RegExp(`(?:window|self|globalThis)?\\.?\\b${variable}\\s*=\\s*`)
+      new RegExp(
+        `(?:(?:window|self|globalThis)\\s*\\[\\s*(['"])${variable}\\1\\s*\\]|(?:(?:window|self|globalThis)\\.)?\\b${variable})\\s*=\\s*`
+      )
     );
     if (!assignment) continue;
 

package/src/shopifyJsonLd.js

@@ -0,0 +1,178 @@
+/**
+ * A Shopify product read from the storefront page's own schema.org JSON-LD.
+ *
+ * shopify-product reads /products/<handle>.json, and some stores refuse that
+ * endpoint while the product page stays public: gymshark.com answers it with
+ * 403 and ships a ProductGroup with one priced Product per size in the page
+ * head (captured 2026-09-04); allbirds.com redirected a retired handle to a
+ * collection page, leaving a bare 404 behind the .json URL. Both surfaces
+ * (MCP server and REST API) fall back to this reader when the endpoint
+ * answers 401, 403, 404 or 410. The fetch itself stays with the caller.
+ *
+ * JSON-LD carries less than the endpoint: no per-variant inventory count, no
+ * compare-at price, no option names beyond what a variant states (size,
+ * color). Those fields are null or empty here, and `source: 'json-ld'` marks
+ * the record so a caller can tell the two readings apart.
+ */
+
+import { load } from 'cheerio';
+import { safeHref } from './urls.js';
+
+function asArray(value) {
+  if (value == null) return [];
+  return Array.isArray(value) ? value : [value];
+}
+
+function flattenLd(node, out = []) {
+  if (!node || typeof node !== 'object') return out;
+  if (Array.isArray(node)) { node.forEach((n) => flattenLd(n, out)); return out; }
+  out.push(node);
+  if (node['@graph']) flattenLd(node['@graph'], out);
+  return out;
+}
+
+const typed = (re) => (node) => asArray(node?.['@type']).some((t) => re.test(String(t)));
+const isProduct = typed(/^product$/i);
+const isProductGroup = typed(/^productgroup$/i);
+const isAggregateOffer = typed(/^aggregateoffer$/i);
+
+function toNumber(value) {
+  if (value == null || value === '') return null;
+  const n = Number(String(value).replace(/[^\d.,-]/g, '').replace(/,(?=\d{3}\b)/g, '').replace(',', '.'));
+  return Number.isFinite(n) ? n : null;
+}
+
+function money(n) {
+  return n == null ? null : n.toFixed(2);
+}
+
+/** Gymshark writes its name as "Arrival 5&quot; Shorts" inside the JSON. */
+function decodeEntities(value) {
+  if (typeof value !== 'string') return null;
+  const text = /&(?:[a-z]+|#\d+|#x[0-9a-f]+);/i.test(value) ? load(`<x>${value}</x>`)('x').text() : value;
+  return text.replace(/\s+/g, ' ').trim() || null;
+}
+
+function availabilityOf(offer) {
+  const a = String(offer?.availability || '');
+  if (!a) return null;
+  if (/InStock|PreOrder|BackOrder|LimitedAvailability|OnlineOnly/i.test(a)) return true;
+  if (/OutOfStock|SoldOut|Discontinued/i.test(a)) return false;
+  return null;
+}
+
+/** The offers of a node, an AggregateOffer's own list included. */
+function offersOf(node) {
+  return asArray(node?.offers)
+    .flatMap((o) => (o && isAggregateOffer(o) && o.offers ? asArray(o.offers) : [o]))
+    .filter(Boolean);
+}
+
+function offerPrice(offer) {
+  return toNumber(offer.price ?? offer.lowPrice);
+}
+
+function imageUrls(image) {
+  return asArray(image)
+    .map((i) => (i && typeof i === 'object' ? i.url || i.contentUrl : i))
+    .map((u) => (typeof u === 'string' ? safeHref(u) : null))
+    .filter(Boolean);
+}
+
+/**
+ * @param {string} html - the product page
+ * @param {string} url - the URL that page was read from (after redirects)
+ * @returns {{ found: true, data: object } | { found: false, reason: string }}
+ */
+export function shopifyProductFromJsonLd(html, url) {
+  const $ = load(html || '');
+  const nodes = [];
+  $('script[type="application/ld+json"]').each((_, el) => {
+    const raw = $(el).html();
+    if (!raw) return;
+    try { flattenLd(JSON.parse(raw), nodes); } catch { /* one bad block does not lose the rest */ }
+  });
+  const product = nodes.find(isProductGroup) || nodes.find(isProduct);
+  if (!product) {
+    return {
+      found: false,
+      reason: `${url} carries no schema.org Product JSON-LD` +
+        (/\/collections\//.test(url)
+          ? ' — the product URL redirected to a collection page, so the product handle no longer exists'
+          : '')
+    };
+  }
+
+  // A ProductGroup (or a Product with hasVariant) prices each variant on the
+  // variant; a plain Product prices its offers directly.
+  const variantNodes = asArray(product.hasVariant).filter(isProduct);
+  const variants = variantNodes.length > 0
+    ? variantNodes.map((v) => {
+      const offers = offersOf(v);
+      const prices = offers.map(offerPrice).filter((n) => n != null);
+      const availabilities = offers.map(availabilityOf).filter((a) => a !== null);
+      const options = ['size', 'color', 'material', 'pattern'].map((k) => decodeEntities(v[k])).filter(Boolean);
+      return {
+        // mpn is the per-size id on gymshark, where sku is shared by the group;
+        // Dawn-theme stores put the per-variant id in sku and have no mpn.
+        id: v.mpn || v.sku || v.productID || null,
+        title: decodeEntities(v.name) || options.join(' / ') || v.sku || null,
+        price: money(prices.length ? Math.min(...prices) : null),
+        compare_at_price: null,
+        sku: v.sku || null,
+        available: availabilities.length ? availabilities.some(Boolean) : null,
+        inventory_quantity: null,
+        options
+      };
+    })
+    : offersOf(product).map((o) => ({
+      id: o.sku || o.mpn || null,
+      title: decodeEntities(o.name) || o.sku || null,
+      price: money(offerPrice(o)),
+      compare_at_price: null,
+      sku: o.sku || null,
+      available: availabilityOf(o),
+      inventory_quantity: null,
+      options: []
+    }));
+
+  const allOffers = variantNodes.length > 0 ? variantNodes.flatMap(offersOf) : offersOf(product);
+  const prices = allOffers.map(offerPrice).filter((n) => n != null);
+  const highs = allOffers.map((o) => toNumber(o.highPrice)).filter((n) => n != null);
+  const priceMin = prices.length ? Math.min(...prices) : null;
+  const priceMax = prices.length ? Math.max(...prices, ...highs) : (highs.length ? Math.max(...highs) : null);
+  const currency = allOffers.map((o) => o.priceCurrency).find(Boolean) || null;
+  const availabilities = allOffers.map(availabilityOf).filter((v) => v !== null);
+  const available = availabilities.length ? availabilities.some(Boolean) : null;
+
+  let handle = null;
+  try { handle = (new URL(url).pathname.match(/\/products\/([^/?#]+)/) || [])[1] || null; } catch { /* keep null */ }
+
+  const brand = product.brand && typeof product.brand === 'object' ? product.brand.name : product.brand;
+  const options = asArray(product.variesBy).map((v) => String(v).split('/').pop().toLowerCase()).filter(Boolean);
+  const productUrl = typeof product.url === 'string' ? safeHref(product.url) : null;
+
+  return {
+    found: true,
+    data: {
+      title: decodeEntities(product.name),
+      vendor: decodeEntities(brand),
+      product_type: decodeEntities(product.category),
+      handle,
+      product_id: product.productGroupID || product.productID || product.sku || null,
+      price: money(priceMin),
+      compare_at_price: null,
+      on_sale: null,
+      currency,
+      price_min: money(priceMin),
+      price_max: money(priceMax),
+      available,
+      variants,
+      options,
+      description: decodeEntities(product.description),
+      images: imageUrls(product.image),
+      url: productUrl || url,
+      source: 'json-ld'
+    }
+  };
+}

package/src/templates.js

@@ -270,6 +270,19 @@ function amazonByline($) {
   // continues into "(Author) Format: Hardcover".
   if (/^by\s/i.test(raw)) return contributor || tidy(raw.replace(/^by\s+/i, '').split('(')[0]);
 
+  // Every other marketplace phrases the book byline in its own language —
+  // "Engelska utgåvan av George Orwell (Författare)", "Wydanie: Angielski
+  // George Orwell (Autor)", "Édition en Anglais de George Orwell (Auteur)" —
+  // and none starts with "by", so the chrome came back whole on amazon.se,
+  // .pl, .com.be and .com.tr (R17, 2026-09-04). The author's own link
+  // carries the bare name on every marketplace.
+  const authorLink = tidy($('#bylineInfo .author a, #bylineInfo a.contributorNameID').first().text());
+  if (authorLink && /\(/.test(raw)) return authorLink;
+
+  // "Marke: Sony", "Marca: Sony", "Marque : Sony" — a localised brand label.
+  const labelled = raw.match(/^[\p{L}\s]{2,20}?\s?:\s*(.+)$/u);
+  if (labelled && !/\(/.test(raw)) return tidy(labelled[1]);
+
   return raw;
 }
 
@@ -356,7 +369,7 @@ function amazonPrice($) {
  */
 function amazonCurrency($, price) {
   if (!price) return null;
-  const code = price.match(/(USD|EUR|GBP|INR|JPY|CAD|AUD|MXN|BRL|SGD|AED|SAR|PLN|TRY)(?![A-Z])/);
+  const code = price.match(/(USD|EUR|GBP|INR|JPY|CAD|AUD|MXN|BRL|SGD|AED|SAR|EGP|PLN|TRY|SEK)(?![A-Z])/);
   if (code) return code[1];
   if (price.includes('₹')) return 'INR';
   if (price.includes('€')) return 'EUR';
@@ -364,9 +377,16 @@ function amazonCurrency($, price) {
   if (price.includes('¥') || price.includes('￥')) return 'JPY';
   if (price.includes('R$')) return 'BRL';
   if (price.includes('zł')) return 'PLN';
-  if (price.includes('₺')) return 'TRY';
+  // amazon.com.tr writes "460,67TL" (R17, 2026-09-04); the lira sign is rarer.
+  if (price.includes('₺') || /(?<![A-Za-z])TL(?![A-Za-z])/.test(price)) return 'TRY';
+  // Arabic-script marketplaces: dirham, Egyptian pound, riyal.
+  if (/د\.?إ/.test(price)) return 'AED';
+  if (/ج\.?م/.test(price)) return 'EGP';
+  if (/ر\.?س|﷼/.test(price)) return 'SAR';
+  const host = (attr($, 'link[rel="canonical"]', 'href') || '').match(/^https?:\/\/([^/]+)/)?.[1] || '';
+  // "114,30kr" on amazon.se — the only Amazon marketplace priced in kronor.
+  if (/(?<![A-Za-z])kr(?![A-Za-z])/i.test(price)) return /\.se$/.test(host) ? 'SEK' : null;
   if (price.includes('$')) {
-    const host = (attr($, 'link[rel="canonical"]', 'href') || '').match(/^https?:\/\/([^/]+)/)?.[1] || '';
     if (/\.ca$/.test(host)) return 'CAD';
     if (/\.com\.au$/.test(host)) return 'AUD';
     if (/\.com\.mx$/.test(host)) return 'MXN';
@@ -376,6 +396,15 @@ function amazonCurrency($, price) {
   return null;
 }
 
+function hnAbsoluteUrl(href) {
+  if (!href) return href;
+  try {
+    return new URL(href, 'https://news.ycombinator.com/').href;
+  } catch {
+    return href;
+  }
+}
+
 function hnCommentCount(label) {
   const value = (label || '').trim();
   if (!value) return null;
@@ -858,7 +887,9 @@ export const TEMPLATES = [
         stories.push({
           id: $row.attr('id'),
           title: $titleLink.text().trim(),
-          url: safeHref($titleLink.attr('href')),
+          // Text posts (Ask HN, Show HN without a link) carry a relative
+          // "item?id=…" href; resolve it so every story url is absolute.
+          url: safeHref(hnAbsoluteUrl($titleLink.attr('href'))),
           site: $row.find('.sitebit a').text().trim() || null,
           // "1 point" on a fresh story and "3 points" on the rest — strip both.
           score: $score.text().replace(/\s*points?$/, '').trim() || null,