crawlforge-extractors 1.6.1 → 1.6.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.6.1",
+  "version": "1.6.3",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/embeddedState.js

@@ -23,7 +23,11 @@ const STATE_VARIABLES = [
   { name: 'nuxt', variable: '__NUXT__' },
   { name: 'apollo_state', variable: '__APOLLO_STATE__' },
   { name: 'initial_state', variable: '__INITIAL_STATE__' },
-  { name: 'preloaded_state', variable: '__PRELOADED_STATE__' }
+  { name: 'preloaded_state', variable: '__PRELOADED_STATE__' },
+  // nytimes.com ships its whole front page as window.__preloadedData; with
+  // only the four names above, a 1.1 MB page surfaced nothing but its
+  // <script type="application/json"> blocks (R15, 2026-09-04).
+  { name: 'preloaded_data', variable: '__preloadedData' }
 ];
 
 // Script bodies cannot contain a literal "</script", so a non-greedy match is
@@ -43,6 +47,33 @@ const NEXT_F_PUSH_RE = /self\.__next_f\.push\(\s*\[\s*1\s*,\s*/g;
 const APOLLO_TRANSPORT_RE =
   /Symbol\.for\(\s*["']ApolloSSRDataTransport["']\s*\)\s*\]\s*\?\?=\s*\[\s*\]\s*\)\s*\.push\(\s*/g;
 
+// Bare `undefined` is the one JS-only token serializers emit inside otherwise
+// valid JSON (Apollo for still-streaming fields, nytimes' __preloadedData).
+// The lookbehind/lookahead pin it to a value position, so a string containing
+// the word is left alone.
+const BARE_UNDEFINED_RE = /(?<=[:,[])\s*undefined\s*(?=[,}\]])/g;
+
+/**
+ * JSON.parse a literal, healing bare `undefined` values to null when a
+ * strict parse fails. Returns parsed:undefined when the literal is not JSON
+ * even after healing.
+ * @param {string} literal
+ * @returns {{ parsed: unknown, healed: number }}
+ */
+function parseJsonHealingUndefined(literal) {
+  try {
+    return { parsed: JSON.parse(literal), healed: 0 };
+  } catch {
+    const healed = (literal.match(BARE_UNDEFINED_RE) || []).length;
+    if (healed === 0) return { parsed: undefined, healed: 0 };
+    try {
+      return { parsed: JSON.parse(literal.replace(BARE_UNDEFINED_RE, 'null')), healed };
+    } catch {
+      return { parsed: undefined, healed: 0 };
+    }
+  }
+}
+
 /**
  * Read an HTML attribute out of a raw tag's attribute string.
  * @param {string} attrs
@@ -219,11 +250,9 @@ export function extractApolloTransport(html) {
     const literal = readBracketedLiteral(html, APOLLO_TRANSPORT_RE.lastIndex);
     if (!literal) continue;
     APOLLO_TRANSPORT_RE.lastIndex += literal.length;
-    try {
-      pushes.push(JSON.parse(literal.replace(/(?<=[:,[])\s*undefined\s*(?=[,}\]])/g, 'null')));
-    } catch {
-      // Not JSON we can heal — skip this push, keep the rest.
-    }
+    // Not JSON we can heal — skip this push, keep the rest.
+    const { parsed } = parseJsonHealingUndefined(literal);
+    if (parsed !== undefined) pushes.push(parsed);
   }
   return pushes;
 }
@@ -313,12 +342,12 @@ export function extractEmbeddedState(rawHtml) {
     const valueStart = assignment.index + assignment[0].length;
     const literal = readBracketedLiteral(html, valueStart);
     let parsed;
+    let healed = 0;
     if (literal !== null) {
-      try {
-        parsed = JSON.parse(literal);
-      } catch {
-        parsed = undefined;
-      }
+      // nytimes.com's __preloadedData is JSON except for bare `undefined`
+      // values (81 of them on the front page, 2026-09-04) — the same shape
+      // the Apollo transport heals, so heal it the same way.
+      ({ parsed, healed } = parseJsonHealingUndefined(literal));
     }
 
     if (parsed === undefined) {
@@ -341,7 +370,9 @@ export function extractEmbeddedState(rawHtml) {
     }
 
     data[name] = parsed;
-    found.push({ name, variable, bytes: serializedBytes(parsed) });
+    const entry = { name, variable, bytes: serializedBytes(parsed) };
+    if (healed > 0) entry.note = `${healed} bare undefined value(s) read as null`;
+    found.push(entry);
   }
 
   if (jsonScripts.length > 0) {

package/src/templates.js

@@ -273,7 +273,80 @@ function amazonByline($) {
   return raw;
 }
 
-/** "4.7 out of 5 stars" → 4.7 */
+/**
+ * The visible price. Amazon renders it twice: an offscreen string for screen
+ * readers and a whole/decimal/fraction triplet for the eye — and a product
+ * page carries dozens of such blocks that are not this product's price at
+ * all. Reading the first `.a-price` on the page returned, on amazon.de/.it/
+ * .co.jp book pages served without a buy box, the price of the first item in
+ * the "similar products" carousel ("52,13USD" for a book listed from 30,57
+ * USD), and on amazon.nl/.co.uk/.com.au it returned null because the buy
+ * box's offscreen span is a blank " " (all observed 2026-09-04). So the block
+ * is chosen by where it sits: the buy box first (Amazon's own priceToPay
+ * class, or the corePrice/apex containers), then any block outside the
+ * regions that always show OTHER products (carousels, bundles, the comparison
+ * table, the other-sellers link) or a struck-through list price, then the
+ * legacy priceblock ids, and last the selected format swatch — a book without
+ * a buy box shows "ab 30,57 USD" / "da 42,16 €" there, and the qualifier is
+ * kept because that is a from-price, not the price.
+ *
+ * Within a block: the offscreen string when it is there; when it is blank,
+ * the triplet is rebuilt as symbol + whole + separator + fraction, with the
+ * symbol on the side the markup puts it ("€44,85", "39.24£" never occurs but
+ * "32,89€" does). On amazon.com.au the a-price-decimal span can be EMPTY, so
+ * the offscreen string reads "$1105" for A$11.05 — whole "11" + fraction "05"
+ * with no separator, a price 100× too high that every downstream guard
+ * accepts. When the offscreen digits are exactly whole+fraction and nothing
+ * separates the fraction, the separator is put back: "." unless the
+ * marketplace writes its decimals with a comma.
+ */
+const AMAZON_OTHER_PRODUCT_PRICE =
+  '[id^="sims-"], [id^="sp_"], .a-carousel, [data-a-carousel-options], #HLCXComparisonTable, ' +
+  '#olpLinkWidget_feature_div, #dynamic-aod-ingress-box, [id*="sponsored"], [class*="fbt"], ' +
+  '#twister-plus-tool-tip, #twisterPlusPriceSubtotalWWDesktop_feature_div, .a-text-price';
+const AMAZON_BUY_BOX_PRICE =
+  '.priceToPay, .apexPriceToPay, #corePrice_feature_div *, #corePriceDisplay_desktop_feature_div *, ' +
+  '#apex_desktop *, #corePrice_desktop *';
+
+function amazonDecimalSeparator($) {
+  const host = (attr($, 'link[rel="canonical"]', 'href') || '').match(/^https?:\/\/([^/]+)/)?.[1] || '';
+  return /\.(de|fr|es|it|nl|se|pl|com\.br|com\.tr|com\.be)$/.test(host) ? ',' : '.';
+}
+
+function amazonBlockPrice($, block, separator) {
+  const $block = $(block);
+  const offscreen = tidy($block.find('.a-offscreen').first().text());
+  const whole = ($block.find('.a-price-whole').first().text() || '').replace(/\D/g, '');
+  const fraction = ($block.find('.a-price-fraction').first().text() || '').replace(/\D/g, '');
+  if (offscreen) {
+    if (!whole || !fraction) return offscreen;
+    if (offscreen.replace(/\D/g, '') !== whole + fraction) return offscreen;
+    if (new RegExp(`[.,]${fraction}(?!\\d)`).test(offscreen)) return offscreen;
+    return offscreen.replace(new RegExp(`(\\d)(${fraction})(?!\\d)`), `$1${separator}$2`);
+  }
+  if (!whole) return null;
+  const amount = fraction ? `${whole}${separator}${fraction}` : whole;
+  const symbol = tidy($block.find('.a-price-symbol').first().text()) || '';
+  const symbolFirst = $block.find('.a-price-symbol, .a-price-whole').first().hasClass('a-price-symbol');
+  return symbolFirst ? `${symbol}${amount}` : `${amount}${symbol}`;
+}
+
+function amazonPrice($) {
+  const separator = amazonDecimalSeparator($);
+  const own = $('.a-price').filter((_, el) => $(el).closest(AMAZON_OTHER_PRODUCT_PRICE).length === 0);
+  for (const pool of [own.filter(AMAZON_BUY_BOX_PRICE), own]) {
+    for (const block of pool.toArray()) {
+      const price = amazonBlockPrice($, block, separator);
+      if (price) return price;
+    }
+  }
+  return (
+    text($, '#priceblock_ourprice') ||
+    text($, '#priceblock_dealprice') ||
+    tidy(text($, '#tmmSwatches .swatchElement.selected .slot-price'))
+  );
+}
+
 /**
  * ISO 4217 code for an Amazon price string. The add-to-cart form's hidden
  * currencyCode input is absent on pages Amazon serves without a buy box
@@ -303,9 +376,28 @@ function amazonCurrency($, price) {
   return null;
 }
 
+function hnCommentCount(label) {
+  const value = (label || '').trim();
+  if (!value) return null;
+  if (/^discuss$/i.test(value)) return '0';
+  const match = value.match(/^(\d+)\s*comments?$/i);
+  return match ? match[1] : value;
+}
+
+/**
+ * "4.7 out of 5 stars" → 4.7. The first number is not the rating everywhere:
+ * amazon.nl writes "4,8 van 5 sterren" (a comma decimal, read as 4) and
+ * amazon.co.jp "5つ星のうち4.7" (the scale comes first, read as 5) — both
+ * plausible wrong numbers, observed 2026-09-04. Every number is read, comma
+ * decimals included, and when one of two is the 5-star scale the other one is
+ * the rating.
+ */
 function amazonRating(value) {
-  const match = tidy(value)?.match(/([\d.]+)/);
-  return match ? Number.parseFloat(match[1]) : null;
+  const numbers = (tidy(value)?.match(/\d+(?:[.,]\d+)?/g) || [])
+    .map((n) => Number.parseFloat(n.replace(',', '.')));
+  if (numbers.length === 0) return null;
+  if (numbers.length === 1) return numbers[0];
+  return numbers.find((n) => n !== 5) ?? 5;
 }
 
 /** Both "(198,594)" and "198,594 global ratings" mean 198594. */
@@ -571,7 +663,10 @@ export const TEMPLATES = [
     id: 'amazon-product',
     name: 'Amazon Product',
     description: 'Scrape an Amazon product page for title, price, rating, reviews, ASIN, and description.',
-    targetPattern: /amazon\.(com|co\.uk|de|fr|jp|ca|com\.au)/i,
+    // Every marketplace the extractor handles. "jp" alone never matched
+    // amazon.co.jp, and .es/.in/.it worked by explicit id while template:"auto"
+    // refused them (R15, 2026-09-04).
+    targetPattern: /amazon\.(com|co\.uk|co\.jp|de|fr|es|it|nl|se|pl|ca|in|sg|ae|sa|eg|com\.au|com\.br|com\.mx|com\.be|com\.tr)/i,
     extract($) {
       // Amazon serves its robot check as HTTP 200: a "Continue shopping" page
       // whose only form posts to /errors/validateCaptcha. Every selector below
@@ -591,7 +686,7 @@ export const TEMPLATES = [
         .map(fullSizeImage)
         .filter(Boolean);
 
-      const price = text($, '.a-price .a-offscreen') || text($, '#priceblock_ourprice') || text($, '#priceblock_dealprice');
+      const price = amazonPrice($);
 
       return {
         title: tidy(text($, '#productTitle')),
@@ -749,7 +844,9 @@ export const TEMPLATES = [
     id: 'hacker-news-front-page',
     name: 'Hacker News Front Page',
     description: 'Scrape the Hacker News front page for a list of stories with title, URL, score, and comment count.',
-    targetPattern: /news\.ycombinator\.com(\/news)?\/?$/i,
+    // The same story table serves /newest, /front, /best, /ask, /show, /jobs
+    // and /active, and every one of them pages with ?p=N.
+    targetPattern: /news\.ycombinator\.com(?:\/(?:news|newest|front|best|ask|show|jobs|active))?\/?(?:[?#].*)?$/i,
     extract($) {
       const stories = [];
       $('tr.athing').each((_, el) => {
@@ -769,8 +866,10 @@ export const TEMPLATES = [
           // ".age a" wraps the relative age string ("3 hours ago"); its href is the item permalink.
           posted: $subtext.find('.age a').text().trim() || null,
           // The comments link is also an item?id= link, so exclude the age anchor.
-          // Job posts have no comments link at all -> null.
-          comments: $subtext.find('a[href*="item"]').not('.age a').last().text().trim() || null
+          // "1053 comments", "1 comment" or "discuss" (none yet) become one
+          // shape, a bare count, like score above. Job posts have no comments
+          // link at all -> null.
+          comments: hnCommentCount($subtext.find('a[href*="item"]').not('.age a').last().text())
         });
       });
       return { stories: stories.slice(0, 30), scraped_at: new Date().toISOString() };