crawlforge-extractors 1.6.0 → 1.6.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.6.0",
+  "version": "1.6.2",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/embeddedState.js

@@ -23,7 +23,11 @@ const STATE_VARIABLES = [
   { name: 'nuxt', variable: '__NUXT__' },
   { name: 'apollo_state', variable: '__APOLLO_STATE__' },
   { name: 'initial_state', variable: '__INITIAL_STATE__' },
-  { name: 'preloaded_state', variable: '__PRELOADED_STATE__' }
+  { name: 'preloaded_state', variable: '__PRELOADED_STATE__' },
+  // nytimes.com ships its whole front page as window.__preloadedData; with
+  // only the four names above, a 1.1 MB page surfaced nothing but its
+  // <script type="application/json"> blocks (R15, 2026-09-04).
+  { name: 'preloaded_data', variable: '__preloadedData' }
 ];
 
 // Script bodies cannot contain a literal "</script", so a non-greedy match is
@@ -43,6 +47,33 @@ const NEXT_F_PUSH_RE = /self\.__next_f\.push\(\s*\[\s*1\s*,\s*/g;
 const APOLLO_TRANSPORT_RE =
   /Symbol\.for\(\s*["']ApolloSSRDataTransport["']\s*\)\s*\]\s*\?\?=\s*\[\s*\]\s*\)\s*\.push\(\s*/g;
 
+// Bare `undefined` is the one JS-only token serializers emit inside otherwise
+// valid JSON (Apollo for still-streaming fields, nytimes' __preloadedData).
+// The lookbehind/lookahead pin it to a value position, so a string containing
+// the word is left alone.
+const BARE_UNDEFINED_RE = /(?<=[:,[])\s*undefined\s*(?=[,}\]])/g;
+
+/**
+ * JSON.parse a literal, healing bare `undefined` values to null when a
+ * strict parse fails. Returns parsed:undefined when the literal is not JSON
+ * even after healing.
+ * @param {string} literal
+ * @returns {{ parsed: unknown, healed: number }}
+ */
+function parseJsonHealingUndefined(literal) {
+  try {
+    return { parsed: JSON.parse(literal), healed: 0 };
+  } catch {
+    const healed = (literal.match(BARE_UNDEFINED_RE) || []).length;
+    if (healed === 0) return { parsed: undefined, healed: 0 };
+    try {
+      return { parsed: JSON.parse(literal.replace(BARE_UNDEFINED_RE, 'null')), healed };
+    } catch {
+      return { parsed: undefined, healed: 0 };
+    }
+  }
+}
+
 /**
  * Read an HTML attribute out of a raw tag's attribute string.
  * @param {string} attrs
@@ -219,11 +250,9 @@ export function extractApolloTransport(html) {
     const literal = readBracketedLiteral(html, APOLLO_TRANSPORT_RE.lastIndex);
     if (!literal) continue;
     APOLLO_TRANSPORT_RE.lastIndex += literal.length;
-    try {
-      pushes.push(JSON.parse(literal.replace(/(?<=[:,[])\s*undefined\s*(?=[,}\]])/g, 'null')));
-    } catch {
-      // Not JSON we can heal — skip this push, keep the rest.
-    }
+    // Not JSON we can heal — skip this push, keep the rest.
+    const { parsed } = parseJsonHealingUndefined(literal);
+    if (parsed !== undefined) pushes.push(parsed);
   }
   return pushes;
 }
@@ -313,12 +342,12 @@ export function extractEmbeddedState(rawHtml) {
     const valueStart = assignment.index + assignment[0].length;
     const literal = readBracketedLiteral(html, valueStart);
     let parsed;
+    let healed = 0;
     if (literal !== null) {
-      try {
-        parsed = JSON.parse(literal);
-      } catch {
-        parsed = undefined;
-      }
+      // nytimes.com's __preloadedData is JSON except for bare `undefined`
+      // values (81 of them on the front page, 2026-09-04) — the same shape
+      // the Apollo transport heals, so heal it the same way.
+      ({ parsed, healed } = parseJsonHealingUndefined(literal));
     }
 
     if (parsed === undefined) {
@@ -341,7 +370,9 @@ export function extractEmbeddedState(rawHtml) {
     }
 
     data[name] = parsed;
-    found.push({ name, variable, bytes: serializedBytes(parsed) });
+    const entry = { name, variable, bytes: serializedBytes(parsed) };
+    if (healed > 0) entry.note = `${healed} bare undefined value(s) read as null`;
+    found.push(entry);
   }
 
   if (jsonScripts.length > 0) {

package/src/templates.js

@@ -274,6 +274,66 @@ function amazonByline($) {
 }
 
 /** "4.7 out of 5 stars" → 4.7 */
+/**
+ * ISO 4217 code for an Amazon price string. The add-to-cart form's hidden
+ * currencyCode input is absent on pages Amazon serves without a buy box
+ * (amazon.de and amazon.in book pages, R14 2026-09-03), so the code is read
+ * off the price itself: an explicit code ("52,02USD"), else the symbol. A
+ * bare "$" belongs to several marketplaces, told apart by the canonical host.
+ */
+/**
+ * The visible price. Amazon renders it twice: an offscreen string for screen
+ * readers and a whole/decimal/fraction triplet for the eye. On amazon.com.au
+ * (observed 2026-09-04) the a-price-decimal span is EMPTY, so the offscreen
+ * string reads "$1105" for A$11.05 — whole "11" + fraction "05" with no
+ * separator, a price 100× too high that every downstream guard accepts. When
+ * the offscreen digits are exactly whole+fraction and nothing separates the
+ * fraction, the separator is put back: "." unless the marketplace writes its
+ * decimals with a comma.
+ */
+function amazonPrice($) {
+  const offscreen = text($, '.a-price .a-offscreen');
+  if (!offscreen) return null;
+  const whole = (text($, '.a-price .a-price-whole') || '').replace(/\D/g, '');
+  const fraction = (text($, '.a-price .a-price-fraction') || '').replace(/\D/g, '');
+  if (!whole || !fraction) return offscreen;
+  if (offscreen.replace(/\D/g, '') !== whole + fraction) return offscreen;
+  if (new RegExp(`[.,]${fraction}(?!\\d)`).test(offscreen)) return offscreen;
+  const host = (attr($, 'link[rel="canonical"]', 'href') || '').match(/^https?:\/\/([^/]+)/)?.[1] || '';
+  const separator = /\.(de|fr|es|it|nl|se|pl|com\.br|com\.tr|com\.be)$/.test(host) ? ',' : '.';
+  return offscreen.replace(new RegExp(`(\\d)(${fraction})(?!\\d)`), `$1${separator}$2`);
+}
+
+function amazonCurrency($, price) {
+  if (!price) return null;
+  const code = price.match(/(USD|EUR|GBP|INR|JPY|CAD|AUD|MXN|BRL|SGD|AED|SAR|PLN|TRY)(?![A-Z])/);
+  if (code) return code[1];
+  if (price.includes('₹')) return 'INR';
+  if (price.includes('€')) return 'EUR';
+  if (price.includes('£')) return 'GBP';
+  if (price.includes('¥') || price.includes('￥')) return 'JPY';
+  if (price.includes('R$')) return 'BRL';
+  if (price.includes('zł')) return 'PLN';
+  if (price.includes('₺')) return 'TRY';
+  if (price.includes('$')) {
+    const host = (attr($, 'link[rel="canonical"]', 'href') || '').match(/^https?:\/\/([^/]+)/)?.[1] || '';
+    if (/\.ca$/.test(host)) return 'CAD';
+    if (/\.com\.au$/.test(host)) return 'AUD';
+    if (/\.com\.mx$/.test(host)) return 'MXN';
+    if (/\.sg$/.test(host)) return 'SGD';
+    return 'USD';
+  }
+  return null;
+}
+
+function hnCommentCount(label) {
+  const value = (label || '').trim();
+  if (!value) return null;
+  if (/^discuss$/i.test(value)) return '0';
+  const match = value.match(/^(\d+)\s*comments?$/i);
+  return match ? match[1] : value;
+}
+
 function amazonRating(value) {
   const match = tidy(value)?.match(/([\d.]+)/);
   return match ? Number.parseFloat(match[1]) : null;
@@ -542,7 +602,10 @@ export const TEMPLATES = [
     id: 'amazon-product',
     name: 'Amazon Product',
     description: 'Scrape an Amazon product page for title, price, rating, reviews, ASIN, and description.',
-    targetPattern: /amazon\.(com|co\.uk|de|fr|jp|ca|com\.au)/i,
+    // Every marketplace the extractor handles. "jp" alone never matched
+    // amazon.co.jp, and .es/.in/.it worked by explicit id while template:"auto"
+    // refused them (R15, 2026-09-04).
+    targetPattern: /amazon\.(com|co\.uk|co\.jp|de|fr|es|it|nl|se|pl|ca|in|sg|ae|sa|eg|com\.au|com\.br|com\.mx|com\.be|com\.tr)/i,
     extract($) {
       // Amazon serves its robot check as HTTP 200: a "Continue shopping" page
       // whose only form posts to /errors/validateCaptcha. Every selector below
@@ -562,12 +625,17 @@ export const TEMPLATES = [
         .map(fullSizeImage)
         .filter(Boolean);
 
+      const price = amazonPrice($) || text($, '#priceblock_ourprice') || text($, '#priceblock_dealprice');
+
       return {
         title: tidy(text($, '#productTitle')),
-        price: text($, '.a-price .a-offscreen') || text($, '#priceblock_ourprice') || text($, '#priceblock_dealprice'),
+        price,
         // Amazon ships no priceCurrency meta tag — the ISO code is a hidden
-        // field on the add-to-cart form.
-        currency: attr($, 'input[name*="currencyCode"]', 'value') || attr($, 'meta[itemprop="priceCurrency"]', 'content'),
+        // field on the add-to-cart form, and off the price where there is none.
+        currency:
+          attr($, 'input[name*="currencyCode"]', 'value') ||
+          attr($, 'meta[itemprop="priceCurrency"]', 'content') ||
+          amazonCurrency($, price),
         rating: amazonRating(attr($, '#acrPopover', 'title') || text($, '#averageCustomerReviews .a-icon-alt')),
         review_count: amazonCount(text($, '#acrCustomerReviewText') || text($, '[data-hook="total-review-count"]')),
         asin: text($, 'input#ASIN') || attr($, 'input[name="ASIN"]', 'value'),
@@ -729,13 +797,16 @@ export const TEMPLATES = [
           title: $titleLink.text().trim(),
           url: safeHref($titleLink.attr('href')),
           site: $row.find('.sitebit a').text().trim() || null,
-          score: $score.text().replace(' points', '').trim() || null,
+          // "1 point" on a fresh story and "3 points" on the rest — strip both.
+          score: $score.text().replace(/\s*points?$/, '').trim() || null,
           author: $subtext.find('.hnuser').text().trim() || null,
           // ".age a" wraps the relative age string ("3 hours ago"); its href is the item permalink.
           posted: $subtext.find('.age a').text().trim() || null,
           // The comments link is also an item?id= link, so exclude the age anchor.
-          // Job posts have no comments link at all -> null.
-          comments: $subtext.find('a[href*="item"]').not('.age a').last().text().trim() || null
+          // "1053 comments", "1 comment" or "discuss" (none yet) become one
+          // shape, a bare count, like score above. Job posts have no comments
+          // link at all -> null.
+          comments: hnCommentCount($subtext.find('a[href*="item"]').not('.age a').last().text())
         });
       });
       return { stories: stories.slice(0, 30), scraped_at: new Date().toISOString() };