npm - crawlforge-extractors - Versions diffs - 1.6.2 → 1.6.4 - Mend

crawlforge-extractors 1.6.2 → 1.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (4) hide show

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.6.2",
+  "version": "1.6.4",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/body.js CHANGED Viewed

@@ -26,6 +26,8 @@ export class BodyTooLargeError extends Error {
  * @param {Uint8Array} bytes
  * @returns {string}
  */
+export const META_CHARSET_SNIFF_BYTES = 8192;
 export function detectCharset(response, bytes) {
   const contentType = response.headers?.get?.('content-type') || '';
   const headerMatch = /charset=["']?([\w-]+)/i.exec(contentType);
@@ -33,10 +35,14 @@ export function detectCharset(response, bytes) {
     return headerMatch[1].trim().toLowerCase();
   }
-  // <meta charset> tags must appear within the first 1024 bytes per the
-  // HTML5 spec's prescan algorithm; ASCII-range bytes decode identically
-  // under latin1 regardless of the document's real encoding.
-  const sniffLength = Math.min(bytes.byteLength, 1024);
+  // The HTML5 prescan algorithm requires a <meta charset> within the first
+  // 1024 bytes, but real pages break the rule: vector.co.jp/magazine/softnews
+  // (Shift_JIS, no charset in the Content-Type header) declares it at byte
+  // 1293, behind a comment block, and every browser still decodes it
+  // correctly. Sniff a full 8 KB, which covers every <head> seen in the
+  // wild without decoding the whole body twice. ASCII-range bytes decode
+  // identically under latin1 regardless of the document's real encoding.
+  const sniffLength = Math.min(bytes.byteLength, META_CHARSET_SNIFF_BYTES);
   const sniffText = new TextDecoder('latin1').decode(bytes.subarray(0, sniffLength));
   const metaMatch =
     /<meta[^>]+charset=["']?([\w-]+)/i.exec(sniffText) ||

package/src/embeddedState.js CHANGED Viewed

@@ -23,6 +23,10 @@ const STATE_VARIABLES = [
   { name: 'nuxt', variable: '__NUXT__' },
   { name: 'apollo_state', variable: '__APOLLO_STATE__' },
   { name: 'initial_state', variable: '__INITIAL_STATE__' },
+  // tumblr.com spells it with three underscores and assigns it in bracket
+  // notation: window['___INITIAL_STATE___'] = {...} (R17, 2026-09-04). Same
+  // key as the two-underscore form; the first one found on a page wins.
+  { name: 'initial_state', variable: '___INITIAL_STATE___' },
   { name: 'preloaded_state', variable: '__PRELOADED_STATE__' },
   // nytimes.com ships its whole front page as window.__preloadedData; with
   // only the four names above, a 1.1 MB page surfaced nothing but its
@@ -334,8 +338,13 @@ export function extractEmbeddedState(rawHtml) {
   }
   for (const { name, variable } of STATE_VARIABLES) {
+    if (data[name] !== undefined) continue;
+    // Dot or bracket notation on window/self/globalThis, or a bare/var
+    // assignment; \b keeps MY__INITIAL_STATE__ from matching __INITIAL_STATE__.
     const assignment = html.match(
-      new RegExp(`(?:window|self|globalThis)?\\.?\\b${variable}\\s*=\\s*`)
+      new RegExp(
+        `(?:(?:window|self|globalThis)\\s*\\[\\s*(['"])${variable}\\1\\s*\\]|(?:(?:window|self|globalThis)\\.)?\\b${variable})\\s*=\\s*`
+      )
     );
     if (!assignment) continue;

package/src/templates.js CHANGED Viewed

@@ -270,10 +270,96 @@ function amazonByline($) {
   // continues into "(Author) Format: Hardcover".
   if (/^by\s/i.test(raw)) return contributor || tidy(raw.replace(/^by\s+/i, '').split('(')[0]);
+  // Every other marketplace phrases the book byline in its own language —
+  // "Engelska utgåvan av George Orwell (Författare)", "Wydanie: Angielski
+  // George Orwell (Autor)", "Édition en Anglais de George Orwell (Auteur)" —
+  // and none starts with "by", so the chrome came back whole on amazon.se,
+  // .pl, .com.be and .com.tr (R17, 2026-09-04). The author's own link
+  // carries the bare name on every marketplace.
+  const authorLink = tidy($('#bylineInfo .author a, #bylineInfo a.contributorNameID').first().text());
+  if (authorLink && /\(/.test(raw)) return authorLink;
+  // "Marke: Sony", "Marca: Sony", "Marque : Sony" — a localised brand label.
+  const labelled = raw.match(/^[\p{L}\s]{2,20}?\s?:\s*(.+)$/u);
+  if (labelled && !/\(/.test(raw)) return tidy(labelled[1]);
   return raw;
 }
-/** "4.7 out of 5 stars" → 4.7 */
+/**
+ * The visible price. Amazon renders it twice: an offscreen string for screen
+ * readers and a whole/decimal/fraction triplet for the eye — and a product
+ * page carries dozens of such blocks that are not this product's price at
+ * all. Reading the first `.a-price` on the page returned, on amazon.de/.it/
+ * .co.jp book pages served without a buy box, the price of the first item in
+ * the "similar products" carousel ("52,13USD" for a book listed from 30,57
+ * USD), and on amazon.nl/.co.uk/.com.au it returned null because the buy
+ * box's offscreen span is a blank " " (all observed 2026-09-04). So the block
+ * is chosen by where it sits: the buy box first (Amazon's own priceToPay
+ * class, or the corePrice/apex containers), then any block outside the
+ * regions that always show OTHER products (carousels, bundles, the comparison
+ * table, the other-sellers link) or a struck-through list price, then the
+ * legacy priceblock ids, and last the selected format swatch — a book without
+ * a buy box shows "ab 30,57 USD" / "da 42,16 €" there, and the qualifier is
+ * kept because that is a from-price, not the price.
+ *
+ * Within a block: the offscreen string when it is there; when it is blank,
+ * the triplet is rebuilt as symbol + whole + separator + fraction, with the
+ * symbol on the side the markup puts it ("€44,85", "39.24£" never occurs but
+ * "32,89€" does). On amazon.com.au the a-price-decimal span can be EMPTY, so
+ * the offscreen string reads "$1105" for A$11.05 — whole "11" + fraction "05"
+ * with no separator, a price 100× too high that every downstream guard
+ * accepts. When the offscreen digits are exactly whole+fraction and nothing
+ * separates the fraction, the separator is put back: "." unless the
+ * marketplace writes its decimals with a comma.
+ */
+const AMAZON_OTHER_PRODUCT_PRICE =
+  '[id^="sims-"], [id^="sp_"], .a-carousel, [data-a-carousel-options], #HLCXComparisonTable, ' +
+  '#olpLinkWidget_feature_div, #dynamic-aod-ingress-box, [id*="sponsored"], [class*="fbt"], ' +
+  '#twister-plus-tool-tip, #twisterPlusPriceSubtotalWWDesktop_feature_div, .a-text-price';
+const AMAZON_BUY_BOX_PRICE =
+  '.priceToPay, .apexPriceToPay, #corePrice_feature_div *, #corePriceDisplay_desktop_feature_div *, ' +
+  '#apex_desktop *, #corePrice_desktop *';
+function amazonDecimalSeparator($) {
+  const host = (attr($, 'link[rel="canonical"]', 'href') || '').match(/^https?:\/\/([^/]+)/)?.[1] || '';
+  return /\.(de|fr|es|it|nl|se|pl|com\.br|com\.tr|com\.be)$/.test(host) ? ',' : '.';
+}
+function amazonBlockPrice($, block, separator) {
+  const $block = $(block);
+  const offscreen = tidy($block.find('.a-offscreen').first().text());
+  const whole = ($block.find('.a-price-whole').first().text() || '').replace(/\D/g, '');
+  const fraction = ($block.find('.a-price-fraction').first().text() || '').replace(/\D/g, '');
+  if (offscreen) {
+    if (!whole || !fraction) return offscreen;
+    if (offscreen.replace(/\D/g, '') !== whole + fraction) return offscreen;
+    if (new RegExp(`[.,]${fraction}(?!\\d)`).test(offscreen)) return offscreen;
+    return offscreen.replace(new RegExp(`(\\d)(${fraction})(?!\\d)`), `$1${separator}$2`);
+  }
+  if (!whole) return null;
+  const amount = fraction ? `${whole}${separator}${fraction}` : whole;
+  const symbol = tidy($block.find('.a-price-symbol').first().text()) || '';
+  const symbolFirst = $block.find('.a-price-symbol, .a-price-whole').first().hasClass('a-price-symbol');
+  return symbolFirst ? `${symbol}${amount}` : `${amount}${symbol}`;
+}
+function amazonPrice($) {
+  const separator = amazonDecimalSeparator($);
+  const own = $('.a-price').filter((_, el) => $(el).closest(AMAZON_OTHER_PRODUCT_PRICE).length === 0);
+  for (const pool of [own.filter(AMAZON_BUY_BOX_PRICE), own]) {
+    for (const block of pool.toArray()) {
+      const price = amazonBlockPrice($, block, separator);
+      if (price) return price;
+    }
+  }
+  return (
+    text($, '#priceblock_ourprice') ||
+    text($, '#priceblock_dealprice') ||
+    tidy(text($, '#tmmSwatches .swatchElement.selected .slot-price'))
+  );
+}
 /**
  * ISO 4217 code for an Amazon price string. The add-to-cart form's hidden
  * currencyCode input is absent on pages Amazon serves without a buy box
@@ -281,32 +367,9 @@ function amazonByline($) {
  * off the price itself: an explicit code ("52,02USD"), else the symbol. A
  * bare "$" belongs to several marketplaces, told apart by the canonical host.
  */
-/**
- * The visible price. Amazon renders it twice: an offscreen string for screen
- * readers and a whole/decimal/fraction triplet for the eye. On amazon.com.au
- * (observed 2026-09-04) the a-price-decimal span is EMPTY, so the offscreen
- * string reads "$1105" for A$11.05 — whole "11" + fraction "05" with no
- * separator, a price 100× too high that every downstream guard accepts. When
- * the offscreen digits are exactly whole+fraction and nothing separates the
- * fraction, the separator is put back: "." unless the marketplace writes its
- * decimals with a comma.
- */
-function amazonPrice($) {
-  const offscreen = text($, '.a-price .a-offscreen');
-  if (!offscreen) return null;
-  const whole = (text($, '.a-price .a-price-whole') || '').replace(/\D/g, '');
-  const fraction = (text($, '.a-price .a-price-fraction') || '').replace(/\D/g, '');
-  if (!whole || !fraction) return offscreen;
-  if (offscreen.replace(/\D/g, '') !== whole + fraction) return offscreen;
-  if (new RegExp(`[.,]${fraction}(?!\\d)`).test(offscreen)) return offscreen;
-  const host = (attr($, 'link[rel="canonical"]', 'href') || '').match(/^https?:\/\/([^/]+)/)?.[1] || '';
-  const separator = /\.(de|fr|es|it|nl|se|pl|com\.br|com\.tr|com\.be)$/.test(host) ? ',' : '.';
-  return offscreen.replace(new RegExp(`(\\d)(${fraction})(?!\\d)`), `$1${separator}$2`);
-}
 function amazonCurrency($, price) {
   if (!price) return null;
-  const code = price.match(/(USD|EUR|GBP|INR|JPY|CAD|AUD|MXN|BRL|SGD|AED|SAR|PLN|TRY)(?![A-Z])/);
+  const code = price.match(/(USD|EUR|GBP|INR|JPY|CAD|AUD|MXN|BRL|SGD|AED|SAR|EGP|PLN|TRY|SEK)(?![A-Z])/);
   if (code) return code[1];
   if (price.includes('₹')) return 'INR';
   if (price.includes('€')) return 'EUR';
@@ -314,9 +377,16 @@ function amazonCurrency($, price) {
   if (price.includes('¥') || price.includes('￥')) return 'JPY';
   if (price.includes('R$')) return 'BRL';
   if (price.includes('zł')) return 'PLN';
-  if (price.includes('₺')) return 'TRY';
+  // amazon.com.tr writes "460,67TL" (R17, 2026-09-04); the lira sign is rarer.
+  if (price.includes('₺') || /(?<![A-Za-z])TL(?![A-Za-z])/.test(price)) return 'TRY';
+  // Arabic-script marketplaces: dirham, Egyptian pound, riyal.
+  if (/د\.?إ/.test(price)) return 'AED';
+  if (/ج\.?م/.test(price)) return 'EGP';
+  if (/ر\.?س|﷼/.test(price)) return 'SAR';
+  const host = (attr($, 'link[rel="canonical"]', 'href') || '').match(/^https?:\/\/([^/]+)/)?.[1] || '';
+  // "114,30kr" on amazon.se — the only Amazon marketplace priced in kronor.
+  if (/(?<![A-Za-z])kr(?![A-Za-z])/i.test(price)) return /\.se$/.test(host) ? 'SEK' : null;
   if (price.includes('$')) {
-    const host = (attr($, 'link[rel="canonical"]', 'href') || '').match(/^https?:\/\/([^/]+)/)?.[1] || '';
     if (/\.ca$/.test(host)) return 'CAD';
     if (/\.com\.au$/.test(host)) return 'AUD';
     if (/\.com\.mx$/.test(host)) return 'MXN';
@@ -326,6 +396,15 @@ function amazonCurrency($, price) {
   return null;
 }
+function hnAbsoluteUrl(href) {
+  if (!href) return href;
+  try {
+    return new URL(href, 'https://news.ycombinator.com/').href;
+  } catch {
+    return href;
+  }
+}
 function hnCommentCount(label) {
   const value = (label || '').trim();
   if (!value) return null;
@@ -334,9 +413,20 @@ function hnCommentCount(label) {
   return match ? match[1] : value;
 }
+/**
+ * "4.7 out of 5 stars" → 4.7. The first number is not the rating everywhere:
+ * amazon.nl writes "4,8 van 5 sterren" (a comma decimal, read as 4) and
+ * amazon.co.jp "5つ星のうち4.7" (the scale comes first, read as 5) — both
+ * plausible wrong numbers, observed 2026-09-04. Every number is read, comma
+ * decimals included, and when one of two is the 5-star scale the other one is
+ * the rating.
+ */
 function amazonRating(value) {
-  const match = tidy(value)?.match(/([\d.]+)/);
-  return match ? Number.parseFloat(match[1]) : null;
+  const numbers = (tidy(value)?.match(/\d+(?:[.,]\d+)?/g) || [])
+    .map((n) => Number.parseFloat(n.replace(',', '.')));
+  if (numbers.length === 0) return null;
+  if (numbers.length === 1) return numbers[0];
+  return numbers.find((n) => n !== 5) ?? 5;
 }
 /** Both "(198,594)" and "198,594 global ratings" mean 198594. */
@@ -625,7 +715,7 @@ export const TEMPLATES = [
         .map(fullSizeImage)
         .filter(Boolean);
-      const price = amazonPrice($) || text($, '#priceblock_ourprice') || text($, '#priceblock_dealprice');
+      const price = amazonPrice($);
       return {
         title: tidy(text($, '#productTitle')),
@@ -783,7 +873,9 @@ export const TEMPLATES = [
     id: 'hacker-news-front-page',
     name: 'Hacker News Front Page',
     description: 'Scrape the Hacker News front page for a list of stories with title, URL, score, and comment count.',
-    targetPattern: /news\.ycombinator\.com(\/news)?\/?$/i,
+    // The same story table serves /newest, /front, /best, /ask, /show, /jobs
+    // and /active, and every one of them pages with ?p=N.
+    targetPattern: /news\.ycombinator\.com(?:\/(?:news|newest|front|best|ask|show|jobs|active))?\/?(?:[?#].*)?$/i,
     extract($) {
       const stories = [];
       $('tr.athing').each((_, el) => {
@@ -795,7 +887,9 @@ export const TEMPLATES = [
         stories.push({
           id: $row.attr('id'),
           title: $titleLink.text().trim(),
-          url: safeHref($titleLink.attr('href')),
+          // Text posts (Ask HN, Show HN without a link) carry a relative
+          // "item?id=…" href; resolve it so every story url is absolute.
+          url: safeHref(hnAbsoluteUrl($titleLink.attr('href'))),
           site: $row.find('.sitebit a').text().trim() || null,
           // "1 point" on a fresh story and "3 points" on the rest — strip both.
           score: $score.text().replace(/\s*points?$/, '').trim() || null,