crawlforge-extractors 1.4.0 → 1.5.0

package/README.md

@@ -168,9 +168,9 @@ calls `eval`.
 ## Templates
 
 **Pages and products.** `shopify-product` · `shopify-collection` ·
-`amazon-product` · `linkedin-profile` · `github-repo` · `youtube-video` ·
-`tweet` · `reddit-thread` · `hacker-news-front-page` · `producthunt-launch` ·
-`stackoverflow-question` · `npm-package`
+`amazon-product` · `github-repo` · `youtube-video` · `reddit-thread` ·
+`hacker-news-front-page` · `producthunt-launch` · `stackoverflow-question` ·
+`npm-package`
 
 **Job boards** (`src/connectors/ats.js`). `greenhouse-jobs` ·
 `lever-postings` · `ashby-jobs` · `workable-jobs` · `recruitee-offers` ·
@@ -197,6 +197,13 @@ large board past 4 MB. `lever-postings` declares `crawlDelaySeconds: 1`,
 which `api.lever.co/robots.txt` asks for and the calling surface's host rate
 limiter is expected to honour.
 
+`stackoverflow-question` reads the Stack Exchange API rather than the rendered
+page: stackoverflow.com answers every non-browser fetch (curl, node, and a
+browser User-Agent alike) with a Cloudflare 403, so the old selector extractor
+never saw a document. The API is keyless — 300 requests per day per IP — and
+one request carries the question, its owner and every answer; the template
+returns the accepted answer first, then by score, with bodies as plain text.
+
 `nhtsa-vin` decodes a VIN through the NHTSA vPIC API — the ~154 returned
 fields are curated into a named vehicle shape with the API's empty-string
 "not applicable" normalised to `null`, the full set kept under `raw`, and the
@@ -205,8 +212,22 @@ swallowed, because a partial decode is a real answer. `npi-provider` reads the
 CMS NPI Registry — a public professional registry — and passes its records
 through as published. Neither needs a key.
 
-`reddit-thread` is registered here but reddit.com blocks plain fetchers; the
-REST API steers those callers to its `reddit_search` tool instead.
+`reddit-thread` reads the Arctic Shift community archive
+(`arctic-shift.photon-reddit.com/api/posts/ids`) rather than reddit.com, which
+403s every non-browser client and disallows everything in robots.txt. One
+keyless request returns the post record — title, subreddit, author, score,
+upvote ratio, comment count, body, flair, removal state. The comment tree is
+a second endpoint and belongs to the calling surface's `reddit_search` tool,
+which the returned `id` feeds directly.
+
+`linkedin-profile` and `tweet` are **retired** (2026-08-30) and live in
+`RETIRED_TEMPLATES` with the reason: LinkedIn's robots.txt disallows every
+path for all agents but its own crawler and profiles sit behind an auth wall;
+X's robots.txt disallows every path for generic agents and its keyless embed
+endpoints (`cdn.syndication.twimg.com`, `publish.x.com/oembed`) are disallowed
+by their own robots.txt. `retiredTemplate(idOrUrl)` answers for either an id or
+a URL they handled, so a surface can return the reason instead of "unknown
+template".
 
 `smartrecruiters-postings` is deliberately **not** shipped: SmartRecruiters
 documents the endpoint publicly, but `api.smartrecruiters.com/robots.txt`

package/index.d.ts

@@ -100,6 +100,18 @@ export interface TemplateListResult {
 
 export declare const TEMPLATES: ScrapeTemplate[];
 
+/**
+ * Templates withdrawn because there is no compliant way to reach the data,
+ * by id: the URL shape each one handled, and why it is gone.
+ */
+export declare const RETIRED_TEMPLATES: Record<string, { targetPattern: RegExp; reason: string }>;
+
+/**
+ * The retired template a caller is reaching for — by id, or by a URL one of
+ * them handled — or null.
+ */
+export declare function retiredTemplate(idOrUrl: string): { id: string; reason: string } | null;
+
 export declare class TemplateRegistry {
   /** @param templates injectable, so a test can register a fixture template. */
   constructor(templates?: ScrapeTemplate[]);

package/index.js

@@ -14,7 +14,7 @@
  * caller has already issued rather than a URL.
  */
 
-export { TemplateRegistry, TEMPLATES } from './src/templates.js';
+export { TemplateRegistry, TEMPLATES, RETIRED_TEMPLATES, retiredTemplate } from './src/templates.js';
 export { TemplateRegistry as default } from './src/templates.js';
 
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/templates.js

@@ -115,6 +115,36 @@ function normalizeTags(tags) {
 }
 
 /** body_html is a rendered HTML fragment; callers want the copy, not the markup. */
+/** Stack Exchange timestamps are epoch seconds. */
+function epochToIso(seconds) {
+  return typeof seconds === 'number' && Number.isFinite(seconds)
+    ? new Date(seconds * 1000).toISOString()
+    : null;
+}
+
+/**
+ * Stack Exchange API filter created once via /2.3/filters/create with
+ * include=question.body;question.answers;answer.body;question.accepted_answer_id
+ * on base=default (filters are permanent and shareable per the API docs).
+ * The default base keeps the response wrapper (.items, .quota_remaining) and
+ * the standard question/answer fields; the includes add the bodies and the
+ * nested answers so one request carries the whole thread.
+ */
+const STACKEXCHANGE_FILTER = '!20aKG._8Oscv*6djs8Pgm';
+
+/**
+ * The community Reddit archive reddit-thread reads. reddit.com answers every
+ * non-browser client with a 403 (IP/TLS-reputation based, stealth browsers
+ * included) and its robots.txt disallows everything; the archive's robots.txt
+ * allows all agents and its /api/posts/ids answers keyless.
+ */
+const ARCTIC_SHIFT_BASE = 'https://arctic-shift.photon-reddit.com';
+
+/** The base36 post id in a reddit.com post URL, or null when there is none. */
+function redditPostId(url) {
+  return /\/comments\/([a-z0-9]+)/i.exec(url)?.[1] ?? null;
+}
+
 function htmlToText(html) {
   if (!html) return null;
   const text = load(`<div>${html}</div>`)('div').text().replace(/\s+/g, ' ').trim();
@@ -544,24 +574,6 @@ export const TEMPLATES = [
     }
   },
 
-  {
-    id: 'linkedin-profile',
-    name: 'LinkedIn Profile',
-    description: 'Scrape a LinkedIn public profile for name, headline, location, and about section.',
-    targetPattern: /linkedin\.com\/in\//i,
-    extract($) {
-      return {
-        name: text($, 'h1') || text($, '.top-card-layout__title'),
-        headline: text($, '.top-card-layout__headline') || text($, 'h2'),
-        location: text($, '.top-card-layout__first-subline') || text($, '.profile-info-subheader'),
-        about: text($, '.core-section-container__content p') || text($, '.summary'),
-        connections: text($, '.top-card__connections'),
-        current_company: text($, '.top-card-layout__card-inner-full-width .top-card-link'),
-        note: 'LinkedIn requires authentication for full profiles. This template works on public profile pages only.'
-      };
-    }
-  },
-
   {
     id: 'github-repo',
     name: 'GitHub Repository',
@@ -629,37 +641,58 @@ export const TEMPLATES = [
     }
   },
 
-  {
-    id: 'tweet',
-    name: 'Tweet / X Post',
-    description: 'Scrape a tweet/X post for text, author, timestamp, likes, and retweets from the Open Graph / structured data.',
-    targetPattern: /(twitter|x)\.com\/[^/]+\/status\//i,
-    extract($) {
-      return {
-        text: attr($, 'meta[property="og:description"]', 'content'),
-        author: attr($, 'meta[property="og:title"]', 'content'),
-        url: attr($, 'meta[property="og:url"]', 'content') || attr($, 'link[rel="canonical"]', 'href'),
-        image: attr($, 'meta[property="og:image"]', 'content'),
-        note: 'X.com requires JavaScript rendering for full tweet data. Structured metadata is returned from static HTML.'
-      };
-    }
-  },
-
   {
     id: 'reddit-thread',
     name: 'Reddit Thread',
-    description: 'Scrape a Reddit thread for title, subreddit, score, comment count, author, and top-level comments.',
-    targetPattern: /reddit\.com\/r\/[^/]+\/comments\//i,
-    extract($) {
+    description: 'Read a Reddit post — title, subreddit, author, score, upvote ratio, comment count, body and flair — from the Arctic Shift archive, since reddit.com blocks plain fetchers. For the comment tree, pass the returned id to the reddit_search tool in thread mode.',
+    targetPattern: /reddit\.com\/(?:r\/[^/]+\/)?comments\/[a-z0-9]+/i,
+
+    resolveUrl(url) {
+      const id = redditPostId(url);
+      if (!id) return url;
+      return `${ARCTIC_SHIFT_BASE}/api/posts/ids?ids=${id}`;
+    },
+
+    extractRaw(body, url) {
+      let doc;
+      try {
+        doc = JSON.parse(body);
+      } catch {
+        throw new Error(
+          `Not an Arctic Shift document: ${url} did not return JSON. ` +
+          'This template reads the Arctic Shift archive, not reddit.com.'
+        );
+      }
+      // The archive reports a bad request as {data:null, error:"..."}; a post
+      // it has never captured is an empty data list.
+      if (doc && doc.error) {
+        throw new Error(`Arctic Shift error: ${doc.error}`);
+      }
+      const post = Array.isArray(doc?.data) ? doc.data[0] : null;
+      if (!post) {
+        throw new Error(`No Reddit post at ${url}: the Arctic Shift archive has no record of it.`);
+      }
+
       return {
-        title: attr($, 'meta[property="og:title"]', 'content') || text($, 'h1'),
-        subreddit: text($, 'a[href*="/r/"][class*="subreddit"]') || (($('title').text().match(/r\/([^•]+)/) || [])[1] || '').trim(),
-        score: text($, '[data-score]') || attr($, '[itemprop="upvoteCount"]', 'content'),
-        author: text($, 'a[href*="/user/"]'),
-        posted: attr($, 'time[datetime]', 'datetime'),
-        body: text($, 'div[data-click-id="text"] p') || attr($, 'meta[property="og:description"]', 'content'),
-        url: attr($, 'meta[property="og:url"]', 'content'),
-        flair: text($, '[class*="flair"]')
+        id: post.id ?? null,
+        title: post.title ?? null,
+        subreddit: post.subreddit ?? null,
+        author: post.author ?? null,
+        score: typeof post.score === 'number' ? post.score : null,
+        upvote_ratio: typeof post.upvote_ratio === 'number' ? post.upvote_ratio : null,
+        num_comments: typeof post.num_comments === 'number' ? post.num_comments : null,
+        posted: epochToIso(post.created_utc),
+        // "[removed]" and "[deleted]" come back as written: that is what the
+        // archive holds, and a caller can tell it from an empty post.
+        body: post.selftext || null,
+        // A link post carries its external URL here; a self post carries its
+        // own permalink, which `url` already reports.
+        link_url: post.is_self ? null : (post.url || null),
+        url: post.permalink ? `https://www.reddit.com${post.permalink}` : null,
+        flair: post.link_flair_text ?? null,
+        over_18: Boolean(post.over_18),
+        removed: post.removed_by_category ?? post._meta?.removal_type ?? null,
+        note: 'Read from the Arctic Shift archive, not reddit.com. Scores and comment counts of content less than ~36h old may read 0/1. For the comment tree call reddit_search with mode:"thread" and this id.'
       };
     }
   },
@@ -668,7 +701,7 @@ export const TEMPLATES = [
     id: 'hacker-news-front-page',
     name: 'Hacker News Front Page',
     description: 'Scrape the Hacker News front page for a list of stories with title, URL, score, and comment count.',
-    targetPattern: /news\.ycombinator\.com(\/news)?$/i,
+    targetPattern: /news\.ycombinator\.com(\/news)?\/?$/i,
     extract($) {
       const stories = [];
       $('tr.athing').each((_, el) => {
@@ -716,29 +749,74 @@ export const TEMPLATES = [
   {
     id: 'stackoverflow-question',
     name: 'Stack Overflow Question',
-    description: 'Scrape a Stack Overflow question for title, body, votes, tags, answers, and accepted answer.',
-    targetPattern: /stackoverflow\.com\/questions\//i,
-    extract($) {
-      const answers = [];
-      $('.answer').each((_, el) => {
-        const $a = $(el);
-        answers.push({
-          votes: $a.find('[itemprop="upvoteCount"]').attr('content') || $a.find('.js-vote-count').text().trim(),
-          accepted: $a.hasClass('accepted-answer'),
-          body: $a.find('.s-prose').first().text().trim().slice(0, 500)
-        });
-      });
+    description:
+      'Read a Stack Overflow question from the Stack Exchange API rather than the rendered page: ' +
+      'title, body, score, views, tags, owner, and the answers with their scores and which one ' +
+      'was accepted. stackoverflow.com answers every non-browser fetch with a Cloudflare 403, so ' +
+      'the page itself yields nothing; the API is keyless (300 requests per day per IP).',
+    targetPattern: /stackoverflow\.com\/questions\/\d+/i,
+
+    /** Point the fetch at the API document for the same question. */
+    resolveUrl(url) {
+      const match = new URL(url).pathname.match(/\/questions\/(\d+)/);
+      if (!match) return url;
+      return `https://api.stackexchange.com/2.3/questions/${match[1]}` +
+        `?site=stackoverflow&filter=${STACKEXCHANGE_FILTER}`;
+    },
+
+    extractRaw(body, url) {
+      let doc;
+      try {
+        doc = JSON.parse(body);
+      } catch {
+        throw new Error(
+          `Not a Stack Exchange API document: ${url} did not return JSON. ` +
+          'This template reads the Stack Exchange API.'
+        );
+      }
+
+      // The API reports its own failures (bad filter, throttled, no such site)
+      // as a 400 with error_* fields; a missing question is an empty items list.
+      if (doc && doc.error_id) {
+        throw new Error(
+          `Stack Exchange API error ${doc.error_id} (${doc.error_name || 'unknown'}): ${doc.error_message || 'no message'}.`
+        );
+      }
+      const question = Array.isArray(doc?.items) ? doc.items[0] : null;
+      if (!question) {
+        throw new Error(`No Stack Overflow question at ${url}: the API returned no items.`);
+      }
+
+      // Accepted answer first, then by score — the order the site shows.
+      const answers = (question.answers || [])
+        .slice()
+        .sort((a, b) => (Number(Boolean(b.is_accepted)) - Number(Boolean(a.is_accepted))) || ((b.score ?? 0) - (a.score ?? 0)));
 
       return {
-        title: text($, '#question-header h1'),
-        body: text($, '.question .s-prose'),
-        votes: text($, '.question .js-vote-count') || attr($, '.question [itemprop="upvoteCount"]', 'content'),
-        views: text($, '.js-view-count') || attr($, 'meta[name="twitter:data1"]', 'content'),
-        tags: list($, '.post-tag'),
-        author: text($, '.question .user-details a'),
-        asked: attr($, '.question time', 'datetime'),
-        answers: answers.slice(0, 5),
-        answered: $('div.accepted-answer').length > 0
+        question_id: question.question_id ?? null,
+        // Titles and display names come HTML-encoded (&quot;, &#39;).
+        title: htmlToText(question.title),
+        body: htmlToText(question.body),
+        votes: question.score ?? null,
+        views: question.view_count ?? null,
+        tags: question.tags || [],
+        author: htmlToText(question.owner?.display_name),
+        author_reputation: question.owner?.reputation ?? null,
+        asked: epochToIso(question.creation_date),
+        last_activity: epochToIso(question.last_activity_date),
+        link: question.link || null,
+        answered: Boolean(question.is_answered),
+        accepted_answer_id: question.accepted_answer_id ?? null,
+        answer_count: question.answer_count ?? answers.length,
+        answers: answers.slice(0, 5).map(a => ({
+          answer_id: a.answer_id ?? null,
+          votes: a.score ?? null,
+          accepted: Boolean(a.is_accepted),
+          author: htmlToText(a.owner?.display_name),
+          posted: epochToIso(a.creation_date),
+          body: (htmlToText(a.body) || '').slice(0, 500) || null
+        })),
+        quota_remaining: doc.quota_remaining ?? null
       };
     }
   },
@@ -815,6 +893,37 @@ export const TEMPLATES = [
   ...GOV_TEMPLATES
 ];
 
+/**
+ * Templates withdrawn because there is no compliant way to reach the data.
+ * They are kept here by id, so a caller naming one gets the reason rather
+ * than "unknown template", and by pattern, so `auto` can say the same for a
+ * URL they would have matched. Verified against each site's robots.txt on
+ * 2026-08-30.
+ */
+export const RETIRED_TEMPLATES = {
+  'linkedin-profile': {
+    targetPattern: /linkedin\.com\/in\//i,
+    reason: 'linkedin.com/robots.txt disallows every path for all agents except LinkedIn\'s own crawler, and profile pages sit behind an authentication wall, so there is no compliant way to read a profile.'
+  },
+  tweet: {
+    targetPattern: /(twitter|x)\.com\/[^/]+\/status\//i,
+    reason: 'x.com/robots.txt disallows every path for generic agents, and the keyless embed endpoints (cdn.syndication.twimg.com, publish.x.com/oembed) are disallowed by their own robots.txt, so a tweet cannot be read without X API credentials.'
+  }
+};
+
+/**
+ * The retired template a caller is reaching for — by id, or by a URL one of
+ * them handled — as { id, reason }, or null.
+ */
+export function retiredTemplate(idOrUrl) {
+  if (typeof idOrUrl !== 'string') return null;
+  if (RETIRED_TEMPLATES[idOrUrl]) return { id: idOrUrl, reason: RETIRED_TEMPLATES[idOrUrl].reason };
+  for (const [id, entry] of Object.entries(RETIRED_TEMPLATES)) {
+    if (entry.targetPattern.test(idOrUrl)) return { id, reason: entry.reason };
+  }
+  return null;
+}
+
 // ── Registry ─────────────────────────────────────────────────────────────────
 
 /**