crawlforge-extractors 1.14.0 → 1.16.0

package/README.md

@@ -202,7 +202,7 @@ URL is reported in `reason` as a retired handle.
 ### Recognising a blocked page
 
 `documentVerdict` says what a fetched document is: the page, a bot-wall
-interstitial (Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel, AWS WAF, F5), an
+interstitial (Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel, AWS WAF, F5, Fastly), an
 HTTP error page, an empty shell, a short error-titled placeholder, or a
 client-rendered app's own short error fallback under a normal title (quora.com's
 "Something went wrong. Wait a moment and try again."). A wall
@@ -291,6 +291,11 @@ deduplicated on the URL without fragment or trailing slash; a
 
 **Government APIs** (`src/connectors/gov.js`). `nhtsa-vin` · `npi-provider`
 
+A template that can run from params instead of a URL lists them in `params`
+(`[{ name, required, description }]`, also in `registry.list()`). A template
+can add `warnings` to its result: `reddit-thread` warns when the URL names a
+different subreddit from the post's.
+
 `shopify-collection` is a list connector: it reads a store's own
 `/collections/<handle>/products.json` and returns every product in the
 collection with the same authoritative price, compare-at price and stock that

package/index.d.ts

@@ -62,6 +62,20 @@ export interface ScrapeTemplate {
    * host rate limiter's job.
    */
   crawlDelaySeconds?: number;
+  /** What listUrl takes. Declared by every template that defines listUrl. */
+  params?: TemplateParam[];
+  /**
+   * Notes about a request that still produced a correct record (reddit-thread:
+   * the URL named the wrong subreddit). run() returns them as `warnings`.
+   */
+  warnings?: (data: Record<string, unknown>, url: string) => string[];
+}
+
+/** One parameter a list connector's listUrl accepts. */
+export interface TemplateParam {
+  name: string;
+  required: boolean;
+  description: string;
 }
 
 export interface TemplateSummary {
@@ -76,6 +90,8 @@ export interface TemplateSummary {
   requires_api_key?: true;
   /** Present only when the template sets credentialRef. */
   credential_ref?: string;
+  /** Present on every template that can be driven by params. */
+  params?: TemplateParam[];
 }
 
 export interface TemplateResult {
@@ -86,6 +102,8 @@ export interface TemplateResult {
   fetchedUrl?: string;
   data: Record<string, unknown>;
   extractedAt: string;
+  /** Present only when the template's warnings hook reported something. */
+  warnings?: string[];
 }
 
 export interface TemplateListResult {
@@ -308,7 +326,7 @@ export default TemplateRegistry;
 
 /** The bot-defence vendor whose interstitial a document is, and what gave it away. */
 export interface ChallengeVerdict {
-  vendor: 'cloudflare' | 'amazon' | 'datadome' | 'perimeterx' | 'akamai' | 'vercel' | 'aws-waf' | 'f5';
+  vendor: 'cloudflare' | 'amazon' | 'datadome' | 'perimeterx' | 'akamai' | 'vercel' | 'aws-waf' | 'f5' | 'fastly';
   evidence: string;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.14.0",
+  "version": "1.16.0",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, embedded-state extraction, and query-scoped highlights. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/blockedPage.js

@@ -1,7 +1,7 @@
 /**
  * blockedPage.js — decide whether a fetched document is the page or a wall.
  *
- * Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel, AWS WAF and F5 answer
+ * Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel, AWS WAF, F5 and Fastly answer
  * a blocked request with a page of their own, often under HTTP 200: a title, some prose
  * and a challenge script. Reported as a successful scrape, that page hides
  * the block — producthunt.com came back "success:true, title: Just a
@@ -80,6 +80,19 @@ const CHALLENGES = [
     title: /^request rejected$/i,
     markers: /the requested url was rejected\.?\s*please consult with your administrator/i,
     evidence: 'an F5 "Request Rejected" blocking page'
+  },
+  {
+    // Fastly's client challenge: HTTP 200, the title "Client Challenge", a
+    // <noscript> asking for JavaScript and a loader for
+    // /_fs-ch-<token>/script.js, every asset under that same prefix
+    // (lemonde.fr to a plain fetch, 3 KB, `via: 1.1 varnish` and
+    // `x-served-by: cache-…`, 2026-10-03). Its only text outside <noscript>
+    // is a hidden "A required part of this site couldn't load" notice, which
+    // is why it read as a page rather than an empty shell.
+    vendor: 'fastly',
+    title: /^client challenge$/i,
+    markers: /\/_fs-ch-[\w-]+\//,
+    evidence: 'a Fastly client challenge script'
   }
 ];
 

package/src/connectors/ats.js

@@ -232,6 +232,11 @@ export const ATS_TEMPLATES = [
       'payload: Stripe\'s 571-job board is 349 KB without them and 4.2 MB with them.',
     targetPattern: /(boards|job-boards|boards-api)\.greenhouse\.io\//i,
 
+    params: [
+      { name: 'company', required: true, description: 'Board token, the path segment in https://job-boards.greenhouse.io/<token>.' },
+      { name: 'content', required: false, description: 'true adds each job\'s description (much larger response).' }
+    ],
+
     /**
      * `company` is Greenhouse's board token — the path segment in
      * https://job-boards.greenhouse.io/<token>, not the company's display name.
@@ -322,6 +327,12 @@ export const ATS_TEMPLATES = [
      */
     crawlDelaySeconds: 1,
 
+    params: [
+      { name: 'company', required: true, description: 'The path segment in https://jobs.lever.co/<company>.' },
+      { name: 'skip', required: false, description: 'Postings to skip, for paging.' },
+      { name: 'limit', required: false, description: 'Postings to return, for paging.' }
+    ],
+
     /** `company` is the path segment in https://jobs.lever.co/<company>. */
     listUrl(params = {}) {
       const company = requireParam(
@@ -404,6 +415,11 @@ export const ATS_TEMPLATES = [
       'a fraction of that without.',
     targetPattern: /(jobs|api)\.ashbyhq\.com\//i,
 
+    params: [
+      { name: 'company', required: true, description: 'Jobs page name, the segment in https://jobs.ashbyhq.com/<name>.' },
+      { name: 'descriptions', required: false, description: 'true keeps each job\'s plain-text description.' }
+    ],
+
     /** `company` is Ashby's jobs page name — the segment in https://jobs.ashbyhq.com/<name>. */
     listUrl(params = {}) {
       const company = requireParam(
@@ -495,6 +511,11 @@ export const ATS_TEMPLATES = [
       'documented ?details=true parameter.',
     targetPattern: /(apply\.workable\.com|www\.workable\.com\/api\/accounts|[\w-]+\.workable\.com\/(jobs|spi))/i,
 
+    params: [
+      { name: 'company', required: true, description: 'Account subdomain, the path segment in https://apply.workable.com/<subdomain>.' },
+      { name: 'details', required: false, description: 'true adds each job\'s description.' }
+    ],
+
     /** `company` is the account subdomain — the first part of the signed-in Workable URL. */
     listUrl(params = {}) {
       const company = requireParam(
@@ -580,6 +601,10 @@ export const ATS_TEMPLATES = [
       'plain-text description for every open role in one request.',
     targetPattern: /\.recruitee\.com\//i,
 
+    params: [
+      { name: 'company', required: true, description: 'The subdomain in https://<company>.recruitee.com.' }
+    ],
+
     /** `company` is the Recruitee subdomain in https://<company>.recruitee.com. */
     listUrl(params = {}) {
       const company = requireParam(
@@ -672,6 +697,12 @@ export const ATS_TEMPLATES = [
     // which robots disallows anyway) still does not match.
     targetPattern: /teamtailor\.com\/?(?:jobs(\.rss)?\/?)?(\?|#|$)/i,
 
+    params: [
+      { name: 'company', required: true, description: 'The subdomain in https://<company>.teamtailor.com.' },
+      { name: 'per_page', required: false, description: 'Jobs per feed page (feed default 100).' },
+      { name: 'offset', required: false, description: 'Jobs to skip, for paging.' }
+    ],
+
     /** `company` is the subdomain in https://<company>.teamtailor.com. */
     listUrl(params = {}) {
       const company = requireParam(

package/src/connectors/gov.js

@@ -199,6 +199,11 @@ export const GOV_TEMPLATES = [
       'unknown positions), and vPIC\'s own error codes are reported rather than swallowed.',
     targetPattern: /vpic\.nhtsa\.dot\.gov\/api\/vehicles\/DecodeVin/i,
 
+    params: [
+      { name: 'vin', required: true, description: 'The VIN to decode; "*" marks an unknown position, as in 5UXWX7C5*BA.' },
+      { name: 'modelYear', required: false, description: 'Model year, which vPIC says improves decode accuracy.' }
+    ],
+
     /**
      * @param {{ vin: string, modelYear?: string|number }} params
      */
@@ -342,6 +347,17 @@ export const GOV_TEMPLATES = [
       'NPI and joins nothing to it.',
     targetPattern: /npiregistry\.cms\.hhs\.gov\/api/i,
 
+    // At least one search criterion is required; none of them is on its own.
+    params: [
+      ...NPI_SEARCH_FIELDS.map(name => ({
+        name,
+        required: false,
+        description: 'Registry search criterion, in its own spelling. Pass at least one.'
+      })),
+      { name: 'limit', required: false, description: `Records to return, 1-${NPI_MAX_LIMIT} (default ${NPI_DEFAULT_LIMIT}).` },
+      { name: 'skip', required: false, description: `Records to skip, 0-${NPI_MAX_SKIP}.` }
+    ],
+
     /**
      * @param {Record<string, string|number>} params — the registry's own search fields
      */

package/src/pageText.js

@@ -22,8 +22,10 @@ const BLOCK_SELECTOR =
 const SENTINEL = '\uE000';
 
 /**
- * Flatten elements to text, one line per block element. Table cells on a row
- * are joined by a space. Works on a detached clone, so the caller's tree is
+ * Flatten elements to text, one line per block element: a block starts and
+ * ends a line, so inline text just before one is not welded to it (Readability
+ * turns "July 2023<br><br>If you…" into "July 2023<p>If you…</p>", which read
+ * "July 2023If you…", R24 3.1). Table cells on a row are joined by a space. Works on a detached clone, so the caller's tree is
  * untouched. Callers remove `<script>`/`<style>` first: their text is read
  * like any other.
  *
@@ -36,7 +38,7 @@ export function flattenText($, root = $('body')) {
   const $root = root.clone();
   $root.find('br').replaceWith(SENTINEL);
   $root.find('td, th').after(' ');
-  $root.find(BLOCK_SELECTOR).after(SENTINEL);
+  $root.find(BLOCK_SELECTOR).before(SENTINEL).after(SENTINEL);
   return $root
     .toArray()
     .map((el) => $(el).text())

package/src/templates.js

@@ -32,6 +32,12 @@
  * Defining extractList is the only signal that a template is a list connector;
  * there is no `kind` field. A list connector may also define
  * resolveUrl/targetPattern, so a caller can pass a URL instead of params.
+ * Every template with a listUrl declares what it takes, which list() publishes:
+ *   params                — [{ name, required, description }]
+ *
+ * One more optional hook reports what the caller should know about a request
+ * that still produced a correct record; run() returns it as `warnings`:
+ *   warnings(data,url)    — string[]
  *
  * A connector against a key-based API declares it:
  *   requiresApiKey: true
@@ -519,6 +525,37 @@ function githubCounter($, sel) {
   return /\d/.test(exact || '') ? exact : text($, sel);
 }
 
+/**
+ * A count as an integer, or null. The sidebar payload carries exact numbers
+ * (stargazerCount: 32407) and a counter's title= the same number with commas
+ * ("32,407"); the counter's visible text is rounded ("32.4k") and reads as
+ * null here rather than as a number it is not.
+ */
+function githubCount(value) {
+  if (typeof value === 'number') return Number.isInteger(value) ? value : null;
+  const digits = typeof value === 'string' ? value.replace(/,/g, '').trim() : '';
+  return /^\d+$/.test(digits) ? Number.parseInt(digits, 10) : null;
+}
+
+/** README excerpts are capped: the first paragraph of some READMEs is an essay. */
+const README_SUMMARY_MAX = 500;
+
+/**
+ * The README's first paragraph with text in it. The rendered README is in the
+ * served page (article.markdown-body); a leading paragraph of badges is all
+ * <img> and has no text, so it is skipped.
+ */
+function githubReadmeSummary($) {
+  for (const el of $('article.markdown-body p').toArray()) {
+    const paragraph = $(el).text().replace(/\s+/g, ' ').trim();
+    if (!paragraph) continue;
+    return paragraph.length > README_SUMMARY_MAX
+      ? `${paragraph.slice(0, README_SUMMARY_MAX - 1).trimEnd()}…`
+      : paragraph;
+  }
+  return null;
+}
+
 // ── Template definitions ─────────────────────────────────────────────────────
 
 export const TEMPLATES = [
@@ -601,6 +638,13 @@ export const TEMPLATES = [
       return parsed.toString();
     },
 
+    params: [
+      { name: 'store', required: true, description: 'Storefront domain, e.g. "www.allbirds.com".' },
+      { name: 'collection', required: true, description: 'Collection handle, e.g. "mens" from /collections/mens.' },
+      { name: 'limit', required: false, description: 'Products per page, 1-250 (Shopify default 30).' },
+      { name: 'page', required: false, description: 'Page number, 1 or more.' }
+    ],
+
     /**
      * Build the listing URL from params instead of a URL.
      * @param {{ store: string, collection: string, limit?: number, page?: number }} params
@@ -749,8 +793,24 @@ export const TEMPLATES = [
   {
     id: 'github-repo',
     name: 'GitHub Repository',
-    description: 'Scrape a GitHub repository page for stars, forks, description, language, topics, and README summary.',
-    targetPattern: /github\.com\/[^/]+\/[^/]+\/?$/i,
+    description:
+      'Scrape a GitHub repository page for description, exact star, fork, watcher and open-issue ' +
+      'counts, topics, licence, homepage and a README summary (the README\'s first paragraph). ' +
+      'A /tree/<branch>/… link reads the repository\'s overview page. Language and last-push ' +
+      'date are null on the page GitHub serves logged out — its browser fetches them after load.',
+    targetPattern: /github\.com\/[^/]+\/[^/]+(?:\/tree\/.+)?\/?$/i,
+
+    /**
+     * A /tree/<branch>/<path> link is a directory listing: no About sidebar,
+     * no README of the repository. The repo-level facts this template returns
+     * live on the overview page, so that is what gets fetched.
+     */
+    resolveUrl(url) {
+      const parsed = safeUrl(url);
+      const repo = parsed && /^\/([^/]+)\/([^/]+)\/tree\//.exec(parsed.pathname);
+      return repo ? `${parsed.origin}/${repo[1]}/${repo[2]}` : url;
+    },
+
     extract($) {
       const about = githubSidebarAbout($);
       return {
@@ -761,11 +821,13 @@ export const TEMPLATES = [
         description: about
           ? tidy(about.description)
           : tidy(text($, 'p.f4.my-3')) || githubOgDescription($),
-        stars: text($, '#repo-stars-counter-star') || text($, '[aria-label*="stargazers"]'),
-        forks: text($, '#repo-network-counter') || text($, '[aria-label*="forks"]'),
+        // Exact integers: the payload's counts first, then the counter's
+        // title=. The counter's text ("32.4k") is rounded and never used.
+        stars: githubCount(about?.stargazerCount) ?? githubCount(attr($, '#repo-stars-counter-star', 'title')),
+        forks: githubCount(about?.forksCount) ?? githubCount(attr($, '#repo-network-counter', 'title')),
         // React (logged-out) layout has no watchers aria-label; the count is
         // the <strong> right after the single octicon-eye.
-        watchers: text($, '.octicon-eye + strong') || text($, '[aria-label*="watchers"]'),
+        watchers: githubCount(about?.watcherCount) ?? githubCount(text($, '.octicon-eye + strong')),
         // Language and the last-push date appear nowhere in the logged-out
         // page (verified 2026-08-26 across five User-Agents, embedded JSON
         // included): the client fetches them after load from header-gated
@@ -781,7 +843,8 @@ export const TEMPLATES = [
         // When the sidebar payload is present it is authoritative: an empty
         // website means "no homepage", not "go scrape some external link".
         homepage: safeHref(about ? about.website : attr($, 'a[href][rel="noopener noreferrer"]', 'href')),
-        open_issues: githubCounter($, '#issues-repo-tab-count') || text($, '.Counter[aria-label*="issue"]')
+        open_issues: githubCount(githubCounter($, '#issues-repo-tab-count') || text($, '.Counter[aria-label*="issue"]')),
+        readme_summary: githubReadmeSummary($)
       };
     }
   },
@@ -790,7 +853,19 @@ export const TEMPLATES = [
     id: 'youtube-video',
     name: 'YouTube Video',
     description: 'Scrape a YouTube video page for title, channel, views, likes, publish date, and description.',
-    targetPattern: /youtube\.com\/watch/i,
+    targetPattern: /youtube\.com\/watch|youtu\.be\/[\w-]+/i,
+
+    /**
+     * youtu.be/<id> is the share-button link. youtu.be only redirects, so the
+     * watch page for the same id is what gets fetched.
+     */
+    resolveUrl(url) {
+      const parsed = safeUrl(url);
+      if (!parsed || !/^(?:www\.)?youtu\.be$/i.test(parsed.hostname)) return url;
+      const id = parsed.pathname.split('/').filter(Boolean)[0];
+      return id ? `https://www.youtube.com/watch?v=${encodeURIComponent(id)}` : url;
+    },
+
     extract($) {
       return {
         title: attr($, 'meta[name="title"]', 'content') || attr($, 'meta[property="og:title"]', 'content'),
@@ -866,6 +941,21 @@ export const TEMPLATES = [
         removed: post.removed_by_category ?? post._meta?.removal_type ?? null,
         note: 'Read from the Arctic Shift archive, not reddit.com. Scores and comment counts of content less than ~36h old may read 0/1. For the comment tree call reddit_search with mode:"thread" and this id.'
       };
+    },
+
+    /**
+     * Only the post id reaches the archive, so a URL naming the wrong
+     * subreddit still returns the right post — ids are unique across Reddit,
+     * and reddit.com itself redirects such a URL to the real one. That is
+     * worth a warning, not a refusal: the caller gets the post it named.
+     */
+    warnings(data, url) {
+      const named = /reddit\.com\/r\/([^/?#]+)\/comments\//i.exec(url ?? '')?.[1];
+      if (!named || !data.subreddit || named.toLowerCase() === data.subreddit.toLowerCase()) return [];
+      return [
+        `The URL names r/${named}, but post ${data.id} is in r/${data.subreddit}. Reddit post ids ` +
+        'are unique, so this is the post the id names; data.url is its real address.'
+      ];
     }
   },
 
@@ -1177,6 +1267,8 @@ export class TemplateRegistry {
       targetPattern: t.targetPattern ? t.targetPattern.toString() : null,
       // extractList is the only signal that a template returns N entities.
       mode: t.extractList ? 'list' : 'entity',
+      // What listUrl takes, so a caller can drive a connector without a URL.
+      ...(t.params ? { params: t.params } : {}),
       ...(t.requiresApiKey ? { requires_api_key: true } : {}),
       ...(t.credentialRef ? { credential_ref: t.credentialRef } : {})
     }));
@@ -1254,13 +1346,18 @@ export class TemplateRegistry {
       );
     }
 
+    // Something about the request worth telling the caller that does not make
+    // the record wrong (reddit-thread: the URL named the wrong subreddit).
+    const warnings = template.warnings ? template.warnings(data, url) : [];
+
     return {
       template: id,
       template_name: template.name,
       url,
       ...(fetchedUrl !== url ? { fetchedUrl } : {}),
       data,
-      extractedAt: new Date().toISOString()
+      extractedAt: new Date().toISOString(),
+      ...(warnings.length > 0 ? { warnings } : {})
     };
   }