crawlforge-extractors 1.2.3 → 1.4.0

package/README.md

@@ -1,7 +1,8 @@
 # crawlforge-extractors
 
 Extraction logic shared by the [CrawlForge](https://www.crawlforge.dev) MCP server and REST API:
-site-specific scrape templates, response body reading, and structural fingerprinting.
+site-specific scrape templates, response body reading, structural fingerprinting, and
+embedded-state extraction.
 
 ## Why this package exists
 
@@ -49,6 +50,55 @@ to `extract($)` with a cheerio document otherwise.
 A template that rejects a response as not its own throws — surface that to the
 caller as a bad request, not a server error.
 
+### Picking a template from a URL
+
+```js
+registry.detect('https://www.allbirds.com/collections/mens'); // → shopify-collection
+registry.detect('https://example.com/about');                 // → null
+```
+
+`detect()` matches on the `targetPattern` each template already carried.
+Ranking is deterministic: a pattern that names a host outranks one that only
+matches a path shape, so `amazon-product` wins an Amazon URL that happens to
+contain `/products/`, which `shopify-product` also matches. Remaining ties go
+to registration order.
+
+### List connectors
+
+A template that defines `extractList` returns N entities from one call instead
+of one entity from one page. `listUrl(params)` builds the request from a plain
+object, and `runList()` mirrors `run()`'s envelope:
+
+```js
+const template = registry.get('shopify-collection');
+
+// By params…
+const url = template.listUrl({ store: 'www.allbirds.com', collection: 'mens', limit: 250 });
+// …or from a collection URL the user already has.
+const alsoUrl = template.resolveUrl('https://www.allbirds.com/collections/mens');
+
+const body = await (await fetch(url)).text();
+const { data } = await registry.runList('shopify-collection', body, { url });
+data.items; // one entity per product, same field shape as shopify-product
+data.count; // items.length, unless the source declares a larger total_available
+```
+
+`registry.list()` reports `mode: 'list'` or `'entity'` per template, derived
+from the presence of `extractList` — there is no stored `kind` field.
+
+### Templates that need an API key
+
+A connector against a key-based API declares `requiresApiKey: true` and
+`credentialRef: 'SOME_ENV_VAR'`, surfaced by `list()` as `requires_api_key` and
+`credential_ref`. This package never reads `process.env`: the consumer resolves
+the variable and passes the key in as `params.apiKey`, and `listUrl` throws an
+error naming the variable when it is missing — an actionable message beats a
+401 passed through from someone else's API.
+
+`new TemplateRegistry(templates)` takes an alternative template set, which is
+how the credential path is tested without shipping a connector nobody has a key
+for.
+
 ### Reading a response body
 
 `readBody` decodes with the body's real charset and refuses to buffer past a
@@ -85,15 +135,85 @@ A signature is the page's tag vocabulary plus its element-count-by-depth
 histogram — a few dozen keys, small enough to store next to a change-tracking
 baseline instead of keeping the whole DOM.
 
+### Reading a page's embedded state
+
+`extractEmbeddedState` returns the JSON a page already ships in its own HTML —
+`__NEXT_DATA__`, RSC flight chunks (`self.__next_f`), `__NUXT__`,
+`__APOLLO_STATE__`, `__INITIAL_STATE__`, `__PRELOADED_STATE__` and
+`<script type="application/json">` blocks. No LLM is involved, so the values are
+the site's own and cannot be fabricated.
+
+```js
+import { extractEmbeddedState, selectJsonPath } from 'crawlforge-extractors';
+
+const { data, found, warnings } = extractEmbeddedState(rawHtml);
+// found -> [{ name: 'next_data', variable: '__NEXT_DATA__', bytes: 439333 }]
+
+selectJsonPath(data, 'next_data.props.pageProps.events.0.name');
+```
+
+Pass the **raw** HTML. Every source lives in a `<script>` tag, so a document
+whose scripts have been stripped has nothing left to read.
+
+Payloads are never truncated — a half-serialized object is worse than a big
+one. `selectJsonPath` is how a caller asks for less: dotted keys and array
+indexes only, no wildcards, filters or recursive descent. A path that does not
+resolve throws naming the keys that *were* available at the point it stopped,
+so a typo comes back fixable rather than empty.
+
+A source that is present but is not JSON — Nuxt 2's IIFE wrapper, Nuxt 3's
+unquoted-key object literal — is reported in `warnings` unparsed. Nothing here
+calls `eval`.
+
 ## Templates
 
-`shopify-product` · `amazon-product` · `linkedin-profile` · `github-repo` ·
-`youtube-video` · `tweet` · `reddit-thread` · `hacker-news-front-page` ·
-`producthunt-launch` · `stackoverflow-question` · `npm-package`
+**Pages and products.** `shopify-product` · `shopify-collection` ·
+`amazon-product` · `linkedin-profile` · `github-repo` · `youtube-video` ·
+`tweet` · `reddit-thread` · `hacker-news-front-page` · `producthunt-launch` ·
+`stackoverflow-question` · `npm-package`
+
+**Job boards** (`src/connectors/ats.js`). `greenhouse-jobs` ·
+`lever-postings` · `ashby-jobs` · `workable-jobs` · `recruitee-offers` ·
+`teamtailor-jobs`
+
+**Government APIs** (`src/connectors/gov.js`). `nhtsa-vin` · `npi-provider`
+
+`shopify-collection` is a list connector: it reads a store's own
+`/collections/<handle>/products.json` and returns every product in the
+collection with the same authoritative price, compare-at price and stock that
+`shopify-product` returns for one, so the two cannot disagree. Pass a
+collection URL or `{ store, collection }`. Shopify serves 30 products per page
+by default and 250 at most, so a large collection needs `page`.
+
+The job-board connectors read each platform's own documented public postings
+API, so a board's jobs come back exact rather than parsed out of a rendered
+page. All six normalise onto one job shape — `id`, `title`, `url`, `location`,
+`department`, `team`, `employment_type`, `remote`, `published_at`,
+`updated_at`, `description`, `source` — so two platforms union without
+per-source mapping, and a field the platform does not carry is `null` rather
+than guessed. Pass a board URL or `{ company }`. Greenhouse defaults to
+summary records; `content: true` adds the full HTML descriptions and takes a
+large board past 4 MB. `lever-postings` declares `crawlDelaySeconds: 1`,
+which `api.lever.co/robots.txt` asks for and the calling surface's host rate
+limiter is expected to honour.
+
+`nhtsa-vin` decodes a VIN through the NHTSA vPIC API — the ~154 returned
+fields are curated into a named vehicle shape with the API's empty-string
+"not applicable" normalised to `null`, the full set kept under `raw`, and the
+API's own `ErrorCode`/`ErrorText` surfaced as `decode_errors` rather than
+swallowed, because a partial decode is a real answer. `npi-provider` reads the
+CMS NPI Registry — a public professional registry — and passes its records
+through as published. Neither needs a key.
 
 `reddit-thread` is registered here but reddit.com blocks plain fetchers; the
 REST API steers those callers to its `reddit_search` tool instead.
 
+`smartrecruiters-postings` is deliberately **not** shipped: SmartRecruiters
+documents the endpoint publicly, but `api.smartrecruiters.com/robots.txt`
+disallows everything for every agent except `LinkedInBot`. Reaching it would
+mean overriding robots.txt on every call, which is not a connector's decision
+to make for its caller.
+
 ## Tests
 
 ```bash

package/index.d.ts

@@ -3,13 +3,25 @@ import type { load } from 'cheerio';
 /** The parsed-document type cheerio's load() returns. */
 export type CheerioDoc = ReturnType<typeof load>;
 
+/** What extractList returns: N entities, plus whatever meta the connector has. */
+export interface TemplateList extends Record<string, unknown> {
+  items: Record<string, unknown>[];
+  /** items.length, unless the payload declares a larger total. */
+  count: number;
+  /** Present only when the source declares a total beyond this page. */
+  total_available?: number;
+}
+
 export interface ScrapeTemplate {
   /** Slug callers pass as the `template` parameter. */
   id: string;
   name: string;
   description: string;
-  /** URLs this template handles. */
-  targetPattern: RegExp;
+  /**
+   * URLs this template handles, and what detect() matches on. Absent on a list
+   * connector reached by params rather than by URL.
+   */
+  targetPattern?: RegExp;
   /**
    * Extract from the parsed page. Absent on templates that read a
    * machine-readable endpoint instead — those define extractRaw.
@@ -26,14 +38,44 @@ export interface ScrapeTemplate {
    * a bad request, not a server error.
    */
   extractRaw?: (body: string, url: string) => Record<string, unknown>;
+  /**
+   * Build the URL to fetch from a plain params object. Throws an Error naming
+   * the parameter when a required one is missing — including the API key,
+   * which arrives as params.apiKey.
+   */
+  listUrl?: (params?: Record<string, unknown>) => string;
+  /**
+   * Parse the response into N entities. Defining this is the only thing that
+   * makes a template a list connector; there is no `kind` field.
+   */
+  extractList?: (body: string, url?: string) => TemplateList;
+  /** The connector reads a key-based API. */
+  requiresApiKey?: true;
+  /**
+   * Name of the env var the CONSUMER reads for that key. This package never
+   * touches process.env; the key is passed in as params.apiKey.
+   */
+  credentialRef?: string;
+  /**
+   * Crawl-delay the platform's robots.txt asks for, in seconds. The
+   * connector does not fetch, so honouring it is the calling surface's
+   * host rate limiter's job.
+   */
+  crawlDelaySeconds?: number;
 }
 
 export interface TemplateSummary {
   id: string;
   name: string;
   description: string;
-  /** targetPattern rendered as a string, for JSON responses. */
-  targetPattern: string;
+  /** targetPattern rendered as a string, or null on a params-only connector. */
+  targetPattern: string | null;
+  /** Derived from extractList: 'list' returns N entities, 'entity' returns one. */
+  mode: 'list' | 'entity';
+  /** Present only when the template sets requiresApiKey. */
+  requires_api_key?: true;
+  /** Present only when the template sets credentialRef. */
+  credential_ref?: string;
 }
 
 export interface TemplateResult {
@@ -46,16 +88,43 @@ export interface TemplateResult {
   extractedAt: string;
 }
 
+export interface TemplateListResult {
+  template: string;
+  template_name: string;
+  /** Whichever of the two the caller reached the endpoint with. */
+  url?: string;
+  params?: Record<string, unknown>;
+  data: TemplateList;
+  extractedAt: string;
+}
+
 export declare const TEMPLATES: ScrapeTemplate[];
 
 export declare class TemplateRegistry {
+  /** @param templates injectable, so a test can register a fixture template. */
+  constructor(templates?: ScrapeTemplate[]);
   list(): TemplateSummary[];
   get(id: string): ScrapeTemplate | undefined;
+  /**
+   * Pick the template that handles a URL, or null when none does. A pattern
+   * naming a host outranks one matching only a path shape; remaining ties go to
+   * registration order.
+   */
+  detect(url: string | null | undefined): ScrapeTemplate | null;
   /**
    * Run a template against a fetched body. Templates never fetch: the caller
    * owns SSRF policy, timeouts and billing.
    */
   run(id: string, body: string, url: string, fetchedUrl?: string): Promise<TemplateResult>;
+  /**
+   * Run a list connector against a fetched body — N entities where run()
+   * returns one. Throws when the template defines no extractList.
+   */
+  runList(
+    id: string,
+    body: string,
+    context?: { url?: string; params?: Record<string, unknown> }
+  ): Promise<TemplateListResult>;
 }
 
 /** Default cap on a buffered response body: 25 MB. */
@@ -107,4 +176,43 @@ export declare function structuralSimilarity(
   current: Partial<StructureSignature> | null | undefined
 ): number;
 
+/** One embedded-state payload a page carries, as reported in `found`. */
+export interface EmbeddedStateSource {
+  /** Path-safe key this payload is addressed by, e.g. "next_data". */
+  name: string;
+  /** The raw thing it was read from, e.g. "__NEXT_DATA__", "self.__next_f". */
+  variable: string;
+  /** Serialized size of this payload alone. */
+  bytes: number;
+  /** Present when the shape needs explaining (RSC rows, json_scripts blocks). */
+  note?: string;
+}
+
+export interface EmbeddedStateResult {
+  /** Payloads keyed by `name`; empty when the page ships no readable state. */
+  data: Record<string, unknown>;
+  found: EmbeddedStateSource[];
+  /** Sources seen but not parsed, and blocks that were not valid JSON. */
+  warnings: string[];
+}
+
+/**
+ * Find the JSON state a page already ships in its own HTML: __NEXT_DATA__,
+ * RSC flight chunks (self.__next_f), __NUXT__, __APOLLO_STATE__,
+ * __INITIAL_STATE__, __PRELOADED_STATE__ and <script type="application/json">.
+ *
+ * Pass the RAW html. A script-stripped document has nothing left to read.
+ */
+export declare function extractEmbeddedState(rawHtml: string): EmbeddedStateResult;
+
+/** Split a path into its segments. Dotted keys and array indexes only. */
+export declare function parseJsonPath(path: string): string[];
+
+/**
+ * Resolve a path against a parsed object. Not JSONPath: no wildcards, filters,
+ * slices or recursive descent. Throws naming the keys that were available at
+ * the point it stopped.
+ */
+export declare function selectJsonPath(root: unknown, path: string): unknown;
+
 export default TemplateRegistry;

package/index.js

@@ -25,3 +25,7 @@ export {
 } from './src/body.js';
 
 export { structureSignature, structuralSimilarity } from './src/structure.js';
+
+export { extractEmbeddedState } from './src/embeddedState.js';
+
+export { parseJsonPath, selectJsonPath } from './src/jsonPath.js';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.2.3",
-  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, and structural fingerprinting. One implementation, so the two surfaces cannot drift apart.",
+  "version": "1.4.0",
+  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",
   "types": "./index.d.ts",
@@ -11,18 +11,33 @@
       "default": "./index.js"
     }
   },
-  "files": ["index.js", "index.d.ts", "src/", "README.md", "LICENSE"],
+  "files": [
+    "index.js",
+    "index.d.ts",
+    "src/",
+    "README.md",
+    "LICENSE"
+  ],
   "scripts": {
     "test": "node --test tests/*.test.js"
   },
   "dependencies": {
     "cheerio": "^1.1.2"
   },
-  "keywords": ["crawlforge", "scraping", "extraction", "templates", "charset", "change-detection"],
+  "keywords": [
+    "crawlforge",
+    "scraping",
+    "extraction",
+    "templates",
+    "charset",
+    "change-detection"
+  ],
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/mysleekdesigns/crawlforge-extractors.git"
   },
-  "engines": { "node": ">=18" }
+  "engines": {
+    "node": ">=18"
+  }
 }