crawlforge-extractors 1.12.0 → 1.13.0

package/README.md

@@ -252,6 +252,32 @@ boundary, and an ASCII period does not split after `Dr.`, `e.g.`, `Node.js`
 or `3.14`. A CJK query matches by character bigrams, so no segmenter is
 needed.
 
+### Page text and link records
+
+`flattenText` and `extractLinkRecords` are what `extract_text` and
+`extract_links` return on both surfaces, so the same page reads the same
+through the MCP server and the REST API.
+
+```js
+import { load } from 'cheerio';
+import { flattenText, extractLinkRecords } from 'crawlforge-extractors';
+
+const $ = load(html);
+$('script, style, noscript').remove();
+flattenText($);                    // "Hi\nthere" for <h1>Hi</h1><p>there</p>
+flattenText($, $('article'));      // only the matched elements
+extractLinkRecords($, { pageUrl: finalUrl });
+// [{ href: 'https://example.com/about', text: 'About', type: 'internal',
+//    domain: 'example.com', rel: null, original_href: '/about' }, …]
+```
+
+Text is one line per block element, table cells on a row joined by a space.
+Links resolve against `baseUrl`, else the page's `<base href>`, else the page
+URL; `mailto:`, `tel:` and `javascript:` links are `type: "other"`, and only
+`mailto:`/`tel:` keep an href (nothing runnable is returned). Links are
+deduplicated on the URL without fragment or trailing slash; a
+`#fragment`-only href is skipped unless `includeAnchors` is set.
+
 ## Templates
 
 **Pages and products.** `shopify-product` · `shopify-collection` ·

package/index.d.ts

@@ -454,3 +454,42 @@ export declare function redactPii(
   text: string,
   options?: { entities?: string[]; replaceStyle?: 'tag' | 'mask' | 'remove' }
 ): { text: string; redaction: PiiRedaction };
+
+/**
+ * Flatten elements to text, one line per block element (`<h1>Hi</h1><p>there</p>`
+ * reads "Hi\nthere"); table cells on a row are joined by a space. Reads a
+ * detached clone. `root` defaults to `<body>`; several matched elements are
+ * each read, a line break between them. Remove `<script>`/`<style>` first.
+ */
+export declare function flattenText($: CheerioDoc, root?: ReturnType<CheerioDoc>): string;
+
+/** One `<a href>` as extract_links returns it on both surfaces. */
+export interface LinkRecord {
+  /** Absolute for http(s); as written for an anchor or an unresolvable relative href; for `other`, kept only for `mailto:`/`tel:`, else null. */
+  href: string | null;
+  /** The link's text, whitespace collapsed; may be empty. */
+  text: string;
+  /** `internal` = same hostname as the page; `other` = any non-web scheme (`mailto:`, `tel:`, `javascript:`, …). */
+  type: 'internal' | 'external' | 'anchor' | 'other' | 'relative';
+  /** Hostname of an http(s) link, else null. */
+  domain: string | null;
+  /** The `rel` attribute, else null. */
+  rel: string | null;
+  /** The href as written on the page (null where `href` is null). */
+  original_href: string | null;
+  /** The `title` attribute, when present. */
+  title?: string;
+}
+
+/**
+ * Every `<a href>` as a `LinkRecord`, in document order. The resolution base
+ * is `baseUrl`, else the document's `<base href>` resolved against `pageUrl`,
+ * else `pageUrl`; internal versus external is judged by `pageUrl`'s hostname
+ * (`baseUrl`'s without one). `#fragment`-only hrefs are skipped unless
+ * `includeAnchors`. Deduplicated (unless `dedupe: false`) on the URL without
+ * fragment or trailing slash, keeping the first.
+ */
+export declare function extractLinkRecords(
+  $: CheerioDoc,
+  options?: { pageUrl?: string; baseUrl?: string; includeAnchors?: boolean; dedupe?: boolean }
+): LinkRecord[];

package/index.js

@@ -39,3 +39,7 @@ export { detectChallengePage, documentVerdict, SOFT_ERROR_MAX_CHARS } from './sr
 export { segmentUnits, rankUnits } from './src/highlights.js';
 
 export { redactPii, REGEX_ENTITIES, MODEL_ONLY_ENTITIES, DEFAULT_REPLACE_STYLE } from './src/pii.js';
+
+export { flattenText } from './src/pageText.js';
+
+export { extractLinkRecords } from './src/links.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.12.0",
+  "version": "1.13.0",
   "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, embedded-state extraction, and query-scoped highlights. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",

package/src/links.js

@@ -0,0 +1,144 @@
+/**
+ * The link records extract_links returns on both surfaces.
+ *
+ * The MCP server and the REST route used to read links their own way: one
+ * compared origins, the other hostnames; one honoured `<base href>`, the other
+ * did not; one dropped `javascript:` links, the other counted them as
+ * external; neither normalised before deduping. The same URL gave different
+ * counts. Both now read links here.
+ */
+
+import { safeHref } from './urls.js';
+
+/**
+ * Non-web schemes whose href is kept on a `type: "other"` record. Neither
+ * runs anything when followed, which is what `safeHref` guards against; every
+ * other scheme (`javascript:`, `data:`, `vbscript:`, …) keeps its record with
+ * `href` and `original_href` null.
+ */
+const KEPT_OTHER_SCHEMES = new Set(['mailto:', 'tel:']);
+
+/**
+ * The URL a link is deduplicated under: no fragment, no trailing slash on a
+ * non-root path. The record keeps the href as written on the page.
+ * @param {URL} url
+ * @returns {string}
+ */
+function dedupeKey(url) {
+  const key = new URL(url.href);
+  key.hash = '';
+  if (key.pathname.length > 1 && key.pathname.endsWith('/')) key.pathname = key.pathname.slice(0, -1);
+  return key.href;
+}
+
+/**
+ * @param {string|undefined} value
+ * @returns {URL|null}
+ */
+function parseUrl(value) {
+  if (!value) return null;
+  try {
+    return new URL(value);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Every `<a href>` in the document as a record, in document order.
+ *
+ * - `type`: `internal` (same hostname as the page), `external`, `anchor` (an
+ *   href that is only a `#fragment`, left as written), `other` (any non-web
+ *   scheme: `mailto:`, `tel:`, `javascript:`, …) or `relative` (a relative
+ *   href with no page or base URL to resolve it against).
+ * - The resolution base is `baseUrl` when given, else the document's
+ *   `<base href>` resolved against `pageUrl`, else `pageUrl`. Internal versus
+ *   external is judged against `pageUrl`'s host (`baseUrl`'s when there is no
+ *   page URL).
+ * - Deduplicated (unless `dedupe: false`) on the URL without its fragment and
+ *   without a trailing slash; the first occurrence is kept.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {{ pageUrl?: string, baseUrl?: string, includeAnchors?: boolean, dedupe?: boolean }} [options]
+ * @returns {{ href: string|null, text: string, type: 'internal'|'external'|'anchor'|'other'|'relative',
+ *   domain: string|null, rel: string|null, original_href: string|null, title?: string }[]}
+ */
+export function extractLinkRecords($, { pageUrl, baseUrl, includeAnchors = false, dedupe = true } = {}) {
+  const page = parseUrl(pageUrl);
+  let base = parseUrl(baseUrl);
+  if (!base) {
+    const baseHref = $('base[href]').first().attr('href');
+    let docBase = null;
+    if (baseHref) {
+      try {
+        docBase = new URL(baseHref, page ?? undefined);
+      } catch {
+        // An invalid <base href> is ignored, as a browser does.
+      }
+    }
+    base = docBase || page;
+  }
+  const host = (page || base)?.hostname ?? null;
+
+  const records = [];
+  const seen = new Set();
+
+  $('a[href]').each((_, element) => {
+    const $el = $(element);
+    const raw = ($el.attr('href') || '').trim();
+    if (!raw) return;
+
+    const text = $el.text().replace(/\s+/g, ' ').trim();
+    const rel = $el.attr('rel')?.trim() || null;
+    const title = $el.attr('title')?.trim();
+
+    let record;
+    let key;
+    if (raw.startsWith('#')) {
+      if (!includeAnchors) return;
+      record = { href: raw, text, type: 'anchor', domain: null, rel, original_href: raw };
+      key = `anchor:${raw}`;
+    } else {
+      let url;
+      try {
+        url = base ? new URL(raw, base) : new URL(raw);
+      } catch {
+        // No base to resolve against: a relative href stays as written.
+        if (base || /^[a-z][a-z0-9+.-]*:/i.test(raw)) return;
+        record = { href: raw, text, type: 'relative', domain: null, rel, original_href: raw };
+        key = `relative:${raw}`;
+      }
+      if (url && (url.protocol === 'http:' || url.protocol === 'https:')) {
+        record = {
+          href: url.href,
+          text,
+          type: url.hostname === host ? 'internal' : 'external',
+          domain: url.hostname,
+          rel,
+          original_href: safeHref(raw)
+        };
+        key = dedupeKey(url);
+      } else if (url) {
+        const kept = KEPT_OTHER_SCHEMES.has(url.protocol);
+        record = {
+          href: kept ? url.href : null,
+          text,
+          type: 'other',
+          domain: null,
+          rel,
+          original_href: kept ? raw : null
+        };
+        key = `other:${url.href}`;
+      }
+    }
+
+    if (dedupe) {
+      if (seen.has(key)) return;
+      seen.add(key);
+    }
+    if (title) record.title = title;
+    records.push(record);
+  });
+
+  return records;
+}

package/src/pageText.js

@@ -0,0 +1,47 @@
+/**
+ * Block-aware text flattening, shared by extract_text on both surfaces.
+ *
+ * Cheerio's .text() joins adjacent elements with no separator: `<h1>Hi</h1>
+ * <p>there</p>` reads "Hithere", and table rows and list items weld together
+ * ("1.Story title329 points"). The REST route returned exactly that while the
+ * MCP server's copy kept one line per block; this is the MCP copy
+ * (`flattenBodyText` in its `_fetchAndParse.js`), moved here so the two
+ * surfaces return the same text.
+ */
+
+/** Elements whose end is a line break. */
+const BLOCK_SELECTOR =
+  'p, div, li, tr, h1, h2, h3, h4, h5, h6, blockquote, pre, table, ul, ol, dl, dt, dd, section, article, ' +
+  'header, footer, main, nav, aside, figure, figcaption, form, fieldset, caption';
+
+// Block boundaries are marked with a U+E000 private-use sentinel so the HTML
+// source's own insignificant newlines can be collapsed to spaces first, and
+// only the sentinels become line breaks. (NUL won't survive: .after() and
+// .replaceWith() parse their argument as HTML and the parser strips NUL;
+// U+E000 passes through and never occurs in real page text.)
+const SENTINEL = '\uE000';
+
+/**
+ * Flatten elements to text, one line per block element. Table cells on a row
+ * are joined by a space. Works on a detached clone, so the caller's tree is
+ * untouched. Callers remove `<script>`/`<style>` first: their text is read
+ * like any other.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {import('cheerio').Cheerio<any>} [root] the elements to read; default `<body>`.
+ *   Several matched elements are each read, a line break between them.
+ * @returns {string}
+ */
+export function flattenText($, root = $('body')) {
+  const $root = root.clone();
+  $root.find('br').replaceWith(SENTINEL);
+  $root.find('td, th').after(' ');
+  $root.find(BLOCK_SELECTOR).after(SENTINEL);
+  return $root
+    .toArray()
+    .map((el) => $(el).text())
+    .join(SENTINEL)
+    .replace(/\s+/g, ' ')
+    .replace(/ ?(?:\uE000 ?)+/g, '\n')
+    .trim();
+}