@dpopsuev/web-spider 0.10.4

package/dist/throttle.js

@@ -0,0 +1,85 @@
+/**
+ * Per-domain request throttle with exponential backoff and jitter.
+ *
+ * Enforces a minimum gap between requests to the same hostname.
+ * On 429/503, backs off exponentially and respects Retry-After headers.
+ * Shared instances should be passed into spider() and crawl() so that
+ * all requests to a domain coordinate through one rate limiter.
+ */
+function sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}
+function parseRetryAfter(header) {
+    if (!header)
+        return 0;
+    const seconds = parseInt(header, 10);
+    if (!isNaN(seconds))
+        return seconds * 1_000;
+    const date = new Date(header).getTime();
+    if (!isNaN(date))
+        return Math.max(0, date - Date.now());
+    return 0;
+}
+export class DomainThrottle {
+    constructor(opts = {}) {
+        this.states = new Map();
+        this.minDelayMs = opts.minDelayMs ?? 500;
+        this.backoffBaseMs = opts.backoffBaseMs ?? 1_000;
+        this.backoffCapMs = opts.backoffCapMs ?? 30_000;
+        this.maxRetries = opts.maxRetries ?? 3;
+    }
+    state(host) {
+        let s = this.states.get(host);
+        if (!s) {
+            s = { lastAt: 0, backoffUntil: 0, errors: 0 };
+            this.states.set(host, s);
+        }
+        return s;
+    }
+    /** Wait until the domain's rate limit and backoff have cleared. */
+    async wait(url) {
+        const s = this.state(new URL(url).hostname);
+        const minDelay = s.minDelayMs ?? this.minDelayMs;
+        const now = Date.now();
+        const delay = Math.max(Math.max(0, s.backoffUntil - now), Math.max(0, s.lastAt + minDelay - now));
+        if (delay > 0)
+            await sleep(delay);
+        s.lastAt = Date.now();
+    }
+    /** Record a successful request — resets backoff for the domain. */
+    success(url) {
+        const s = this.state(new URL(url).hostname);
+        s.errors = 0;
+        s.backoffUntil = 0;
+    }
+    /**
+     * Record a rate-limit hit. Applies exponential backoff with jitter,
+     * using Retry-After header when present. Returns the wait duration in ms.
+     */
+    rateLimit(url, retryAfterHeader) {
+        const s = this.state(new URL(url).hostname);
+        s.errors++;
+        const retryAfterMs = parseRetryAfter(retryAfterHeader);
+        const jitter = Math.random() * this.backoffBaseMs;
+        const backoffMs = Math.min(this.backoffCapMs, this.backoffBaseMs * 2 ** (s.errors - 1) + jitter);
+        const waitMs = Math.max(retryAfterMs, backoffMs);
+        s.backoffUntil = Date.now() + waitMs;
+        return waitMs;
+    }
+    /**
+     * Override the minimum delay for a specific domain.
+     * Used to honour robots.txt Crawl-delay directives.
+     */
+    setDomainDelay(host, ms) {
+        this.state(host).minDelayMs = ms;
+    }
+}
+/**
+ * Factory — avoids jiti/Bun CJS re-export interop where class constructors
+ * accessed through a re-export chain can appear undefined at call site.
+ * Use this in extension code instead of `new DomainThrottle()`.
+ */
+export function createThrottle(opts) {
+    return new DomainThrottle(opts);
+}
+//# sourceMappingURL=throttle.js.map

package/dist/tree.d.ts

@@ -0,0 +1,34 @@
+import type { DOMNode, TreeHit } from "./types.js";
+/**
+ * Build a simplified semantic DOM tree from Readability article HTML.
+ *
+ * The root is always an "article" node. Presentational wrappers are collapsed,
+ * single-child chains are simplified, and only semantic tags survive.
+ */
+export declare function buildTree(articleHtml: string, baseUrl: string): DOMNode;
+/**
+ * Navigate to a specific node by its dot-bracket path.
+ * Returns null if the path does not exist in the tree.
+ *
+ * @example navigateTree(tree, "article.section[1].pre[0]")
+ */
+export declare function navigateTree(root: DOMNode, path: string): DOMNode | null;
+export interface QueryTreeOptions {
+    /** Max hits to return (default 10). */
+    topN?: number;
+    /** Context chars around match in snippet (default 100). */
+    snippetRadius?: number;
+}
+/**
+ * Fuzzy-search a DOM tree for a query string.
+ *
+ * Returns hits ranked by score. Each hit is the nearest semantic ancestor
+ * that contains the match (a section, li, pre, p — not a raw div). This
+ * means code blocks and table rows are always returned whole.
+ *
+ * @example
+ * const hits = queryTree(tree, "declaration merge", { topN: 3 })
+ * // hits[0].node is the full <section> containing that heading
+ */
+export declare function queryTree(root: DOMNode, query: string, opts?: QueryTreeOptions): TreeHit[];
+//# sourceMappingURL=tree.d.ts.map

package/dist/tree.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tree.d.ts","sourceRoot":"","sources":["../src/tree.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AA6MnD;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAmBvE;AAeD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAGxE;AAoED,MAAM,WAAW,gBAAgB;IAChC,uCAAuC;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,OAAO,EAAE,CAqC9F"}

package/dist/tree.js

@@ -0,0 +1,354 @@
+import { parseHTML } from "linkedom";
+// ---------------------------------------------------------------------------
+// Semantic tag sets
+// ---------------------------------------------------------------------------
+/**
+ * Tags that are kept as-is in the simplified tree.
+ * Everything else is either collapsed (single-child wrappers) or stripped.
+ */
+const BLOCK_TAGS = new Set([
+    "article",
+    "section",
+    "main",
+    "aside",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "p",
+    "blockquote",
+    "pre",
+    "figure",
+    "figcaption",
+    "ul",
+    "ol",
+    "li",
+    "table",
+    "thead",
+    "tbody",
+    "tfoot",
+    "tr",
+    "th",
+    "td",
+    "details",
+    "summary",
+]);
+const INLINE_TAGS = new Set(["a", "code", "strong", "em", "abbr", "time", "mark", "s", "del", "ins"]);
+const SEMANTIC_TAGS = new Set([...BLOCK_TAGS, ...INLINE_TAGS]);
+/** Tags whose subtrees should be flattened to a single text node. */
+const LEAF_CONTAINERS = new Set(["h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "td", "th", "figcaption", "summary"]);
+/** Tags where we want to preserve full child structure. */
+const BRANCH_CONTAINERS = new Set([
+    "pre",
+    "ul",
+    "ol",
+    "table",
+    "thead",
+    "tbody",
+    "tfoot",
+    "tr",
+    "section",
+    "article",
+    "aside",
+    "blockquote",
+    "details",
+    "figure",
+]);
+// ---------------------------------------------------------------------------
+// Tree building
+// ---------------------------------------------------------------------------
+/**
+ * Extract the language from a <code> or <pre> element's class attribute.
+ * Turndown and most syntax highlighters use class="language-typescript" etc.
+ */
+function extractLang(el) {
+    const cls = el.getAttribute("class") ?? "";
+    const m = /language-([a-zA-Z0-9_+-]+)/.exec(cls);
+    return m ? m[1] : undefined;
+}
+/** Flatten all descendant text content into one trimmed string. */
+function flattenText(el) {
+    return (el.textContent ?? "").replace(/\s+/g, " ").trim();
+}
+/**
+ * Recursively build a DOMNode from an Element.
+ * Returns null if the element has no meaningful content.
+ */
+function buildNode(el, pathPrefix, siblingIndex) {
+    const tag = el.tagName.toLowerCase();
+    // Compute path segment
+    const count = siblingIndex.get(tag) ?? 0;
+    siblingIndex.set(tag, count + 1);
+    const segment = count === 0 ? tag : `${tag}[${count}]`;
+    const path = pathPrefix ? `${pathPrefix}.${segment}` : segment;
+    // Attrs
+    const attrs = {};
+    if (tag === "a") {
+        const href = el.getAttribute("href");
+        if (href && !href.startsWith("javascript:"))
+            attrs.href = href;
+    }
+    if (tag === "code" || tag === "pre") {
+        const lang = extractLang(el) ?? extractLang(el.querySelector("code") ?? el);
+        if (lang)
+            attrs.lang = lang;
+    }
+    if (tag === "abbr") {
+        const title = el.getAttribute("title");
+        if (title)
+            attrs.title = title;
+    }
+    if (tag === "time") {
+        const dt = el.getAttribute("datetime");
+        if (dt)
+            attrs.datetime = dt;
+    }
+    // Leaf containers — flatten to text
+    if (LEAF_CONTAINERS.has(tag)) {
+        const text = flattenText(el);
+        if (!text)
+            return null;
+        return {
+            tag,
+            path,
+            text,
+            ...(Object.keys(attrs).length > 0 ? { attrs } : {}),
+        };
+    }
+    // pre — treat the entire block (including nested <code>) as one leaf
+    if (tag === "pre") {
+        const text = (el.textContent ?? "").trimEnd();
+        if (!text.trim())
+            return null;
+        return {
+            tag,
+            path,
+            text,
+            ...(Object.keys(attrs).length > 0 ? { attrs } : {}),
+        };
+    }
+    // Branch containers — recurse into children
+    const children = [];
+    const childIndex = new Map();
+    for (const child of Array.from(el.children)) {
+        const childTag = child.tagName.toLowerCase();
+        if (SEMANTIC_TAGS.has(childTag)) {
+            const node = buildNode(child, path, childIndex);
+            if (node)
+                children.push(node);
+        }
+        else {
+            // Non-semantic wrapper: collapse by recursing with the same path/index
+            const collapsed = collapseWrapper(child, path, childIndex);
+            children.push(...collapsed);
+        }
+    }
+    if (children.length === 0) {
+        // Branch with no semantic children — try as text leaf
+        const text = flattenText(el);
+        if (!text)
+            return null;
+        return { tag, path, text, ...(Object.keys(attrs).length > 0 ? { attrs } : {}) };
+    }
+    // Collapse single-child branches with the same tag family
+    if (children.length === 1 && !BRANCH_CONTAINERS.has(tag)) {
+        // Promote the child up, but keep the parent path
+        return children[0];
+    }
+    return {
+        tag,
+        path,
+        children,
+        ...(Object.keys(attrs).length > 0 ? { attrs } : {}),
+    };
+}
+/**
+ * Collapse a non-semantic wrapper element, returning its semantic descendants.
+ */
+function collapseWrapper(el, pathPrefix, siblingIndex) {
+    const results = [];
+    for (const child of Array.from(el.children)) {
+        const childTag = child.tagName.toLowerCase();
+        if (SEMANTIC_TAGS.has(childTag)) {
+            const node = buildNode(child, pathPrefix, siblingIndex);
+            if (node)
+                results.push(node);
+        }
+        else {
+            results.push(...collapseWrapper(child, pathPrefix, siblingIndex));
+        }
+    }
+    // If no semantic children found, treat wrapper text as a paragraph
+    if (results.length === 0) {
+        const text = flattenText(el);
+        if (text.length > 20) {
+            const count = siblingIndex.get("p") ?? 0;
+            siblingIndex.set("p", count + 1);
+            const segment = count === 0 ? "p" : `p[${count}]`;
+            results.push({ tag: "p", path: `${pathPrefix}.${segment}`, text });
+        }
+    }
+    return results;
+}
+/**
+ * Build a simplified semantic DOM tree from Readability article HTML.
+ *
+ * The root is always an "article" node. Presentational wrappers are collapsed,
+ * single-child chains are simplified, and only semantic tags survive.
+ */
+export function buildTree(articleHtml, baseUrl) {
+    const { document } = parseHTML(`<html><body>${articleHtml}</body></html>`, { url: baseUrl });
+    const body = document.body;
+    const children = [];
+    const siblingIndex = new Map();
+    for (const child of Array.from(body.children)) {
+        const childTag = child.tagName.toLowerCase();
+        if (SEMANTIC_TAGS.has(childTag)) {
+            const node = buildNode(child, "article", siblingIndex);
+            if (node)
+                children.push(node);
+        }
+        else {
+            const collapsed = collapseWrapper(child, "article", siblingIndex);
+            children.push(...collapsed);
+        }
+    }
+    return { tag: "article", path: "article", children };
+}
+// ---------------------------------------------------------------------------
+// Tree navigation
+// ---------------------------------------------------------------------------
+/** Collect all nodes in the tree as a flat list (depth-first). */
+function allNodes(node) {
+    const result = [node];
+    if (node.children) {
+        for (const child of node.children)
+            result.push(...allNodes(child));
+    }
+    return result;
+}
+/**
+ * Navigate to a specific node by its dot-bracket path.
+ * Returns null if the path does not exist in the tree.
+ *
+ * @example navigateTree(tree, "article.section[1].pre[0]")
+ */
+export function navigateTree(root, path) {
+    const nodes = allNodes(root);
+    return nodes.find((n) => n.path === path) ?? null;
+}
+// ---------------------------------------------------------------------------
+// Tree fuzzy search
+// ---------------------------------------------------------------------------
+/** Extract all text content from a node recursively. */
+function nodeText(node) {
+    if (node.text)
+        return node.text;
+    if (!node.children)
+        return "";
+    return node.children.map(nodeText).join(" ");
+}
+/** Semantic "block" tags that make good hit containers. */
+const HIT_CONTAINERS = new Set([
+    "section",
+    "article",
+    "aside",
+    "blockquote",
+    "details",
+    "li",
+    "pre",
+    "p",
+    "figure",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "tr",
+]);
+/**
+ * Score text against a query using token overlap + exact phrase bonus.
+ * Returns 0–1.
+ */
+function scoreText(text, queryTokens, fullQuery) {
+    if (!text)
+        return 0;
+    const lower = text.toLowerCase();
+    let score = lower.includes(fullQuery) ? 0.6 : 0;
+    const perToken = 0.4 / Math.max(queryTokens.length, 1);
+    for (const qt of queryTokens) {
+        if (lower.includes(qt))
+            score += perToken;
+    }
+    return Math.min(score, 1);
+}
+/** Build a short snippet around the best match position. */
+function buildSnippet(text, fullQuery, queryTokens, radius = 100) {
+    const lower = text.toLowerCase();
+    let pos = lower.indexOf(fullQuery);
+    if (pos === -1) {
+        for (const qt of queryTokens) {
+            const p = lower.indexOf(qt);
+            if (p !== -1) {
+                pos = p;
+                break;
+            }
+        }
+    }
+    if (pos === -1)
+        pos = 0;
+    const start = Math.max(0, pos - radius);
+    const end = Math.min(text.length, pos + Math.max(fullQuery.length, 1) + radius);
+    const raw = text.slice(start, end).replace(/\s+/g, " ").trim();
+    return (start > 0 ? "…" : "") + raw + (end < text.length ? "…" : "");
+}
+/**
+ * Fuzzy-search a DOM tree for a query string.
+ *
+ * Returns hits ranked by score. Each hit is the nearest semantic ancestor
+ * that contains the match (a section, li, pre, p — not a raw div). This
+ * means code blocks and table rows are always returned whole.
+ *
+ * @example
+ * const hits = queryTree(tree, "declaration merge", { topN: 3 })
+ * // hits[0].node is the full <section> containing that heading
+ */
+export function queryTree(root, query, opts = {}) {
+    const { topN = 10, snippetRadius = 100 } = opts;
+    if (!query.trim())
+        return [];
+    const fullQuery = query.trim().toLowerCase();
+    const queryTokens = fullQuery.split(/\s+/).filter((t) => t.length > 1);
+    const nodes = allNodes(root);
+    const hits = [];
+    for (const node of nodes) {
+        // Only return hit containers — not intermediate wrappers, not the root.
+        if (!HIT_CONTAINERS.has(node.tag))
+            continue;
+        if (node.path === "article")
+            continue; // root always matches everything — skip it
+        const text = nodeText(node);
+        const score = scoreText(text, queryTokens, fullQuery);
+        if (score === 0)
+            continue;
+        hits.push({
+            path: node.path,
+            score,
+            node,
+            snippet: buildSnippet(text, fullQuery, queryTokens, snippetRadius),
+        });
+    }
+    // Deduplicate: if a parent and child both match, keep only the more specific
+    // (higher-scoring) one. If scores are equal, prefer the ancestor (more context).
+    const deduped = hits
+        .sort((a, b) => b.score - a.score)
+        .filter((hit, i, arr) => {
+        // Remove this hit if a better-scoring ancestor is already in the list
+        return !arr.slice(0, i).some((other) => hit.path.startsWith(`${other.path}.`) && other.score >= hit.score);
+    });
+    return deduped.slice(0, topN);
+}
+//# sourceMappingURL=tree.js.map

package/dist/types.d.ts

@@ -0,0 +1,189 @@
+/** Selects how much content spider() returns. */
+export type PageView = "lean" | "full" | "tree";
+/**
+ * A single node in the simplified DOM tree.
+ *
+ * The tree is built from the Readability article HTML with all presentational
+ * wrapper elements collapsed. Only semantically meaningful tags survive.
+ * Single-child chains (div > div > p) are reduced to the leaf (p).
+ *
+ * Paths use bracket notation for siblings of the same tag:
+ *   "article.section[1].pre[0].code"
+ *
+ * Agents can:
+ *   - Read the tree to understand page structure without fetching full markdown.
+ *   - Call navigateTree(tree, path) to extract one exact node.
+ *   - Call queryTree(tree, query) to fuzzy-search and get matching subtrees.
+ */
+export interface DOMNode {
+    /** HTML tag name, lower-cased. */
+    tag: string;
+    /** Stable dot-bracket path from the tree root, e.g. "article.section[1].pre[0].code". */
+    path: string;
+    /**
+     * Text content of this node.
+     * For leaf nodes: the raw text. For branch nodes: concatenated descendant text.
+     * Omitted when the node has children to avoid duplication.
+     */
+    text?: string;
+    /**
+     * Semantically useful attributes only.
+     * a → href, code → lang (from class="language-*"), abbr → title.
+     */
+    attrs?: Record<string, string>;
+    /** Child nodes. Present on branch nodes, absent on leaves. */
+    children?: DOMNode[];
+}
+/** A hit returned by queryTree — a matching subtree with score and context. */
+export interface TreeHit {
+    /** Dot-bracket path of the matching node. */
+    path: string;
+    /** Score 0–1. Higher is a better match. */
+    score: number;
+    /** The matching node (may be a branch — e.g. a whole section). */
+    node: DOMNode;
+    /** Short context around the best match, ≤ 200 chars. */
+    snippet: string;
+}
+/** Dominant content type of a chunk — detected from the markdown buffer. */
+export type ChunkType = "text" | "code" | "table" | "list" | "blockquote";
+/** One embeddable, self-contained segment of a page. The unit of RAG. */
+export interface Chunk {
+    /** Stable reference: "<url>#chunk-<index>" */
+    id: string;
+    index: number;
+    /** Nearest ancestor heading, empty string if none */
+    heading: string;
+    /** Clean Markdown text */
+    text: string;
+    wordCount: number;
+    /** Dominant content type — lets agents skip code/table chunks when summarising. */
+    contentType: ChunkType;
+}
+/**
+ * A single image scraped from a page.
+ *
+ * Storage contract:
+ *   - base64 is populated when the image is small enough to store inline.
+ *   - filePath is populated when the image has been spilled to disk.
+ *   - At least one of base64 or filePath is present on a hydrated ImageRef.
+ *
+ * LLM wire format (works with OpenAI, Anthropic, Together, Gemini):
+ *   `data:${mimeType};base64,${base64}`
+ */
+export interface ImageRef {
+    /** Original absolute src URL of the image. */
+    src: string;
+    /** Base64-encoded image bytes. Omitted when the image is stored on disk. */
+    base64?: string;
+    /** MIME type detected from Content-Type or src extension, e.g. "image/jpeg". */
+    mimeType: string;
+    /** Alt text from the <img> tag, empty string when absent. */
+    alt: string;
+    /** Path to the binary file when the image has been persisted to disk. */
+    filePath?: string;
+}
+/** An outbound link — one edge in the knowledge graph. */
+export interface Link {
+    href: string;
+    text: string;
+    isExternal: boolean;
+    /**
+     * Where in the page the link was found.
+     * "body"  — inside the article content (strongest signal).
+     * "nav"   — inside nav, header, footer, or aside (navigation chrome).
+     */
+    rel: "body" | "nav";
+}
+/**
+ * Minimal link for lean views — isExternal omitted (inferable from the URL).
+ * Saves tokens when pages carry hundreds of links.
+ */
+export interface LeanLink {
+    href: string;
+    text: string;
+}
+/**
+ * Compact page view — identity, metadata, and structural outline only.
+ * No chunk text, no markdown body. Use when deciding whether/where to dig
+ * deeper. Roughly 5–20× fewer tokens than a full SpideredPage.
+ *
+ * Headings are flat markdown strings ("## Section") rather than objects —
+ * same information, ~half the tokens.
+ */
+export interface LeanPage {
+    readonly view: "lean";
+    url: string;
+    domain: string;
+    /** Canonical URL when it differs from the fetched URL (og:url / link[rel=canonical]). */
+    canonicalUrl?: string;
+    title: string;
+    description?: string;
+    author?: string;
+    publishedAt?: string;
+    lang: string;
+    /** Extracted topic tags — from meta keywords and article:tag. Compact vocabulary for grouping. */
+    tags: string[];
+    wordCount: number;
+    readingTimeMinutes: number;
+    /** How many RAG chunks a full view would produce. */
+    chunkCount: number;
+    /** Heading outline as flat markdown strings, e.g. "## Section Name". */
+    headings: string[];
+    /** Outbound links — href + anchor text only. */
+    links: LeanLink[];
+    /** True when the page appears JS-rendered — metadata may be partial. */
+    jsRendered?: boolean;
+    /**
+     * Number of other spidered pages that link to this page.
+     * Populated when a PageGraph is passed to toLean(). Omitted otherwise.
+     * Higher = more authoritative within the crawled corpus.
+     */
+    inboundCount?: number;
+}
+/**
+ * A fully spidered page.
+ *
+ * Follows the Local Materialized View rule: every field is a named,
+ * independently readable value — never a serialized blob. Agents read
+ * individual fields; RAG embeds individual chunks; graph walkers follow
+ * individual links.
+ */
+export interface SpideredPage {
+    url: string;
+    domain: string;
+    fetchedAt: string;
+    /** Canonical URL when it differs from the fetched URL (og:url / link[rel=canonical]). */
+    canonicalUrl?: string;
+    title: string;
+    description: string;
+    author: string;
+    publishedAt: string;
+    lang: string;
+    /** Extracted topic tags — from meta keywords and article:tag. */
+    tags: string[];
+    wordCount: number;
+    readingTimeMinutes: number;
+    /** Heading outline — h1/h2/h3 only */
+    headings: Array<{
+        level: 1 | 2 | 3;
+        text: string;
+    }>;
+    /** RAG-ready chunks */
+    chunks: Chunk[];
+    /** Outbound links from this page */
+    links: Link[];
+    /**
+     * Images scraped from the article content.
+     * Only populated when spider() is called with captureImages: true.
+     */
+    images?: ImageRef[];
+    markdown: string;
+    /**
+     * True when the page appears to be JavaScript-rendered (Readability
+     * found no content). metadata and links are still populated where
+     * possible; chunks and markdown are empty.
+     */
+    jsRendered?: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,iDAAiD;AACjD,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAMhD;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,OAAO;IACvB,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,yFAAyF;IACzF,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;CACrB;AAED,+EAA+E;AAC/E,MAAM,WAAW,OAAO;IACvB,6CAA6C;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,IAAI,EAAE,OAAO,CAAC;IACd,wDAAwD;IACxD,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,4EAA4E;AAC5E,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,YAAY,CAAC;AAE1E,yEAAyE;AACzE,MAAM,WAAW,KAAK;IACrB,8CAA8C;IAC9C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,OAAO,EAAE,MAAM,CAAC;IAChB,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,mFAAmF;IACnF,WAAW,EAAE,SAAS,CAAC;CACvB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,QAAQ;IACxB,8CAA8C;IAC9C,GAAG,EAAE,MAAM,CAAC;IACZ,4EAA4E;IAC5E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gFAAgF;IAChF,QAAQ,EAAE,MAAM,CAAC;IACjB,6DAA6D;IAC7D,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,0DAA0D;AAC1D,MAAM,WAAW,IAAI;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,GAAG,EAAE,MAAM,GAAG,KAAK,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,QAAQ;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAGtB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,kGAAkG;IAClG,IAAI,EAAE,MAAM,EAAE,CAAC;IAGf,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qDAAqD;IACrD,UAAU,EAAE,MAAM,CAAC;IAGnB,wEAAwE;IACxE,QAAQ,EAAE,MAAM,EAAE,CAAC;IAGnB,gDAAgD;IAChD,KAAK,EAAE,QAAQ,EAAE,CAAC;IAElB,wEAAwE;IACxE,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACtB;AAID;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAE5B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,yFAAyF;IACzF,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,iEAAiE;IACjE,IAAI,EAAE,MAAM,EAAE,CAAC;IAGf,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAG3B,sCAAsC;IACtC,QAAQ,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpD,uBAAuB;IACvB,MAAM,EAAE,KAAK,EAAE,CAAC;IAGhB,oCAAoC;IACpC,KAAK,EAAE,IAAI,EAAE,CAAC;IAGd;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGpB,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;CACrB"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/views.d.ts

@@ -0,0 +1,17 @@
+/**
+ * View transformations — business logic that converts a SpideredPage into
+ * one of the available view shapes. Separated from types.ts which is pure
+ * data-shape definitions.
+ */
+import type { PageGraph } from "./graph.js";
+import type { LeanPage, SpideredPage } from "./types.js";
+/**
+ * Downgrade a full SpideredPage to a LeanPage.
+ *
+ * Pass a PageGraph as the second argument to populate `inboundCount` —
+ * the number of other spidered pages that link to this one. Agents can
+ * use this as a lightweight authority signal when ranking results from
+ * a crawl without running a full PageRank pass.
+ */
+export declare function toLean(page: SpideredPage, graph?: PageGraph): LeanPage;
+//# sourceMappingURL=views.d.ts.map