@tforgach/axi-fetch 0.1.0

package/dist/fetcher.d.ts

@@ -0,0 +1,26 @@
+export interface FetchedPage {
+    html: string;
+    /** URL after any redirects. */
+    finalUrl: string;
+    status: number;
+    contentType: string;
+}
+export interface FetchUrlOptions {
+    timeout?: number;
+    userAgent?: string;
+    /** Use the on-disk response cache. Default: true. */
+    cache?: boolean;
+    /** Cache freshness window in ms. */
+    cacheTtl?: number;
+}
+/** Validate and normalize a user-supplied URL, defaulting the scheme to https. */
+export declare function normalizeUrl(input: string): URL;
+/** Find an HTML `<meta http-equiv="refresh">` target, resolved against `base`. */
+export declare function metaRefreshTarget(html: string, base: string): string | null;
+/**
+ * Fetch a URL as HTML. Follows HTTP redirects (native) and HTML meta-refresh
+ * redirects (manually, up to MAX_META_HOPS), decodes by charset, enforces a
+ * timeout, and fails loud with structured errors.
+ */
+export declare function fetchUrl(input: string, options?: FetchUrlOptions): Promise<FetchedPage>;
+//# sourceMappingURL=fetcher.d.ts.map

package/dist/fetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../src/fetcher.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,oCAAoC;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,kFAAkF;AAClF,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,CAyB/C;AA2BD,kFAAkF;AAClF,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAc3E;AA4FD;;;;GAIG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAuBtB"}

package/dist/fetcher.js

@@ -0,0 +1,170 @@
+import { AxiError } from "axi-sdk-js";
+import { readCache, writeCache } from "./cache.js";
+const DEFAULT_TIMEOUT = 10_000;
+const DEFAULT_USER_AGENT = "axi-fetch/0.1 (+https://github.com/travisforgach/axi-fetch)";
+// How many HTML meta-refresh hops to follow (native fetch handles HTTP 3xx).
+const MAX_META_HOPS = 3;
+// Retry transient failures (network blips, 429, 5xx) with exponential backoff.
+const MAX_RETRIES = 2;
+const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Validate and normalize a user-supplied URL, defaulting the scheme to https. */
+export function normalizeUrl(input) {
+    const trimmed = input.trim();
+    // Only default the scheme when none is present; an explicit non-http(s)
+    // scheme must fall through to the protocol check below and be rejected.
+    const hasScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed);
+    const withScheme = hasScheme ? trimmed : `https://${trimmed}`;
+    let url;
+    try {
+        url = new URL(withScheme);
+    }
+    catch {
+        throw new AxiError(`Invalid URL: ${input}`, "VALIDATION_ERROR", [
+            "Pass a valid http(s) URL, e.g. `axi-fetch https://example.com/article`",
+        ]);
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        throw new AxiError(`Unsupported URL scheme: ${url.protocol}`, "VALIDATION_ERROR", ["Only http and https URLs are supported"]);
+    }
+    return url;
+}
+/**
+ * Decode a response body honoring its charset. Native `Response.text()` assumes
+ * UTF-8; real-world pages are sometimes windows-1252, iso-8859-1, shift_jis, etc.
+ * Charset comes from the content-type header, or a `<meta charset>` in the head.
+ */
+function decodeBody(buffer, contentType) {
+    let charset = /charset=([^;]+)/i.exec(contentType)?.[1]?.trim().toLowerCase();
+    if (!charset) {
+        // Peek at the head bytes (ASCII-safe) for a <meta charset>.
+        const head = new TextDecoder("latin1").decode(buffer.slice(0, 2048));
+        charset =
+            /<meta[^>]+charset=["']?\s*([\w-]+)/i.exec(head)?.[1]?.toLowerCase() ??
+                undefined;
+    }
+    if (!charset || charset === "utf-8" || charset === "utf8") {
+        return new TextDecoder("utf-8").decode(buffer);
+    }
+    try {
+        return new TextDecoder(charset).decode(buffer);
+    }
+    catch {
+        // Unknown label (TextDecoder throws on unsupported encodings) — fall back.
+        return new TextDecoder("utf-8").decode(buffer);
+    }
+}
+/** Find an HTML `<meta http-equiv="refresh">` target, resolved against `base`. */
+export function metaRefreshTarget(html, base) {
+    for (const tag of html.match(/<meta\b[^>]*>/gi) ?? []) {
+        if (!/http-equiv\s*=\s*["']?\s*refresh/i.test(tag))
+            continue;
+        // Backreference so inner quotes (e.g. content="0; url='...'") don't truncate.
+        const content = /content\s*=\s*(["'])(.*?)\1/is.exec(tag)?.[2] ?? "";
+        const urlPart = /url\s*=\s*(.+)$/i.exec(content.trim())?.[1];
+        if (!urlPart)
+            continue;
+        try {
+            return new URL(urlPart.trim().replace(/^['"]|['"]$/g, ""), base).href;
+        }
+        catch {
+            return null;
+        }
+    }
+    return null;
+}
+/** Perform a single fetch, validate it, and decode the body by charset. */
+async function fetchOnce(input, timeout, userAgent) {
+    const url = normalizeUrl(input);
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+    let response;
+    try {
+        response = await fetch(url, {
+            redirect: "follow",
+            signal: controller.signal,
+            headers: {
+                "User-Agent": userAgent,
+                Accept: "text/html,application/xhtml+xml",
+            },
+        });
+    }
+    catch (error) {
+        if (error instanceof Error && error.name === "AbortError") {
+            throw new AxiError(`Request timed out after ${timeout}ms: ${url.href}`, "TIMEOUT", ["Increase the timeout with `--timeout <ms>`"]);
+        }
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new AxiError(`Failed to fetch ${url.href}: ${reason}`, "FETCH_FAILED");
+    }
+    finally {
+        clearTimeout(timer);
+    }
+    if (!response.ok) {
+        const where = `${response.status} ${response.statusText} for ${url.href}`;
+        if (response.status === 401 || response.status === 403) {
+            throw new AxiError(`HTTP ${where} — the site appears to be blocking automated requests`, "FORBIDDEN", [
+                "Some sites block bots; this usually can't be worked around from a plain fetch",
+            ]);
+        }
+        // 429/5xx are transient — code them so the retry wrapper knows to retry.
+        const code = RETRYABLE_STATUS.has(response.status) ? "SERVER_ERROR" : "HTTP_ERROR";
+        throw new AxiError(`HTTP ${where}`, code);
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    if (contentType && !/text\/html|application\/xhtml\+xml/i.test(contentType)) {
+        throw new AxiError(`Unsupported content type "${contentType}" for ${url.href}`, "NOT_HTML", ["axi-fetch only handles HTML pages in the MVP"]);
+    }
+    const html = decodeBody(await response.arrayBuffer(), contentType);
+    return { html, finalUrl: response.url || url.href, status: response.status, contentType };
+}
+/** Transient failures worth another attempt: network blips and 429/5xx. */
+function isRetryable(error) {
+    return (error instanceof AxiError &&
+        (error.code === "FETCH_FAILED" || error.code === "SERVER_ERROR"));
+}
+/** fetchOnce with exponential backoff on transient failures. */
+async function fetchWithRetry(input, timeout, userAgent) {
+    let lastError;
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+        try {
+            return await fetchOnce(input, timeout, userAgent);
+        }
+        catch (error) {
+            lastError = error;
+            if (!isRetryable(error) || attempt === MAX_RETRIES)
+                break;
+            await sleep(250 * 2 ** attempt);
+        }
+    }
+    throw lastError;
+}
+/**
+ * Fetch a URL as HTML. Follows HTTP redirects (native) and HTML meta-refresh
+ * redirects (manually, up to MAX_META_HOPS), decodes by charset, enforces a
+ * timeout, and fails loud with structured errors.
+ */
+export async function fetchUrl(input, options = {}) {
+    const timeout = options.timeout ?? DEFAULT_TIMEOUT;
+    const userAgent = options.userAgent ?? DEFAULT_USER_AGENT;
+    const useCache = options.cache ?? true;
+    // Canonical cache key (also validates the URL up front).
+    const key = normalizeUrl(input).href;
+    if (useCache) {
+        const cached = await readCache(key, options.cacheTtl);
+        if (cached)
+            return cached;
+    }
+    let current = input;
+    for (let hop = 0;; hop++) {
+        const page = await fetchWithRetry(current, timeout, userAgent);
+        const refresh = metaRefreshTarget(page.html, page.finalUrl);
+        if (refresh && refresh !== page.finalUrl && hop < MAX_META_HOPS) {
+            current = refresh;
+            continue;
+        }
+        if (useCache)
+            await writeCache(key, page);
+        return page;
+    }
+}
+//# sourceMappingURL=fetcher.js.map

package/dist/fetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../src/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEnD,MAAM,eAAe,GAAG,MAAM,CAAC;AAC/B,MAAM,kBAAkB,GACtB,6DAA6D,CAAC;AAChE,6EAA6E;AAC7E,MAAM,aAAa,GAAG,CAAC,CAAC;AACxB,+EAA+E;AAC/E,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEtE,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAmB1E,kFAAkF;AAClF,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,wEAAwE;IACxE,wEAAwE;IACxE,MAAM,SAAS,GAAG,0BAA0B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,OAAO,EAAE,CAAC;IAE9D,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,QAAQ,CAAC,gBAAgB,KAAK,EAAE,EAAE,kBAAkB,EAAE;YAC9D,wEAAwE;SACzE,CAAC,CAAC;IACL,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,QAAQ,CAChB,2BAA2B,GAAG,CAAC,QAAQ,EAAE,EACzC,kBAAkB,EAClB,CAAC,wCAAwC,CAAC,CAC3C,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,MAAmB,EAAE,WAAmB;IAC1D,IAAI,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9E,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,4DAA4D;QAC5D,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;QACrE,OAAO;YACL,qCAAqC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;gBACpE,SAAS,CAAC;IACd,CAAC;IACD,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,OAAO,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QAC1D,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,2EAA2E;QAC3E,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;AACH,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,IAAY;IAC1D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC;QACtD,IAAI,CAAC,mCAAmC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAC7D,8EAA8E;QAC9E,MAAM,OAAO,GAAG,+BAA+B,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;QACxE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2EAA2E;AAC3E,KAAK,UAAU,SAAS,CACtB,KAAa,EACb,OAAe,EACf,SAAiB;IAEjB,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IAE5D,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC1B,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,SAAS;gBACvB,MAAM,EAAE,iCAAiC;aAC1C;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1D,MAAM,IAAI,QAAQ,CAChB,2BAA2B,OAAO,OAAO,GAAG,CAAC,IAAI,EAAE,EACnD,SAAS,EACT,CAAC,4CAA4C,CAAC,CAC/C,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,MAAM,IAAI,QAAQ,CAAC,mBAAmB,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC;IAC/E,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;QAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvD,MAAM,IAAI,QAAQ,CAChB,QAAQ,KAAK,uDAAuD,EACpE,WAAW,EACX;gBACE,+EAA+E;aAChF,CACF,CAAC;QACJ,CAAC;QACD,yEAAyE;QACzE,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC;QACnF,MAAM,IAAI,QAAQ,CAAC,QAAQ,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,WAAW,IAAI,CAAC,qCAAqC,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,QAAQ,CAChB,6BAA6B,WAAW,SAAS,GAAG,CAAC,IAAI,EAAE,EAC3D,UAAU,EACV,CAAC,8CAA8C,CAAC,CACjD,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,EAAE,WAAW,CAAC,CAAC;IACnE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC;AAC5F,CAAC;AAED,2EAA2E;AAC3E,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,CACL,KAAK,YAAY,QAAQ;QACzB,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC,CACjE,CAAC;AACJ,CAAC;AAED,gEAAgE;AAChE,KAAK,UAAU,cAAc,CAC3B,KAAa,EACb,OAAe,EACf,SAAiB;IAEjB,IAAI,SAAkB,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAClB,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,WAAW;gBAAE,MAAM;YAC1D,MAAM,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,MAAM,SAAS,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,UAA2B,EAAE;IAE7B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;IACnD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC;IACvC,yDAAyD;IACzD,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC;IAErC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACtD,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,IAAI,GAAG,GAAG,CAAC,GAAI,GAAG,EAAE,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5D,IAAI,OAAO,IAAI,OAAO,KAAK,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG,aAAa,EAAE,CAAC;YAChE,OAAO,GAAG,OAAO,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,QAAQ;YAAE,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,18 @@
+import type { AxiFetchOptions, FetchResult } from "./types.js";
+/**
+ * Fetch a URL and return an agent-ready AXI response plus its TOON encoding.
+ *
+ * Pipeline: fetch -> detect type -> extract -> truncate -> build next steps.
+ * Never throws for expected failures upstream throw structured `AxiError`s.
+ */
+export declare function axiFetch(url: string, options?: AxiFetchOptions): Promise<FetchResult>;
+/**
+ * Build an AXI response from already-fetched HTML (no network). Useful for
+ * offline processing, fixtures, and benchmarking the same HTML across formats.
+ */
+export declare function extractFromHtml(html: string, url: string, options?: AxiFetchOptions): FetchResult;
+export { toStructured, toToon } from "./output.js";
+export { fetchUrl, normalizeUrl } from "./fetcher.js";
+export { detectType } from "./typeDetector.js";
+export * from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,eAAe,EAGf,WAAW,EACZ,MAAM,YAAY,CAAC;AAIpB;;;;;GAKG;AACH,wBAAsB,QAAQ,CAC5B,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAQtB;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,eAAoB,GAC5B,WAAW,CAyCb;AAeD,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,cAAc,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,81 @@
+import { fetchUrl } from "./fetcher.js";
+import { detectType } from "./typeDetector.js";
+import { extractArticle } from "./extractors/article.js";
+import { extractGeneric } from "./extractors/generic.js";
+import { toToon } from "./output.js";
+const DEFAULT_MAX_CONTENT_LENGTH = 1500;
+/**
+ * Fetch a URL and return an agent-ready AXI response plus its TOON encoding.
+ *
+ * Pipeline: fetch -> detect type -> extract -> truncate -> build next steps.
+ * Never throws for expected failures upstream throw structured `AxiError`s.
+ */
+export async function axiFetch(url, options = {}) {
+    const page = await fetchUrl(url, {
+        timeout: options.timeout,
+        userAgent: options.userAgent,
+        cache: options.cache,
+        cacheTtl: options.cacheTtl,
+    });
+    return extractFromHtml(page.html, page.finalUrl, options);
+}
+/**
+ * Build an AXI response from already-fetched HTML (no network). Useful for
+ * offline processing, fixtures, and benchmarking the same HTML across formats.
+ */
+export function extractFromHtml(html, url, options = {}) {
+    const includeLinks = options.includeLinks ?? true;
+    const maxContentLength = options.maxContentLength ?? DEFAULT_MAX_CONTENT_LENGTH;
+    const detection = detectType(html, url);
+    let title;
+    let content;
+    if (detection.type === "article") {
+        const article = extractArticle(html, url, includeLinks);
+        title = article.title;
+        content = article.content;
+    }
+    else {
+        const generic = extractGeneric(html, url, includeLinks);
+        title = generic.title;
+        content = generic.content;
+    }
+    const fullLength = content.main.length;
+    if (!options.full && fullLength > maxContentLength) {
+        content = {
+            ...content,
+            main: `${content.main.slice(0, maxContentLength)}…`,
+            truncated: true,
+        };
+    }
+    const axiResponse = {
+        metadata: {
+            url,
+            title,
+            type: detection.type,
+            confidence: detection.confidence,
+            fetchedAt: new Date().toISOString(),
+            contentLength: fullLength,
+        },
+        content,
+        nextSteps: buildNextSteps(content, includeLinks),
+    };
+    return { axiResponse, toonOutput: toToon(axiResponse) };
+}
+function buildNextSteps(content, includeLinks) {
+    const steps = [];
+    if (content.truncated) {
+        steps.push("Re-run with `--full` to get the complete content");
+    }
+    if (!includeLinks) {
+        steps.push("Re-run without `--no-links` to include outbound links");
+    }
+    else if (content.links.length > 0) {
+        steps.push("Follow a link above with `axi-fetch <url>` to go deeper");
+    }
+    return steps;
+}
+export { toStructured, toToon } from "./output.js";
+export { fetchUrl, normalizeUrl } from "./fetcher.js";
+export { detectType } from "./typeDetector.js";
+export * from "./types.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAgB,MAAM,EAAE,MAAM,aAAa,CAAC;AAQnD,MAAM,0BAA0B,GAAG,IAAI,CAAC;AAExC;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,UAA2B,EAAE;IAE7B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE;QAC/B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC,CAAC;IACH,OAAO,eAAe,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AAC5D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,IAAY,EACZ,GAAW,EACX,UAA2B,EAAE;IAE7B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;IAClD,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,0BAA0B,CAAC;IAEhF,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAExC,IAAI,KAAa,CAAC;IAClB,IAAI,OAAgB,CAAC;IACrB,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC;QACxD,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QACtB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAC5B,CAAC;SAAM,CAAC;QACN,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC;QACxD,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QACtB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAC5B,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;IACvC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,UAAU,GAAG,gBAAgB,EAAE,CAAC;QACnD,OAAO,GAAG;YACR,GAAG,OAAO;YACV,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,GAAG;YACnD,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAgB;QAC/B,QAAQ,EAAE;YACR,GAAG;YACH,KAAK;YACL,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,aAAa,EAAE,UAAU;SAC1B;QACD,OAAO;QACP,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,YAAY,CAAC;KACjD,CAAC;IAEF,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,cAAc,CAAC,OAAgB,EAAE,YAAqB;IAC7D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,CAAC;SAAM,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,cAAc,YAAY,CAAC"}

package/dist/output.d.ts

@@ -0,0 +1,14 @@
+import type { AxiResponse } from "./types.js";
+/** A plain object destined for TOON encoding (mirrors the SDK's shape). */
+export type AxiStructuredOutput = Record<string, unknown>;
+/**
+ * Map an AxiResponse to the flat-ish structure we hand to the TOON encoder.
+ * Field order is deliberate: identity first, then content, then aggregates and
+ * next steps. Low-value and empty fields are omitted to save tokens — `fetchedAt`
+ * is dropped entirely (rarely actionable), `truncated` only shows when true, and
+ * trivially small pages get a bare envelope.
+ */
+export declare function toStructured(response: AxiResponse): AxiStructuredOutput;
+/** Encode an AxiResponse as a TOON string (the agent-facing payload). */
+export declare function toToon(response: AxiResponse): string;
+//# sourceMappingURL=output.d.ts.map

package/dist/output.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../src/output.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,2EAA2E;AAC3E,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAM1D;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,WAAW,GAAG,mBAAmB,CA2CvE;AAED,yEAAyE;AACzE,wBAAgB,MAAM,CAAC,QAAQ,EAAE,WAAW,GAAG,MAAM,CAEpD"}

package/dist/output.js

@@ -0,0 +1,54 @@
+import { encode } from "@toon-format/toon";
+// Below this content size the fixed envelope (confidence, contentLength, …)
+// costs more than it's worth, so we emit only the essentials.
+const SMALL_PAGE_CHARS = 200;
+/**
+ * Map an AxiResponse to the flat-ish structure we hand to the TOON encoder.
+ * Field order is deliberate: identity first, then content, then aggregates and
+ * next steps. Low-value and empty fields are omitted to save tokens — `fetchedAt`
+ * is dropped entirely (rarely actionable), `truncated` only shows when true, and
+ * trivially small pages get a bare envelope.
+ */
+export function toStructured(response) {
+    const { metadata, content, nextSteps } = response;
+    const hasStructure = content.sections.length > 0 ||
+        content.codeBlocks.length > 0 ||
+        content.tables.length > 0 ||
+        content.links.length > 0;
+    const isSmall = metadata.contentLength < SMALL_PAGE_CHARS && !hasStructure;
+    const output = {
+        url: metadata.url,
+        title: metadata.title,
+        type: metadata.type,
+    };
+    // Quality/aggregate fields aren't worth the tokens on trivially small pages.
+    if (!isSmall) {
+        output.confidence = Number(metadata.confidence.toFixed(2));
+        output.contentLength = metadata.contentLength;
+    }
+    if (content.truncated) {
+        output.truncated = true;
+    }
+    output.content = content.main;
+    if (content.sections.length > 0) {
+        output.sections = content.sections;
+    }
+    if (content.codeBlocks.length > 0) {
+        output.codeBlocks = content.codeBlocks;
+    }
+    if (content.tables.length > 0) {
+        output.tables = content.tables;
+    }
+    if (content.links.length > 0) {
+        output.links = content.links;
+    }
+    if (nextSteps.length > 0) {
+        output.help = nextSteps;
+    }
+    return output;
+}
+/** Encode an AxiResponse as a TOON string (the agent-facing payload). */
+export function toToon(response) {
+    return encode(toStructured(response));
+}
+//# sourceMappingURL=output.js.map

package/dist/output.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"output.js","sourceRoot":"","sources":["../src/output.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAM3C,4EAA4E;AAC5E,8DAA8D;AAC9D,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,QAAqB;IAChD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC;IAElD,MAAM,YAAY,GAChB,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;QAC3B,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;QAC7B,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,GAAG,gBAAgB,IAAI,CAAC,YAAY,CAAC;IAE3E,MAAM,MAAM,GAAwB;QAClC,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,IAAI,EAAE,QAAQ,CAAC,IAAI;KACpB,CAAC;IAEF,6EAA6E;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC;IAChD,CAAC;IACD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;IAC1B,CAAC;IACD,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAE9B,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IACrC,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IACzC,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC/B,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC;IAC1B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,MAAM,CAAC,QAAqB;IAC1C,OAAO,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;AACxC,CAAC"}

package/dist/typeDetector.d.ts

@@ -0,0 +1,14 @@
+import type { PageType } from "./types.js";
+export interface TypeDetection {
+    type: PageType;
+    /** 0-1 confidence in the detected type. */
+    confidence: number;
+    /** Human-readable signals that drove the decision (for debugging). */
+    hints: string[];
+}
+/**
+ * Rules-based (no LLM) detection of documentation, article, or generic pages.
+ * Signals are additive; confidence reflects how many fired.
+ */
+export declare function detectType(html: string, url: string): TypeDetection;
+//# sourceMappingURL=typeDetector.d.ts.map

package/dist/typeDetector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"typeDetector.d.ts","sourceRoot":"","sources":["../src/typeDetector.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,QAAQ,CAAC;IACf,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AA6CD;;;GAGG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,CA+DnE"}

package/dist/typeDetector.js

@@ -0,0 +1,98 @@
+import * as cheerio from "cheerio";
+const ARTICLE_PATH = /\/(blog|article|articles|news|post|posts|story|p)\//i;
+const ARTICLE_SCHEMA = /(News|Blog|Tech|Scholarly)?Article|BlogPosting/i;
+const DOC_HOST = /^(docs?|developer|devcenter|devdocs|api|readthedocs)\./i;
+const DOC_PATH = /\/(docs?|documentation|reference|guide|guides|manual|handbook|api)(\/|$)/i;
+/**
+ * Documentation detection runs first: docs pages are structurally distinct
+ * (URL conventions + code-heavy) and shouldn't be mislabeled as generic.
+ */
+function detectDocumentation($, url) {
+    const hints = [];
+    let score = 0;
+    try {
+        const parsed = new URL(url);
+        if (DOC_HOST.test(parsed.hostname)) {
+            score += 2;
+            hints.push("docs-host");
+        }
+        if (DOC_PATH.test(parsed.pathname)) {
+            score += 2;
+            hints.push("docs-path");
+        }
+    }
+    catch {
+        // ignore malformed url
+    }
+    // Code-heavy pages are a strong documentation signal.
+    const codeBlocks = $("pre").length;
+    if (codeBlocks >= 3) {
+        score += 2;
+        hints.push("code-heavy");
+    }
+    else if (codeBlocks >= 1) {
+        score += 1;
+        hints.push("has-code");
+    }
+    if (score >= 3) {
+        return { type: "documentation", confidence: Math.min(0.5 + score * 0.1, 0.97), hints };
+    }
+    return null;
+}
+/**
+ * Rules-based (no LLM) detection of documentation, article, or generic pages.
+ * Signals are additive; confidence reflects how many fired.
+ */
+export function detectType(html, url) {
+    const $ = cheerio.load(html);
+    const docs = detectDocumentation($, url);
+    if (docs)
+        return docs;
+    const hints = [];
+    let score = 0;
+    // URL path signal.
+    try {
+        if (ARTICLE_PATH.test(new URL(url).pathname)) {
+            score += 2;
+            hints.push("url-path");
+        }
+    }
+    catch {
+        // ignore malformed url, other signals still apply
+    }
+    // Open Graph type.
+    if ($('meta[property="og:type"]').attr("content")?.toLowerCase() === "article") {
+        score += 3;
+        hints.push("og:type=article");
+    }
+    // Semantic <article> element.
+    if ($("article").length > 0) {
+        score += 2;
+        hints.push("article-tag");
+    }
+    // schema.org markup (JSON-LD @type or microdata itemtype).
+    const ldTypes = $('script[type="application/ld+json"]')
+        .map((_, el) => $(el).text())
+        .get()
+        .join(" ");
+    if (ARTICLE_SCHEMA.test(ldTypes) ||
+        $('[itemtype*="Article" i], [itemtype*="BlogPosting" i]').length > 0) {
+        score += 3;
+        hints.push("schema.org-article");
+    }
+    // Published-date signals.
+    if ($("time[datetime]").length > 0 ||
+        $('meta[property="article:published_time"]').length > 0) {
+        score += 1;
+        hints.push("published-date");
+    }
+    // Map the additive score to a type + confidence.
+    if (score >= 3) {
+        return { type: "article", confidence: Math.min(0.5 + score * 0.1, 0.98), hints };
+    }
+    if (score > 0) {
+        return { type: "article", confidence: 0.4 + score * 0.05, hints };
+    }
+    return { type: "generic", confidence: 0.6, hints: ["no-article-signals"] };
+}
+//# sourceMappingURL=typeDetector.js.map

package/dist/typeDetector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"typeDetector.js","sourceRoot":"","sources":["../src/typeDetector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAWnC,MAAM,YAAY,GAAG,sDAAsD,CAAC;AAC5E,MAAM,cAAc,GAAG,iDAAiD,CAAC;AACzE,MAAM,QAAQ,GAAG,yDAAyD,CAAC;AAC3E,MAAM,QAAQ,GAAG,2EAA2E,CAAC;AAE7F;;;GAGG;AACH,SAAS,mBAAmB,CAAC,CAAqB,EAAE,GAAW;IAC7D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC1B,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,uBAAuB;IACzB,CAAC;IAED,sDAAsD;IACtD,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IACnC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACpB,KAAK,IAAI,CAAC,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3B,CAAC;SAAM,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QAC3B,KAAK,IAAI,CAAC,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzB,CAAC;IAED,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,KAAK,GAAG,GAAG,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC;IACzF,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,GAAW;IAClD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACzC,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC;IAEtB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,mBAAmB;IACnB,IAAI,CAAC;QACH,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7C,KAAK,IAAI,CAAC,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,kDAAkD;IACpD,CAAC;IAED,mBAAmB;IACnB,IACE,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,KAAK,SAAS,EAC1E,CAAC;QACD,KAAK,IAAI,CAAC,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAChC,CAAC;IAED,8BAA8B;IAC9B,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC5B,CAAC;IAED,2DAA2D;IAC3D,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC;SACpD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;SAC5B,GAAG,EAAE;SACL,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,IACE,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC;QAC5B,CAAC,CAAC,sDAAsD,CAAC,CAAC,MAAM,GAAG,CAAC,EACpE,CAAC;QACD,KAAK,IAAI,CAAC,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACnC,CAAC;IAED,0BAA0B;IAC1B,IACE,CAAC,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,CAAC;QAC9B,CAAC,CAAC,yCAAyC,CAAC,CAAC,MAAM,GAAG,CAAC,EACvD,CAAC;QACD,KAAK,IAAI,CAAC,CAAC;QACX,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC/B,CAAC;IAED,iDAAiD;IACjD,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,KAAK,GAAG,GAAG,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC;IACnF,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,GAAG,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;IACpE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,oBAAoB,CAAC,EAAE,CAAC;AAC7E,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Page categories the MVP can extract. Kept intentionally small (AXI principle 2:
+ * minimal schemas). More types (pricing, product) are Phase 2.
+ */
+export type PageType = "article" | "documentation" | "generic";
+export interface AxiFetchOptions {
+    /** Include outbound links in the response. Default: true. */
+    includeLinks?: boolean;
+    /** Network timeout in milliseconds. Default: 10000. */
+    timeout?: number;
+    /**
+     * Max characters of main content before truncation (AXI principle 3).
+     * Default: 1500. Ignored when `full` is true.
+     */
+    maxContentLength?: number;
+    /** Bypass truncation and return full content (the `--full` escape hatch). */
+    full?: boolean;
+    /** Override the User-Agent header. */
+    userAgent?: string;
+    /** Use the on-disk response cache. Default: true. */
+    cache?: boolean;
+    /** Cache freshness window in ms. Default: 15 minutes. */
+    cacheTtl?: number;
+}
+export interface Metadata {
+    url: string;
+    title: string;
+    type: PageType;
+    /** Type-detection confidence, 0-1. */
+    confidence: number;
+    /** ISO timestamp of when the fetch completed. */
+    fetchedAt: string;
+    /** Length (chars) of the full extracted main content, before truncation. */
+    contentLength: number;
+}
+export interface Link {
+    text: string;
+    url: string;
+    kind: "internal" | "external";
+}
+/** A heading in the content outline. Text lives in `Content.main`, not here. */
+export interface Section {
+    heading: string;
+    level: number;
+}
+/** A fenced/`<pre>` code block pulled out of the prose. */
+export interface CodeBlock {
+    /** Detected language (from class hints), if any. */
+    language?: string;
+    code: string;
+}
+/** A tabular block pulled out of the prose (strong TOON fit). */
+export interface Table {
+    headers: string[];
+    rows: string[][];
+}
+export interface Content {
+    /** Primary extracted prose, possibly truncated (see `truncated`). */
+    main: string;
+    /** True when `main` was truncated to `maxContentLength`. */
+    truncated: boolean;
+    /** Heading outline of the page, if any. */
+    sections: Section[];
+    /** Code blocks lifted out of the prose. */
+    codeBlocks: CodeBlock[];
+    /** Data tables lifted out of the prose. */
+    tables: Table[];
+    /** Relevant outbound links, if `includeLinks` was set. */
+    links: Link[];
+}
+export interface AxiResponse {
+    metadata: Metadata;
+    content: Content;
+    /** Contextual next-step suggestions for the agent (AXI principle 9). */
+    nextSteps: string[];
+}
+export interface FetchResult {
+    /** Structured response object (keep internal logic on plain objects). */
+    axiResponse: AxiResponse;
+    /** TOON-encoded rendering of the response (the agent-facing payload). */
+    toonOutput: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,QAAQ,GAAG,SAAS,GAAG,eAAe,GAAG,SAAS,CAAC;AAE/D,MAAM,WAAW,eAAe;IAC9B,6DAA6D;IAC7D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,6EAA6E;IAC7E,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,yDAAyD;IACzD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,QAAQ;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,UAAU,GAAG,UAAU,CAAC;CAC/B;AAED,gFAAgF;AAChF,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,2DAA2D;AAC3D,MAAM,WAAW,SAAS;IACxB,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,iEAAiE;AACjE,MAAM,WAAW,KAAK;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,OAAO;IACtB,qEAAqE;IACrE,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,SAAS,EAAE,OAAO,CAAC;IACnB,2CAA2C;IAC3C,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,2CAA2C;IAC3C,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,2CAA2C;IAC3C,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,0DAA0D;IAC1D,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,wEAAwE;IACxE,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,yEAAyE;IACzE,WAAW,EAAE,WAAW,CAAC;IACzB,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;CACpB"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}