@autorag/librarian 2.5.1 → 2.5.2

package/dist/minsync/types.d.ts

@@ -50,4 +50,11 @@ export interface MinSyncQueryHit {
     readonly path: string;
     readonly score: number;
     readonly text: string;
+    /**
+     * MinSync's per-chunk identity (`doc_id` in its JSON). Several chunks of one
+     * parsed mirror share a `path`, so this is what distinguishes them; without
+     * it every passage of a document collapses into a single evidence id.
+     * Optional because older MinSync builds omit it.
+     */
+    readonly docId?: string;
 }

package/dist/retrieval/engine.d.ts

@@ -15,7 +15,7 @@
  */
 import { DatasourceAccessContext, type DatasourceAccessContextOptions } from "../datasource/access-context.js";
 import { RetrievalMethodRegistry } from "./registry.js";
-import type { RetrievalDiagnostic, RetrievalMethod, RetrievalOptions, RetrievalResult } from "./types.js";
+import type { RetrievalDiagnostic, RetrievalMethod, RetrievalOptions, RetrievalResult, RetrievalUnsearchedSurface } from "./types.js";
 /** Options for constructing a standalone {@link RetrievalEngine}. */
 export interface RetrievalEngineOptions {
     /**
@@ -23,7 +23,12 @@ export interface RetrievalEngineOptions {
      * @default { allowedTags: undefined, allowedScopes: undefined }
      */
     readonly datasourceAccess?: DatasourceAccessContextOptions;
-    /** Default `topK` when the caller omits it. @default 20 */
+    /**
+     * Default `topK` when the caller omits it. This is a safety ceiling on
+     * merged evidence, not a relevance cut: the merger returns every distinct
+     * chunk, so the effective size is whatever the registered methods returned.
+     * @default 500
+     */
     readonly defaultTopK?: number;
     /** Default deduplication flag. @default true */
     readonly defaultDedup?: boolean;
@@ -44,7 +49,7 @@ export interface RetrievalEngineOptions {
  * methods = registry.list()
  * byMethod = retriever.retrieveWithDiagnostics(methods, query, options)
  * filtered = filter.filter(byMethod, methods, ctx, options.scope)
- * merged = merger.merge(filtered, { topK, dedup: true })
+ * merged = merger.merge(filtered, { topK, dedup: true })  // distinct chunks, pure duplicates dropped
  * → { results: merged, diagnostics }
  * ```
  *
@@ -93,6 +98,7 @@ export declare class RetrievalEngine {
     retrieve(query: string, options?: RetrievalOptions): Promise<{
         results: RetrievalResult[];
         diagnostics: RetrievalDiagnostic[];
+        unsearched: RetrievalUnsearchedSurface[];
     }>;
     /**
      * Run retrieval and return results keyed per method (unmerged) with
@@ -102,6 +108,7 @@ export declare class RetrievalEngine {
     retrieveByMethod(query: string, options?: RetrievalOptions): Promise<{
         byMethod: Map<string, RetrievalResult[]>;
         diagnostics: RetrievalDiagnostic[];
+        unsearched: RetrievalUnsearchedSurface[];
     }>;
     /**
      * Post-check for MinSync binary-missing diagnostic.
@@ -111,9 +118,11 @@ export declare class RetrievalEngine {
      * a diagnostic. This method checks whether the minsync binary is unavailable
      * and emits a `minsync-unavailable` diagnostic when every minsync method's
      * result set is empty, exactly matching the existing {@link AutoRAGAgent}
-     * behavior (see `agent.src/agent/agent.ts` `retrieveWithDiagnostics`).
+     * behavior (see `agent.src/agent/agent.ts` `retrieveWithDiagnostics`). The
+     * same check reports the MinSync surface as unsearched, so a caller reading
+     * only the skip report still learns that local sources were not queried.
      */
-    private appendMinSyncUnavailableDiagnostic;
+    private appendMinSyncUnavailable;
     /**
      * Build a datasource access context scoped to the caller's options.
      * Model/tool arguments (allowedTags, allowedScopes) from options can only

package/dist/retrieval/index.d.ts

@@ -3,4 +3,5 @@ export type { MergeOptions } from "./merger.js";
 export { ParallelRetriever, ResultMerger } from "./merger.js";
 export { RetrievalMethodRegistry } from "./registry.js";
 export { matchesVirtualPathScope, normalizeVirtualPath, normalizeVirtualPathScope, virtualPathScopeToRegExp, } from "./scope.js";
-export type { CuratedResult, NumberedResult, RetrievalDiagnostic, RetrievalDiagnosticCode, RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult, RetrievalWithDiagnostics, } from "./types.js";
+export { describeRetrievalError, groupUnsearchedSurfaces, MINSYNC_SURFACE, type RetrievalSkip, retrievalSurfaceFor, } from "./skip.js";
+export type { CuratedResult, NumberedResult, RetrievalDiagnostic, RetrievalDiagnosticCode, RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult, RetrievalUnsearchedSurface, RetrievalWithDiagnostics, } from "./types.js";

package/dist/retrieval/merger.d.ts

@@ -1,19 +1,44 @@
 import type { RetrievalMethod, RetrievalOptions, RetrievalResult, RetrievalWithDiagnostics } from "./types.js";
 export interface MergeOptions {
+    /**
+     * Upper bound on returned evidence. This is a safety ceiling, not a
+     * relevance filter: the merger is expected to return every distinct chunk
+     * it was given. Callers that want the full pool pass a large value.
+     */
     topK: number;
-    /** Retained for API compatibility; exact-source aggregation is always enabled. */
+    /** Retained for API compatibility; pure-duplicate removal is always enabled. */
     dedup: boolean;
 }
+/**
+ * Cross-method evidence collator.
+ *
+ * `search_all_documents` is the *exhaustive* retrieval surface: it fans out to
+ * every configured method and the librarian is expected to judge the evidence
+ * itself. Collapsing that fan-out to one chunk per source threw away most of
+ * what was retrieved — a document whose several passages all matched surfaced
+ * only its single best chunk, and a datasource that returned twenty relevant
+ * messages could be reduced to one. Modern context windows hold the full pool
+ * (a live 16-method run over this repository's own corpus measured ~26k tokens
+ * on average, ~53k at its worst), so withholding it bought nothing.
+ *
+ * This merger therefore preserves every distinct chunk and removes only *pure*
+ * duplicates: the same evidence identity (`source` + `id`) seen more than once.
+ * Ranking still normalizes each method's scores into a comparable 0..1 band and
+ * gives a bounded bonus to sources corroborated by several chunks or methods,
+ * so the strongest evidence stays at the top of a complete list.
+ */
 export declare class ResultMerger {
     merge(results: Map<string, RetrievalResult[]>, options: MergeOptions): RetrievalResult[];
 }
 export declare class ParallelRetriever {
     retrieve(methods: RetrievalMethod[], query: string, options: RetrievalOptions): Promise<Map<string, RetrievalResult[]>>;
     /**
-     * Like {@link retrieve} but also returns path-opaque diagnostics for methods
-     * that failed. Partial results from healthy methods are preserved; failed
-     * methods yield an empty result set plus a diagnostic. The legacy
-     * {@link retrieve} return shape is intentionally unchanged for compatibility.
+     * Like {@link retrieve} but also returns diagnostics for methods that failed,
+     * plus the retrieval surfaces those methods belong to. Partial results from
+     * healthy methods are preserved; a failed method yields an empty result set,
+     * a diagnostic quoting the underlying error, and an entry in `unsearched`
+     * carrying that error verbatim. The legacy {@link retrieve} return shape is
+     * intentionally unchanged for compatibility.
      */
     retrieveWithDiagnostics(methods: RetrievalMethod[], query: string, options: RetrievalOptions): Promise<RetrievalWithDiagnostics>;
 }

package/dist/retrieval/skip.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Skip reporting for the retrieval pipeline.
+ *
+ * A method that throws is skipped, not fatal: healthy methods still answer. The
+ * helpers here turn those skips into a first-class report of the retrieval
+ * *surfaces* (local MinSync files, a datasource) that were not searched.
+ *
+ * The reason is the underlying error, verbatim. AutoRAG does not classify,
+ * rewrite, or suppress it: the operator searching their own machine is the one
+ * who has to debug the failure, so they get the CLI's own words — exit codes,
+ * stderr, paths and all.
+ */
+import type { RetrievalMethodDescriptor, RetrievalUnsearchedSurface } from "./types.js";
+/** Surface label shared by every MinSync-backed local method (vector, BM25, hybrid). */
+export declare const MINSYNC_SURFACE = "minsync";
+/** One method that did not run, with the error that stopped it. */
+export interface RetrievalSkip {
+    readonly method: string;
+    readonly surface: string;
+    /** The underlying failure, verbatim. */
+    readonly reason: string;
+}
+/**
+ * Map a method descriptor onto the surface a caller reasons about: datasource
+ * methods keep their datasource id, MinSync-backed local methods collapse into
+ * one shared surface, and anything else reports under its own method name.
+ */
+export declare function retrievalSurfaceFor(descriptor: RetrievalMethodDescriptor): string;
+/**
+ * Render a thrown value as the text a human needs to debug it. Nothing is
+ * stripped: whatever the method (or the CLI behind it) said comes through,
+ * including paths and exit status. The JS stack is left out because it points at
+ * AutoRAG's own frames, not at the failure the operator has to fix.
+ */
+export declare function describeRetrievalError(error: unknown): string;
+/**
+ * Collapse per-method skips into one entry per surface. Distinct failures on
+ * the same surface are all preserved verbatim in the reason; methods are
+ * merged. Deterministically ordered.
+ */
+export declare function groupUnsearchedSurfaces(skips: readonly RetrievalSkip[]): RetrievalUnsearchedSurface[];

package/dist/retrieval/types.d.ts

@@ -27,17 +27,38 @@ export interface RetrievalMethod {
     retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
 }
 export type RetrievalDiagnosticCode = "retrieval-method-failed" | "minsync-unavailable";
-/** Path-opaque diagnostic emitted by the multi-method retrieval pipeline. */
+/**
+ * A retrieval surface (local MinSync files or one datasource) that was not
+ * searched for this query. Reported next to partial results so a caller that
+ * reads only the result list can still tell the answer is incomplete.
+ */
+export interface RetrievalUnsearchedSurface {
+    /** Surface label: "minsync" for local files, or the datasource id. */
+    surface: string;
+    /** Registered retrieval method names on this surface that did not run. */
+    methods: string[];
+    /**
+     * Why the surface did not run: the underlying error verbatim (CLI stderr,
+     * exit status, paths included). Never classified or suppressed — the operator
+     * debugging their own machine needs the real text.
+     */
+    reason: string;
+}
+/** Diagnostic emitted by the multi-method retrieval pipeline. */
 export interface RetrievalDiagnostic {
     code: RetrievalDiagnosticCode;
     severity: "info" | "warning" | "error";
     message: string;
-    /** Component/method label — never a real filesystem path. */
+    /** Component/method label. */
     source?: string;
+    /** The underlying failure verbatim when the diagnostic reports a method that did not run. */
+    reason?: string;
 }
 export interface RetrievalWithDiagnostics {
     results: Map<string, RetrievalResult[]>;
     diagnostics: RetrievalDiagnostic[];
+    /** Surfaces that were not searched for this query. Empty when every method ran. */
+    unsearched: RetrievalUnsearchedSurface[];
 }
 export interface NumberedResult {
     index: number;

package/dist/web/entities.d.ts

@@ -0,0 +1,10 @@
+/**
+ * HTML entity decoding shared by the web search providers and the fetch
+ * pipeline.
+ *
+ * Single-pass by construction: one alternation regex decodes each entity
+ * exactly once, so an entity produced by decoding (`<` → `<`) is
+ * never re-decoded into markup (CodeQL js/double-escaping). Unknown entities
+ * and bare ampersands pass through untouched.
+ */
+export declare function decodeHtmlEntities(value: string): string;

package/dist/web/fetch/convert.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Binary download and conversion — adapted from oh-my-pi's
+ * `packages/coding-agent/src/web/scrapers/utils.ts` (MIT licensed).
+ *
+ * `convertBinaryPayload` uses AutoRAG's parser registry for PDF/DOCX/PPTX/XLSX.
+ * EPUB and other unsupported extensions fall through to the binary-notice
+ * fallback, since AutoRAG's default registry has no EPUB parser. This is a
+ * deliberate deviation from oh-my-pi (which uses markit for all convertible
+ * types including EPUB) — see the report for rationale.
+ */
+import { buildBinaryNotice } from "./url-target.js";
+export interface BinaryFetchSuccess {
+    ok: true;
+    buffer: Uint8Array;
+    contentDisposition?: string;
+}
+export type BinaryFetchResult = BinaryFetchSuccess | {
+    ok: false;
+    error?: string;
+};
+/**
+ * Fetch binary content from a URL with a size limit.
+ */
+export declare function fetchBinary(url: string, timeout?: number, signal?: AbortSignal, fetchImpl?: typeof fetch): Promise<BinaryFetchResult>;
+export interface ConvertResult {
+    content: string;
+    ok: boolean;
+    error?: string;
+}
+/**
+ * Convert a binary payload to markdown using AutoRAG's parser registry.
+ * Falls back to the binary-notice string when no parser handles the extension.
+ */
+export declare function convertBinaryPayload(buffer: Uint8Array, extension: string, _finalUrl: string, signal?: AbortSignal): Promise<ConvertResult>;
+export { buildBinaryNotice };

package/dist/web/fetch/feeds.d.ts

@@ -0,0 +1,37 @@
+/**
+ * RSS/Atom feed parsing and JSON formatting — ported from oh-my-pi's
+ * `packages/coding-agent/src/tools/fetch.ts` (MIT licensed).
+ *
+ * Feed parsing uses `fast-xml-parser` (already an AutoRAG dependency) instead
+ * of oh-my-pi's `parseHTML`/DOM utils, which are not available here.
+ */
+/**
+ * Read a single HTML attribute from a tag string.
+ */
+export declare function getHtmlAttribute(tag: string, attribute: string): string | null;
+/**
+ * Extract bounded `<head>` markup to avoid expensive whole-page parsing.
+ */
+export declare function extractHeadHtml(html: string): string;
+/**
+ * Parse alternate links from HTML head.
+ */
+export declare function parseAlternateLinks(html: string, pageUrl: string): string[];
+/**
+ * Extract document links from HTML (for PDF/DOCX wrapper pages).
+ */
+export declare function extractDocumentLinks(html: string, baseUrl: string): string[];
+/**
+ * Strip a CDATA wrapper and clean text. Only a full-string wrapper is
+ * unwrapped (anchored): a stray `]]>` inside content is data, not markup.
+ * Entity decoding is single-pass (see `../entities.ts`).
+ */
+export declare function cleanFeedText(text: string): string;
+/**
+ * Parse RSS/Atom feed XML to markdown using fast-xml-parser.
+ */
+export declare function parseFeedToMarkdown(content: string, maxItems?: number): string;
+/**
+ * Pretty-print JSON content.
+ */
+export declare function formatJson(content: string): string;

package/dist/web/fetch/html-renderer.d.ts

@@ -0,0 +1,54 @@
+/**
+ * HTML rendering backend chain — ported from oh-my-pi's
+ * `packages/coding-agent/src/tools/fetch.ts` (MIT licensed).
+ *
+ * FetchProvider order: native → lynx → firecrawl → jina.
+ * trafilatura is dropped (requires oh-my-pi's pip ensureTool).
+ * parallel is dropped (requires oh-my-pi's keyed client port).
+ * lynx is used only when found on PATH via a `which` check.
+ * firecrawl via FIRECRAWL_API_KEY env.
+ * jina via keyless https://r.jina.ai/ with optional JINA_API_KEY.
+ */
+export type FetchProvider = "native" | "lynx" | "firecrawl" | "jina";
+export interface RenderHtmlToTextOptions {
+    /** Overall render budget in seconds (default 30). */
+    timeoutSeconds?: number;
+    signal?: AbortSignal;
+    fetch?: typeof fetch;
+    firecrawlApiKey?: string;
+    jinaApiKey?: string;
+    /**
+     * Inject custom backend runners for testing. Keys must cover every
+     * FetchProvider in FETCH_PROVIDER_ORDER. A runner returns the rendered
+     * markdown or null to signal "not available / low quality / error."
+     */
+    runners?: Record<FetchProvider, () => Promise<string | null>>;
+}
+/**
+ * Parse the `Markdown Content:` marker from a Jina Reader response body.
+ */
+export declare function parseJinaReaderContent(responseBody: string): string | null;
+/**
+ * Check if rendered output looks JS-gated or mostly navigation.
+ */
+export declare function isLowQualityOutput(content: string): boolean;
+/**
+ * Convert HTML to markdown using the vendored Turndown with GFM support.
+ * Script/style elements are dropped by a parser-level turndown rule — never
+ * by regex pre-stripping (CodeQL js/bad-tag-filter,
+ * js/incomplete-multi-character-sanitization).
+ */
+export declare function htmlToBasicMarkdown(html: string): string;
+/**
+ * Render HTML to markdown by trying reader backends in priority order:
+ * native (in-process), lynx, firecrawl, then jina.
+ *
+ * Every backend's output must clear the quality gate (>100 non-whitespace
+ * chars and not isLowQualityOutput) before it is accepted; otherwise the
+ * next backend is tried.
+ */
+export declare function renderHtmlToText(url: string, html: string, options: RenderHtmlToTextOptions): Promise<{
+    content: string;
+    ok: boolean;
+    method: FetchProvider | "none";
+}>;

package/dist/web/fetch/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * URL fetch/render pipeline — self-contained module backing the future
+ * `web_fetch` agent tool. Ported from oh-my-pi (MIT licensed).
+ */
+export { type BinaryFetchResult, type ConvertResult, convertBinaryPayload, fetchBinary } from "./convert.js";
+export { cleanFeedText, extractDocumentLinks, extractHeadHtml, formatJson, getHtmlAttribute, parseAlternateLinks, parseFeedToMarkdown, } from "./feeds.js";
+export { type FetchProvider, htmlToBasicMarkdown, isLowQualityOutput, parseJinaReaderContent, type RenderHtmlToTextOptions, renderHtmlToText, } from "./html-renderer.js";
+export { buildLlmEndpointCandidates, tryContentNegotiation, tryLlmEndpoints, tryMdSuffix } from "./negotiate.js";
+export type { LoadPageOptions, LoadPageResult, RenderResult } from "./page-loader.js";
+export { decodeHtmlEntities, finalizeOutput, loadPage, looksLikeHtml, MAX_BYTES, MAX_OUTPUT_CHARS, } from "./page-loader.js";
+export { type RenderUrlOptions, renderUrl } from "./render.js";
+export { buildBinaryNotice, getExtensionHint, getFilenameExtensionHint, isConvertible, normalizeMime, normalizeUrl, sampleLooksBinary, } from "./url-target.js";

package/dist/web/fetch/negotiate.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Content negotiation and LLM endpoint discovery — ported from oh-my-pi's
+ * `packages/coding-agent/src/tools/fetch.ts` (MIT licensed).
+ */
+import { type FetchImpl } from "./page-loader.js";
+/**
+ * Build llms.txt / llms.md candidates scoped to the requested URL.
+ */
+export declare function buildLlmEndpointCandidates(url: string): string[];
+/**
+ * Try fetching URL with .md appended (llms.txt convention).
+ */
+export declare function tryMdSuffix(url: string, timeout: number, signal?: AbortSignal, fetchImpl?: FetchImpl): Promise<string | null>;
+/**
+ * Try to fetch LLM-friendly endpoints (llms.txt, llms.md).
+ */
+export declare function tryLlmEndpoints(url: string, timeout: number, signal?: AbortSignal, fetchImpl?: FetchImpl): Promise<{
+    content: string;
+    endpoint: string;
+} | null>;
+/**
+ * Try content negotiation for markdown/plain.
+ */
+export declare function tryContentNegotiation(url: string, timeout: number, signal?: AbortSignal, fetchImpl?: FetchImpl): Promise<{
+    content: string;
+    type: string;
+} | null>;

package/dist/web/fetch/page-loader.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Page loader and output finalization — ported from oh-my-pi's
+ * `packages/coding-agent/src/web/scrapers/types.ts` (MIT licensed).
+ *
+ * Provides: loadPage (UA rotation, 429 Retry-After backoff, streaming
+ * maxBytes cap, charset-aware decodeBody), finalizeOutput, looksLikeHtml,
+ * decodeHtmlEntities, MAX_BYTES, MAX_OUTPUT_CHARS, RenderResult types.
+ */
+export interface RenderResult {
+    readonly url: string;
+    readonly finalUrl: string;
+    readonly contentType: string;
+    readonly method: string;
+    readonly content: string;
+    readonly fetchedAt: string;
+    readonly truncated: boolean;
+    readonly notes: string[];
+}
+export declare const MAX_OUTPUT_CHARS = 500000;
+export declare const MAX_BYTES: number;
+/** Transport signature accepted for injection (tests, proxies). */
+export type FetchImpl = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+export interface LoadPageOptions {
+    timeout?: number;
+    /** Transport injection; defaults to global fetch. */
+    fetch?: FetchImpl;
+    headers?: Record<string, string>;
+    method?: string;
+    body?: string;
+    maxBytes?: number;
+    signal?: AbortSignal;
+    skipBodyForContentType?: (contentType: string) => boolean;
+}
+export interface LoadPageResult {
+    content: string;
+    contentType: string;
+    finalUrl: string;
+    ok: boolean;
+    status?: number;
+    truncated?: boolean;
+    error?: string;
+    bodySkipped?: boolean;
+}
+export declare function finalizeOutput(content: string): {
+    content: string;
+    truncated: boolean;
+};
+/**
+ * Fetch a page with timeout and size limit.
+ */
+export declare function loadPage(url: string, options?: LoadPageOptions): Promise<LoadPageResult>;
+/**
+ * Check if content looks like HTML by inspecting the leading tag.
+ */
+export declare function looksLikeHtml(content: string): boolean;
+/**
+ * Decode common HTML entities. Single-pass: an entity produced by decoding
+ * is never re-decoded (see `../entities.ts`).
+ */
+export { decodeHtmlEntities } from "../entities.js";

package/dist/web/fetch/render.d.ts

@@ -0,0 +1,23 @@
+/**
+ * renderUrl — the full URL-fetch/render pipeline — ported from oh-my-pi's
+ * `packages/coding-agent/src/tools/fetch.ts` (MIT licensed).
+ *
+ * Special site handlers (step 1), images, sqlite, archives, and notebooks
+ * are NOT ported. Binary payloads (zip, pdf, etc.) return a textual binary
+ * notice. The markit conversion step uses AutoRAG's parser registry for
+ * PDF/DOCX/PPTX/XLSX; EPUB and other unsupported types fall through to the
+ * binary-notice fallback.
+ */
+import { type RenderResult } from "./page-loader.js";
+export interface RenderUrlOptions {
+    timeoutSeconds?: number;
+    raw?: boolean;
+    signal?: AbortSignal;
+    fetch?: typeof fetch;
+    firecrawlApiKey?: string;
+    jinaApiKey?: string;
+}
+/**
+ * Fetch and render a URL to markdown or text.
+ */
+export declare function renderUrl(url: string, options?: RenderUrlOptions): Promise<RenderResult>;

package/dist/web/fetch/turndown/gfm.d.ts

@@ -0,0 +1,11 @@
+import type { TurndownPlugin } from "./types.js";
+/** Install GitHub fenced code blocks selected by `highlight-source-*` wrappers. */
+export declare const highlightedCodeBlock: TurndownPlugin;
+/** Install GFM strikethrough conversion. */
+export declare const strikethrough: TurndownPlugin;
+/** Install GFM task-list checkbox conversion. */
+export declare const taskListItems: TurndownPlugin;
+/** Install GFM table conversion for tables with a heading row. */
+export declare const tables: TurndownPlugin;
+/** Install every GFM rule used by the coding agent. */
+export declare const gfm: TurndownPlugin;

package/dist/web/fetch/turndown/html.d.ts

@@ -0,0 +1,5 @@
+import type { TurndownNode } from "./types.js";
+/** Parse an HTML fragment into the small DOM subset needed by Turndown. */
+export declare function parseHtmlFragment(html: string): TurndownNode;
+/** Serialize a standards-shaped or internal HTML node. */
+export declare function serializeNode(node: TurndownNode): string;

package/dist/web/fetch/turndown/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Vendored from oh-my-pi (packages/utils/src/turndown) — MIT licensed.
+ * Behavior-compatible Turndown reimplementation used as the `native`
+ * HTML-to-markdown backend by `src/web/fetch/html-renderer.ts`.
+ *
+ * Source: https://github.com/oh-my-pi/oh-my-pi
+ */
+export * from "./gfm.js";
+export { default, default as TurndownService } from "./service.js";
+export * from "./types.js";

package/dist/web/fetch/turndown/service.d.ts

@@ -0,0 +1,21 @@
+import type { ResolvedTurndownOptions, RuleFilter, TurndownNode, TurndownOptions, TurndownPlugin, TurndownRule } from "./types.js";
+/** Convert HTML fragments to Markdown with extensible Turndown-compatible rules. */
+export default class TurndownService {
+    #private;
+    readonly options: ResolvedTurndownOptions;
+    constructor(options?: TurndownOptions);
+    /** Install a highest-priority named conversion rule. */
+    addRule(key: string, rule: TurndownRule): this;
+    /** Preserve matching elements as HTML when no custom rule handles them. */
+    keep(filter: RuleFilter): this;
+    /** Drop matching elements and all of their converted content. */
+    remove(filter: RuleFilter): this;
+    /** Install one plugin or an ordered list of plugins. */
+    use(plugin: TurndownPlugin | readonly TurndownPlugin[]): this;
+    /** Escape Markdown punctuation using Turndown's public escaping rules. */
+    escape(text: string): string;
+    /** Convert an HTML string or standards-shaped DOM node to Markdown. */
+    turndown(input: string | TurndownNode): string;
+    /** Convert only a node's children within the active conversion. */
+    convertChildren(node: TurndownNode): string;
+}

package/dist/web/fetch/turndown/types.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Behavior-compatible reimplementation of turndown's used surface.
+ * Vendored from oh-my-pi (can1357/oh-my-pi, MIT) `packages/utils/src/turndown/types.ts`.
+ */
+/** A DOM-shaped node accepted by conversion rules. */
+export interface TurndownNode {
+    readonly nodeType: number;
+    readonly nodeName: string;
+    readonly parentNode: TurndownNode | null;
+    readonly childNodes: ArrayLike<TurndownNode>;
+    readonly children: ArrayLike<TurndownNode>;
+    readonly firstChild: TurndownNode | null;
+    readonly lastChild: TurndownNode | null;
+    readonly nextSibling: TurndownNode | null;
+    readonly previousSibling: TurndownNode | null;
+    readonly textContent: string | null;
+    readonly outerHTML?: string;
+    getAttribute(name: string): string | null;
+    hasAttribute(name: string): boolean;
+}
+/** Options supported by the HTML-to-Markdown converter. */
+export interface TurndownOptions {
+    headingStyle?: "setext" | "atx";
+    hr?: string;
+    bulletListMarker?: "*" | "-" | "+";
+    codeBlockStyle?: "indented" | "fenced";
+    fence?: string;
+    emDelimiter?: "_" | "*";
+    strongDelimiter?: "**" | "__";
+    linkStyle?: "inlined" | "referenced";
+    linkReferenceStyle?: "full" | "collapsed" | "shortcut";
+    preformattedCode?: boolean;
+    blankReplacement?: ReplacementFunction;
+    keepReplacement?: ReplacementFunction;
+    defaultReplacement?: ReplacementFunction;
+}
+/** Fully resolved options passed to replacement callbacks. */
+export interface ResolvedTurndownOptions {
+    headingStyle: "setext" | "atx";
+    hr: string;
+    bulletListMarker: "*" | "-" | "+";
+    codeBlockStyle: "indented" | "fenced";
+    fence: string;
+    emDelimiter: "_" | "*";
+    strongDelimiter: "**" | "__";
+    linkStyle: "inlined" | "referenced";
+    linkReferenceStyle: "full" | "collapsed" | "shortcut";
+    preformattedCode: boolean;
+    blankReplacement?: ReplacementFunction;
+    keepReplacement?: ReplacementFunction;
+    defaultReplacement?: ReplacementFunction;
+}
+/** A tag name, list of tag names, or predicate selecting nodes for a rule. */
+export type RuleFilter = string | readonly string[] | ((node: TurndownNode, options: ResolvedTurndownOptions) => boolean);
+/** Produces Markdown for a matched node and its already-converted children. */
+export type ReplacementFunction = (content: string, node: TurndownNode, options: ResolvedTurndownOptions) => string;
+/** A named conversion rule accepted by `addRule`. */
+export interface TurndownRule {
+    filter: RuleFilter;
+    replacement: ReplacementFunction;
+}
+/** A plugin that installs one or more conversion rules. */
+export type TurndownPlugin = (service: TurndownServiceLike) => void;
+/** Structural service surface available to plugins. */
+export interface TurndownServiceLike {
+    readonly options: ResolvedTurndownOptions;
+    addRule(key: string, rule: TurndownRule): this;
+    convertChildren(node: TurndownNode): string;
+    keep(filter: RuleFilter): this;
+    remove(filter: RuleFilter): this;
+    turndown(input: string | TurndownNode): string;
+    escape(text: string): string;
+}

package/dist/web/fetch/url-target.d.ts

@@ -0,0 +1,29 @@
+/**
+ * URL normalization, MIME classification, and binary detection — ported from
+ * oh-my-pi's `packages/coding-agent/src/tools/fetch.ts` (MIT licensed).
+ */
+/**
+ * Normalize a URL: repair a collapsed scheme, then add `https://` if missing.
+ */
+export declare function normalizeUrl(url: string): string;
+/**
+ * Normalize a MIME type (lowercase, strip charset/params).
+ */
+export declare function normalizeMime(contentType: string): string;
+export declare function getFilenameExtensionHint(filename: string): string;
+/**
+ * Get extension from URL or Content-Disposition.
+ */
+export declare function getExtensionHint(url: string, contentDisposition?: string): string;
+/**
+ * Check if content type is convertible via AutoRAG's parser registry.
+ */
+export declare function isConvertible(mime: string, extensionHint: string): boolean;
+/**
+ * Heuristic: does the text sample look like binary (NUL bytes or high U+FFFD density)?
+ */
+export declare function sampleLooksBinary(text: string): boolean;
+/**
+ * Build a textual binary notice for unsupported binary payloads.
+ */
+export declare function buildBinaryNotice(finalUrl: string, mime: string, byteLength?: number): string;