@de-otio/chaoskb-client 0.2.0

package/dist/crypto/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../crypto/types.ts"],"names":[],"mappings":"AAqBA,uCAAuC;AACvC,MAAM,CAAN,IAAY,YAOX;AAPD,WAAY,YAAY;IACtB,+EAA+E;IAC/E,qCAAqB,CAAA;IACrB,yFAAyF;IACzF,qCAAqB,CAAA;IACrB,kDAAkD;IAClD,mCAAmB,CAAA;AACrB,CAAC,EAPW,YAAY,KAAZ,YAAY,QAOvB"}

package/dist/pipeline/chunker.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Text chunking for the content pipeline.
+ *
+ * Splits extracted text into overlapping chunks of approximately
+ * `maxTokens` tokens, breaking on sentence boundaries where possible.
+ */
+import type { Chunk } from './types.js';
+/** Configuration for the chunker. */
+export interface ChunkConfig {
+    /** Maximum tokens per chunk (default: 500). */
+    maxTokens: number;
+    /** Token overlap between consecutive chunks (default: 50). */
+    overlapTokens: number;
+}
+/**
+ * Split text into overlapping chunks of approximately `maxTokens` tokens.
+ *
+ * Splitting is performed on sentence boundaries where possible. If a
+ * single sentence exceeds `maxTokens`, it is included as-is in its own
+ * chunk (no mid-sentence splitting).
+ *
+ * @param text - The text to split into chunks.
+ * @param config - Optional chunking configuration.
+ * @returns Array of chunks with content, index, token count, and byte offset.
+ */
+export declare function chunkText(text: string, config?: Partial<ChunkConfig>): Chunk[];
+//# sourceMappingURL=chunker.d.ts.map

package/dist/pipeline/chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../pipeline/chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC,qCAAqC;AACrC,MAAM,WAAW,WAAW;IAC1B,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,8DAA8D;IAC9D,aAAa,EAAE,MAAM,CAAC;CACvB;AAaD;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,KAAK,EAAE,CAoE9E"}

package/dist/pipeline/chunker.js

@@ -0,0 +1,96 @@
+/**
+ * Text chunking for the content pipeline.
+ *
+ * Splits extracted text into overlapping chunks of approximately
+ * `maxTokens` tokens, breaking on sentence boundaries where possible.
+ */
+import { countTokens } from './tokenizer.js';
+const DEFAULT_CONFIG = {
+    maxTokens: 500,
+    overlapTokens: 50,
+};
+/**
+ * Sentence-boundary regex. Matches `.`, `!`, or `?` followed by
+ * whitespace or end of string. Keeps the punctuation with the sentence.
+ */
+const SENTENCE_BOUNDARY_RE = /(?<=[.!?])\s+/;
+/**
+ * Split text into overlapping chunks of approximately `maxTokens` tokens.
+ *
+ * Splitting is performed on sentence boundaries where possible. If a
+ * single sentence exceeds `maxTokens`, it is included as-is in its own
+ * chunk (no mid-sentence splitting).
+ *
+ * @param text - The text to split into chunks.
+ * @param config - Optional chunking configuration.
+ * @returns Array of chunks with content, index, token count, and byte offset.
+ */
+export function chunkText(text, config) {
+    const maxTokens = config?.maxTokens ?? DEFAULT_CONFIG.maxTokens;
+    const overlapTokens = config?.overlapTokens ?? DEFAULT_CONFIG.overlapTokens;
+    if (!text || text.trim().length === 0) {
+        return [];
+    }
+    // Split into sentences
+    const sentences = text.split(SENTENCE_BOUNDARY_RE).filter((s) => s.length > 0);
+    if (sentences.length === 0) {
+        return [];
+    }
+    // Pre-compute token counts for each sentence
+    const sentenceTokens = sentences.map((s) => countTokens(s));
+    const chunks = [];
+    let sentenceIdx = 0;
+    while (sentenceIdx < sentences.length) {
+        // Build a chunk by accumulating sentences up to maxTokens
+        const chunkSentences = [];
+        let chunkTokenCount = 0;
+        const startSentenceIdx = sentenceIdx;
+        while (sentenceIdx < sentences.length) {
+            const stc = sentenceTokens[sentenceIdx];
+            // If adding this sentence would exceed max and we already have content,
+            // stop (unless the chunk is empty — always include at least one sentence).
+            if (chunkTokenCount + stc > maxTokens && chunkSentences.length > 0) {
+                break;
+            }
+            chunkSentences.push(sentences[sentenceIdx]);
+            chunkTokenCount += stc;
+            sentenceIdx++;
+        }
+        const content = chunkSentences.join(' ');
+        // Compute byte offset: sum of byte lengths of all sentences before startSentenceIdx
+        // plus the separator spaces between them
+        const byteOffset = computeByteOffset(sentences, startSentenceIdx);
+        chunks.push({
+            content,
+            index: chunks.length,
+            tokenCount: countTokens(content),
+            byteOffset,
+        });
+        // Apply overlap: back up by enough sentences to cover overlapTokens
+        if (sentenceIdx < sentences.length) {
+            let overlapCount = 0;
+            let backtrack = sentenceIdx - 1;
+            while (backtrack > startSentenceIdx && overlapCount < overlapTokens) {
+                overlapCount += sentenceTokens[backtrack];
+                backtrack--;
+            }
+            // sentenceIdx should start from backtrack + 1 (the first overlap sentence)
+            sentenceIdx = backtrack + 1;
+        }
+    }
+    return chunks;
+}
+/**
+ * Compute the byte offset of the sentence at the given index
+ * within the original text (assuming sentences are joined by single spaces).
+ */
+function computeByteOffset(sentences, targetIdx) {
+    let offset = 0;
+    for (let i = 0; i < targetIdx; i++) {
+        offset += Buffer.byteLength(sentences[i], 'utf-8');
+        // Account for the whitespace separator between sentences
+        offset += 1; // The space that was split on
+    }
+    return offset;
+}
+//# sourceMappingURL=chunker.js.map

package/dist/pipeline/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../pipeline/chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAW7C,MAAM,cAAc,GAAgB;IAClC,SAAS,EAAE,GAAG;IACd,aAAa,EAAE,EAAE;CAClB,CAAC;AAEF;;;GAGG;AACH,MAAM,oBAAoB,GAAG,eAAe,CAAC;AAE7C;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,MAA6B;IACnE,MAAM,SAAS,GAAG,MAAM,EAAE,SAAS,IAAI,cAAc,CAAC,SAAS,CAAC;IAChE,MAAM,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,cAAc,CAAC,aAAa,CAAC;IAE5E,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,uBAAuB;IACvB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,6CAA6C;IAC7C,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5D,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,OAAO,WAAW,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;QACtC,0DAA0D;QAC1D,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,MAAM,gBAAgB,GAAG,WAAW,CAAC;QAErC,OAAO,WAAW,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;YAExC,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,eAAe,GAAG,GAAG,GAAG,SAAS,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnE,MAAM;YACR,CAAC;YAED,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC;YAC5C,eAAe,IAAI,GAAG,CAAC;YACvB,WAAW,EAAE,CAAC;QAChB,CAAC;QAED,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEzC,oFAAoF;QACpF,yCAAyC;QACzC,MAAM,UAAU,GAAG,iBAAiB,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;QAElE,MAAM,CAAC,IAAI,CAAC;YACV,OAAO;YACP,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,UAAU,EAAE,WAAW,CAAC,OAAO,CAAC;YAChC,UAAU;SACX,CAAC,CAAC;QAEH,oEAAoE;QACpE,IAAI,WAAW,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;YACnC,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,IAAI,SAAS,GAAG,WAAW,GAAG,CAAC,CAAC;YAChC,OAAO,SAAS,GAAG,gBAAgB,IAAI,YAAY,GAAG,aAAa,EAAE,CAAC;gBACpE,YAAY,IAAI,cAAc,CAAC,SAAS,CAAC,CAAC;gBAC1C,SAAS,EAAE,CAAC;YACd,CAAC;YACD,2EAA2E;YAC3E,WAAW,GAAG,SAAS,GAAG,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,SAAmB,EAAE,SAAiB;IAC/D,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACnD,yDAAyD;QACzD,MAAM,IAAI,CAAC,CAAC,CAAC,8BAA8B;IAC7C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/pipeline/content-pipeline.d.ts

@@ -0,0 +1,24 @@
+import type { Embedder } from './embedder.js';
+import type { Chunk, EmbeddedChunk, EmbeddingVector, ExtractedContent, IContentPipeline, PipelineConfig } from './types.js';
+/**
+ * Concrete implementation of IContentPipeline.
+ *
+ * Orchestrates fetching, extraction, chunking, embedding, and search
+ * by delegating to the standalone pipeline functions and an injected Embedder.
+ */
+export declare class ContentPipeline implements IContentPipeline {
+    private readonly config;
+    private readonly embedder;
+    constructor(config: Partial<PipelineConfig>, embedder: Embedder);
+    /** Fetch a URL and extract its main article content. */
+    fetchAndExtract(url: string): Promise<ExtractedContent>;
+    /** Split text into overlapping chunks. */
+    chunk(text: string): Chunk[];
+    /** Embed a single text string. */
+    embed(text: string): Promise<EmbeddingVector>;
+    /** Embed multiple chunks and zip results into EmbeddedChunk[]. */
+    embedChunks(chunks: Chunk[]): Promise<EmbeddedChunk[]>;
+    /** Search embeddings for the top-K most similar, returning their indices. */
+    search(query: EmbeddingVector, embeddings: EmbeddingVector[], topK: number): number[];
+}
+//# sourceMappingURL=content-pipeline.d.ts.map

package/dist/pipeline/content-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-pipeline.d.ts","sourceRoot":"","sources":["../../pipeline/content-pipeline.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAI9C,OAAO,KAAK,EACV,KAAK,EACL,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACf,MAAM,YAAY,CAAC;AAEpB;;;;;GAKG;AACH,qBAAa,eAAgB,YAAW,gBAAgB;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;IACjD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;gBAExB,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,QAAQ;IAK/D,wDAAwD;IAClD,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAK7D,0CAA0C;IAC1C,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE;IAO5B,kCAAkC;IAC5B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAInD,kEAAkE;IAC5D,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAS5D,6EAA6E;IAC7E,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CAItF"}

package/dist/pipeline/content-pipeline.js

@@ -0,0 +1,49 @@
+import { chunkText } from './chunker.js';
+import { extractContent } from './extract.js';
+import { fetchUrl } from './fetch.js';
+import { searchEmbeddings } from './search.js';
+/**
+ * Concrete implementation of IContentPipeline.
+ *
+ * Orchestrates fetching, extraction, chunking, embedding, and search
+ * by delegating to the standalone pipeline functions and an injected Embedder.
+ */
+export class ContentPipeline {
+    config;
+    embedder;
+    constructor(config, embedder) {
+        this.config = config;
+        this.embedder = embedder;
+    }
+    /** Fetch a URL and extract its main article content. */
+    async fetchAndExtract(url) {
+        const result = await fetchUrl(url, this.config);
+        return extractContent(result.html, result.finalUrl);
+    }
+    /** Split text into overlapping chunks. */
+    chunk(text) {
+        return chunkText(text, {
+            maxTokens: this.config.maxChunkTokens ?? 500,
+            overlapTokens: this.config.overlapTokens ?? 50,
+        });
+    }
+    /** Embed a single text string. */
+    async embed(text) {
+        return this.embedder.embed(text);
+    }
+    /** Embed multiple chunks and zip results into EmbeddedChunk[]. */
+    async embedChunks(chunks) {
+        const vectors = await this.embedder.embedBatch(chunks.map((c) => c.content));
+        return chunks.map((chunk, i) => ({
+            ...chunk,
+            embedding: vectors[i],
+            model: 'snowflake-arctic-embed-s@384',
+        }));
+    }
+    /** Search embeddings for the top-K most similar, returning their indices. */
+    search(query, embeddings, topK) {
+        const results = searchEmbeddings(query, embeddings, topK);
+        return results.map((r) => r.index);
+    }
+}
+//# sourceMappingURL=content-pipeline.js.map

package/dist/pipeline/content-pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-pipeline.js","sourceRoot":"","sources":["../../pipeline/content-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAU/C;;;;;GAKG;AACH,MAAM,OAAO,eAAe;IACT,MAAM,CAA0B;IAChC,QAAQ,CAAW;IAEpC,YAAY,MAA+B,EAAE,QAAkB;QAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,eAAe,CAAC,GAAW;QAC/B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,OAAO,cAAc,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IACtD,CAAC;IAED,0CAA0C;IAC1C,KAAK,CAAC,IAAY;QAChB,OAAO,SAAS,CAAC,IAAI,EAAE;YACrB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,GAAG;YAC5C,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,EAAE;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,kCAAkC;IAClC,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,WAAW,CAAC,MAAe;QAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7E,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,GAAG,KAAK;YACR,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;YACrB,KAAK,EAAE,8BAA8B;SACtC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,6EAA6E;IAC7E,MAAM,CAAC,KAAsB,EAAE,UAA6B,EAAE,IAAY;QACxE,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QAC1D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;CACF"}

package/dist/pipeline/embedder.d.ts

@@ -0,0 +1,49 @@
+/**
+ * ONNX Runtime embedding using snowflake-arctic-embed-s.
+ *
+ * Provides a high-level Embedder class that loads an ONNX model and
+ * produces 384-dimensional embedding vectors from text input.
+ *
+ * Uses a real BERT WordPiece tokenizer with the model's vocabulary
+ * for proper subword tokenization and meaningful embeddings.
+ */
+import type { EmbeddingVector } from './types.js';
+/**
+ * Embedder wraps an ONNX inference session for producing text embeddings.
+ */
+export declare class Embedder {
+    private session;
+    private vocab;
+    readonly modelPath: string;
+    private readonly vocabPath;
+    /**
+     * @param modelPath - Absolute path to the ONNX model file.
+     * @param vocabPath - Absolute path to the vocab.txt file.
+     */
+    constructor(modelPath: string, vocabPath?: string);
+    /**
+     * Load the ONNX model into memory. Called lazily on first embed() call,
+     * but can be called explicitly to pre-warm.
+     */
+    initialize(): Promise<void>;
+    /**
+     * Embed a single text string into a 384-dimensional vector.
+     *
+     * @param text - The text to embed.
+     * @param prefix - Optional prefix (e.g., "query: " for search queries).
+     * @returns A Float32Array of 384 dimensions.
+     */
+    embed(text: string, prefix?: string): Promise<EmbeddingVector>;
+    /**
+     * Embed multiple texts in a single batch.
+     *
+     * @param texts - Array of text strings to embed.
+     * @returns Array of Float32Array embeddings, one per input text.
+     */
+    embedBatch(texts: string[]): Promise<EmbeddingVector[]>;
+    /**
+     * Release the ONNX session and free memory.
+     */
+    dispose(): void;
+}
+//# sourceMappingURL=embedder.d.ts.map

package/dist/pipeline/embedder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../pipeline/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAMlD;;GAEG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,OAAO,CAAqC;IACpD,OAAO,CAAC,KAAK,CAA2B;IACxC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC;;;OAGG;gBACS,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAMjD;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAqBjC;;;;;;OAMG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAKpE;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAoH7D;;OAEG;IACH,OAAO,IAAI,IAAI;CAQhB"}

package/dist/pipeline/embedder.js

@@ -0,0 +1,195 @@
+/**
+ * ONNX Runtime embedding using snowflake-arctic-embed-s.
+ *
+ * Provides a high-level Embedder class that loads an ONNX model and
+ * produces 384-dimensional embedding vectors from text input.
+ *
+ * Uses a real BERT WordPiece tokenizer with the model's vocabulary
+ * for proper subword tokenization and meaningful embeddings.
+ */
+import * as ort from 'onnxruntime-node';
+import { loadVocabulary, tokenize } from './wordpiece-tokenizer.js';
+/** Maximum sequence length for the model. */
+const MAX_SEQ_LENGTH = 512;
+/**
+ * Embedder wraps an ONNX inference session for producing text embeddings.
+ */
+export class Embedder {
+    session = null;
+    vocab = null;
+    modelPath;
+    vocabPath;
+    /**
+     * @param modelPath - Absolute path to the ONNX model file.
+     * @param vocabPath - Absolute path to the vocab.txt file.
+     */
+    constructor(modelPath, vocabPath) {
+        this.modelPath = modelPath;
+        // Default: vocab.txt in the same directory as the model
+        this.vocabPath = vocabPath ?? modelPath.replace(/[^/\\]+$/, 'vocab.txt');
+    }
+    /**
+     * Load the ONNX model into memory. Called lazily on first embed() call,
+     * but can be called explicitly to pre-warm.
+     */
+    async initialize() {
+        if (this.session) {
+            return;
+        }
+        try {
+            this.session = await ort.InferenceSession.create(this.modelPath, {
+                executionProviders: ['cpu'],
+            });
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            throw new Error(`Failed to load ONNX model at ${this.modelPath}: ${msg}. ` +
+                'Ensure the model file exists and is a valid ONNX model.');
+        }
+        // Load vocabulary
+        this.vocab = loadVocabulary(this.vocabPath);
+    }
+    /**
+     * Embed a single text string into a 384-dimensional vector.
+     *
+     * @param text - The text to embed.
+     * @param prefix - Optional prefix (e.g., "query: " for search queries).
+     * @returns A Float32Array of 384 dimensions.
+     */
+    async embed(text, prefix) {
+        const results = await this.embedBatch([prefix ? `${prefix}${text}` : text]);
+        return results[0];
+    }
+    /**
+     * Embed multiple texts in a single batch.
+     *
+     * @param texts - Array of text strings to embed.
+     * @returns Array of Float32Array embeddings, one per input text.
+     */
+    async embedBatch(texts) {
+        if (!this.session) {
+            await this.initialize();
+        }
+        const session = this.session;
+        const vocab = this.vocab;
+        const batchSize = texts.length;
+        // Tokenize each text into input IDs using real WordPiece tokenizer
+        const allInputIds = [];
+        const allAttentionMask = [];
+        let maxLen = 0;
+        for (const text of texts) {
+            const ids = tokenize(text, vocab, MAX_SEQ_LENGTH);
+            allInputIds.push(ids);
+            allAttentionMask.push(ids.map(() => 1n));
+            maxLen = Math.max(maxLen, ids.length);
+        }
+        // Pad to uniform length
+        for (let i = 0; i < batchSize; i++) {
+            while (allInputIds[i].length < maxLen) {
+                allInputIds[i].push(0n); // PAD token
+                allAttentionMask[i].push(0n);
+            }
+        }
+        // Flatten into typed arrays
+        const inputIdsFlat = new BigInt64Array(batchSize * maxLen);
+        const attentionMaskFlat = new BigInt64Array(batchSize * maxLen);
+        for (let i = 0; i < batchSize; i++) {
+            for (let j = 0; j < maxLen; j++) {
+                inputIdsFlat[i * maxLen + j] = allInputIds[i][j];
+                attentionMaskFlat[i * maxLen + j] = allAttentionMask[i][j];
+            }
+        }
+        // Create ONNX tensors
+        const feeds = {
+            input_ids: new ort.Tensor('int64', inputIdsFlat, [batchSize, maxLen]),
+            attention_mask: new ort.Tensor('int64', attentionMaskFlat, [batchSize, maxLen]),
+        };
+        // Some models expect token_type_ids
+        const inputNames = session.inputNames;
+        if (inputNames.includes('token_type_ids')) {
+            const tokenTypeIds = new BigInt64Array(batchSize * maxLen); // all zeros
+            feeds['token_type_ids'] = new ort.Tensor('int64', tokenTypeIds, [batchSize, maxLen]);
+        }
+        // Run inference
+        const results = await session.run(feeds);
+        // Extract embeddings from output
+        // The model may output under various names; try common ones
+        const outputKey = results['sentence_embedding'] ? 'sentence_embedding' :
+            results['last_hidden_state'] ? 'last_hidden_state' :
+                Object.keys(results)[0];
+        const outputTensor = results[outputKey];
+        const outputData = outputTensor.data;
+        const outputDims = outputTensor.dims;
+        // If output is [batch, seq_len, dim], mean-pool over seq_len
+        // If output is [batch, dim], use directly
+        const embeddings = [];
+        if (outputDims.length === 3) {
+            // [batch, seq_len, dim] — mean pooling with attention mask
+            const seqLen = outputDims[1];
+            const dim = outputDims[2];
+            for (let b = 0; b < batchSize; b++) {
+                const embedding = new Float32Array(dim);
+                let tokenCount = 0;
+                for (let s = 0; s < seqLen; s++) {
+                    if (allAttentionMask[b][s] === 1n) {
+                        tokenCount++;
+                        for (let d = 0; d < dim; d++) {
+                            embedding[d] += outputData[b * seqLen * dim + s * dim + d];
+                        }
+                    }
+                }
+                // Average
+                if (tokenCount > 0) {
+                    for (let d = 0; d < dim; d++) {
+                        embedding[d] /= tokenCount;
+                    }
+                }
+                // L2 normalize
+                embeddings.push(l2Normalize(embedding));
+            }
+        }
+        else if (outputDims.length === 2) {
+            // [batch, dim] — already pooled
+            const dim = outputDims[1];
+            for (let b = 0; b < batchSize; b++) {
+                const embedding = new Float32Array(dim);
+                for (let d = 0; d < dim; d++) {
+                    embedding[d] = outputData[b * dim + d];
+                }
+                embeddings.push(l2Normalize(embedding));
+            }
+        }
+        else {
+            throw new Error(`Unexpected output tensor shape: [${outputDims.join(', ')}]`);
+        }
+        return embeddings;
+    }
+    /**
+     * Release the ONNX session and free memory.
+     */
+    dispose() {
+        if (this.session) {
+            // InferenceSession doesn't have a sync dispose in all versions;
+            // release is best-effort.
+            this.session.release?.();
+            this.session = null;
+        }
+    }
+}
+/**
+ * L2-normalize a vector in place and return it.
+ */
+function l2Normalize(vec) {
+    let sumSq = 0;
+    for (let i = 0; i < vec.length; i++) {
+        sumSq += vec[i] * vec[i];
+    }
+    const norm = Math.sqrt(sumSq);
+    if (norm > 0) {
+        for (let i = 0; i < vec.length; i++) {
+            vec[i] /= norm;
+        }
+    }
+    return vec;
+}
+//# sourceMappingURL=embedder.js.map

package/dist/pipeline/embedder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../pipeline/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,GAAG,MAAM,kBAAkB,CAAC;AAExC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAmB,MAAM,0BAA0B,CAAC;AAErF,6CAA6C;AAC7C,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B;;GAEG;AACH,MAAM,OAAO,QAAQ;IACX,OAAO,GAAgC,IAAI,CAAC;IAC5C,KAAK,GAAsB,IAAI,CAAC;IAC/B,SAAS,CAAS;IACV,SAAS,CAAS;IAEnC;;;OAGG;IACH,YAAY,SAAiB,EAAE,SAAkB;QAC/C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,wDAAwD;QACxD,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,MAAM,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE;gBAC/D,kBAAkB,EAAE,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnE,MAAM,IAAI,KAAK,CACb,gCAAgC,IAAI,CAAC,SAAS,KAAK,GAAG,IAAI;gBACxD,yDAAyD,CAC5D,CAAC;QACJ,CAAC;QAED,kBAAkB;QAClB,IAAI,CAAC,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,MAAe;QACvC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5E,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAQ,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAM,CAAC;QAC1B,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC;QAE/B,mEAAmE;QACnE,MAAM,WAAW,GAAe,EAAE,CAAC;QACnC,MAAM,gBAAgB,GAAe,EAAE,CAAC;QACxC,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC;YAClD,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtB,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACzC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;QAED,wBAAwB;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBACtC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY;gBACrC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,MAAM,YAAY,GAAG,IAAI,aAAa,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;QAC3D,MAAM,iBAAiB,GAAG,IAAI,aAAa,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;QAEhE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChC,YAAY,CAAC,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjD,iBAAiB,CAAC,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,sBAAsB;QACtB,MAAM,KAAK,GAA+B;YACxC,SAAS,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACrE,cAAc,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,iBAAiB,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;SAChF,CAAC;QAEF,oCAAoC;QACpC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACtC,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,IAAI,aAAa,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY;YACxE,KAAK,CAAC,gBAAgB,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;QACvF,CAAC;QAED,gBAAgB;QAChB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEzC,iCAAiC;QACjC,4DAA4D;QAC5D,MAAM,SAAS,GACb,OAAO,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC;YACtD,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;gBACpD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAE1B,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,YAAY,CAAC,IAAoB,CAAC;QACrD,MAAM,UAAU,GAAG,YAAY,CAAC,IAAyB,CAAC;QAE1D,6DAA6D;QAC7D,0CAA0C;QAC1C,MAAM,UAAU,GAAsB,EAAE,CAAC;QAEzC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,2DAA2D;YAC3D,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAW,CAAC;YACvC,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAW,CAAC;YAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;gBACxC,IAAI,UAAU,GAAG,CAAC,CAAC;gBAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAChC,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;wBAClC,UAAU,EAAE,CAAC;wBACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;4BAC7B,SAAS,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,GAAG,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC;wBAC7D,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,UAAU;gBACV,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;oBACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC7B,SAAS,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC;oBAC7B,CAAC;gBACH,CAAC;gBAED,eAAe;gBACf,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;aAAM,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,gCAAgC;YAChC,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAW,CAAC;YACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;gBACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC7B,SAAS,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC;gBACzC,CAAC;gBACD,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,oCAAoC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChF,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,gEAAgE;YAChE,0BAA0B;YAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAiB;IACpC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/pipeline/extract.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Content extraction from HTML using Mozilla Readability.
+ *
+ * Parses HTML with `linkedom` and runs it through Readability to pull out
+ * the main article content, stripped of navigation, ads, and boilerplate.
+ */
+import type { ExtractedContent } from './types.js';
+/**
+ * Extract the main article content from an HTML string.
+ *
+ * @param html - The raw HTML string to extract content from.
+ * @param url - The source URL (used for resolving relative links and metadata).
+ * @returns Extracted content with title, plain text, URL, and byte length.
+ * @throws If no article content can be extracted.
+ */
+export declare function extractContent(html: string, url: string): ExtractedContent;
+//# sourceMappingURL=extract.d.ts.map

package/dist/pipeline/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../pipeline/extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,gBAAgB,CAoD1E"}

package/dist/pipeline/extract.js

@@ -0,0 +1,70 @@
+/**
+ * Content extraction from HTML using Mozilla Readability.
+ *
+ * Parses HTML with `linkedom` and runs it through Readability to pull out
+ * the main article content, stripped of navigation, ads, and boilerplate.
+ */
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+/**
+ * Extract the main article content from an HTML string.
+ *
+ * @param html - The raw HTML string to extract content from.
+ * @param url - The source URL (used for resolving relative links and metadata).
+ * @returns Extracted content with title, plain text, URL, and byte length.
+ * @throws If no article content can be extracted.
+ */
+export function extractContent(html, url) {
+    if (!html || html.trim().length === 0) {
+        throw new Error(`Empty HTML content from ${url}`);
+    }
+    const { document } = parseHTML(html);
+    // Attempt Readability extraction
+    const reader = new Readability(document);
+    const article = reader.parse();
+    let title;
+    let rawContent;
+    if (article && article.textContent && article.textContent.trim().length > 0) {
+        title = article.title || '';
+        rawContent = article.textContent;
+    }
+    else {
+        // Fallback: extract text from body (strip script/style first)
+        // Wrap in a full HTML document to ensure linkedom creates a body element
+        const wrappedHtml = html.includes('<body') ? html : `<html><body>${html}</body></html>`;
+        const { document: fallbackDoc } = parseHTML(wrappedHtml);
+        for (const el of fallbackDoc.querySelectorAll('script, style')) {
+            el.remove();
+        }
+        const body = fallbackDoc.querySelector('body');
+        rawContent = body ? body.textContent ?? '' : '';
+        if (rawContent.trim().length === 0) {
+            throw new Error(`No extractable content from ${url}`);
+        }
+        title = '';
+    }
+    // Fallback title: try <title> tag from a fresh parse
+    if (!title) {
+        const { document: titleDoc } = parseHTML(html);
+        const titleEl = titleDoc.querySelector('title');
+        title = titleEl?.textContent?.trim() ?? '';
+    }
+    // Clean up the text: collapse whitespace runs, trim lines
+    const content = cleanText(rawContent);
+    if (content.length === 0) {
+        throw new Error(`No extractable content from ${url}`);
+    }
+    const byteLength = Buffer.byteLength(content, 'utf-8');
+    return { title, content, url, byteLength };
+}
+/**
+ * Clean extracted text by collapsing whitespace and trimming.
+ */
+function cleanText(text) {
+    return text
+        .replace(/[\t ]+/g, ' ') // collapse horizontal whitespace
+        .replace(/\n{3,}/g, '\n\n') // collapse excessive newlines
+        .replace(/^ +| +$/gm, '') // trim each line
+        .trim();
+}
+//# sourceMappingURL=extract.js.map

package/dist/pipeline/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../pipeline/extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAGrC;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,GAAW;IACtD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAErC,iCAAiC;IACjC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAe,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;IAE/B,IAAI,KAAa,CAAC;IAClB,IAAI,UAAkB,CAAC;IAEvB,IAAI,OAAO,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5E,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAC5B,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,8DAA8D;QAC9D,yEAAyE;QACzE,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,IAAI,gBAAgB,CAAC;QACxF,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;QACzD,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;YAC/D,EAAE,CAAC,MAAM,EAAE,CAAC;QACd,CAAC;QACD,MAAM,IAAI,GAAG,WAAW,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAC/C,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAEhD,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,KAAK,GAAG,EAAE,CAAC;IACb,CAAC;IAED,qDAAqD;IACrD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAChD,KAAK,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,0DAA0D;IAC1D,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IAEtC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAEvD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;AAC7C,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI;SACR,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAO,iCAAiC;SAC/D,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAI,8BAA8B;SAC5D,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAM,iBAAiB;SAC/C,IAAI,EAAE,CAAC;AACZ,CAAC"}

package/dist/pipeline/fetch.d.ts

@@ -0,0 +1,26 @@
+/**
+ * URL fetching for the content pipeline.
+ *
+ * Uses Node.js built-in `fetch` (available since Node 18) with
+ * configurable timeout, redirect limits, and user-agent.
+ */
+import type { PipelineConfig } from './types.js';
+/** Result of a successful URL fetch. */
+export interface FetchResult {
+    /** Raw HTML body. */
+    html: string;
+    /** Final URL after any redirects. */
+    finalUrl: string;
+    /** Content-Type header value. */
+    contentType: string;
+}
+/**
+ * Fetch the HTML content of a URL.
+ *
+ * @param url - The URL to fetch.
+ * @param config - Optional partial pipeline config overrides.
+ * @returns The HTML content, final URL, and content type.
+ * @throws On network errors, non-2xx status codes, or non-HTML content.
+ */
+export declare function fetchUrl(url: string, config?: Partial<PipelineConfig>): Promise<FetchResult>;
+//# sourceMappingURL=fetch.d.ts.map

package/dist/pipeline/fetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../pipeline/fetch.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AASjD,wCAAwC;AACxC,MAAM,WAAW,WAAW;IAC1B,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;GAOG;AACH,wBAAsB,QAAQ,CAC5B,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAC/B,OAAO,CAAC,WAAW,CAAC,CA6EtB"}