@de-otio/chaoskb-client 0.2.0

package/dist/pipeline/fetch.js

@@ -0,0 +1,91 @@
+/**
+ * URL fetching for the content pipeline.
+ *
+ * Uses Node.js built-in `fetch` (available since Node 18) with
+ * configurable timeout, redirect limits, and user-agent.
+ */
+/** Default pipeline configuration values relevant to fetching. */
+const DEFAULTS = {
+    fetchTimeoutMs: 30_000,
+    maxRedirects: 5,
+    userAgent: 'ChaosKB/0.1',
+};
+/**
+ * Fetch the HTML content of a URL.
+ *
+ * @param url - The URL to fetch.
+ * @param config - Optional partial pipeline config overrides.
+ * @returns The HTML content, final URL, and content type.
+ * @throws On network errors, non-2xx status codes, or non-HTML content.
+ */
+export async function fetchUrl(url, config) {
+    const timeoutMs = config?.fetchTimeoutMs ?? DEFAULTS.fetchTimeoutMs;
+    const userAgent = config?.userAgent ?? DEFAULTS.userAgent;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    let response;
+    try {
+        response = await fetch(url, {
+            method: 'GET',
+            headers: {
+                'User-Agent': userAgent,
+                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            },
+            signal: controller.signal,
+            redirect: 'follow',
+        });
+    }
+    catch (error) {
+        clearTimeout(timer);
+        if (error instanceof Error) {
+            if (error.name === 'AbortError') {
+                throw new Error(`Fetch timed out after ${timeoutMs}ms: ${url}`);
+            }
+            // DNS resolution failures
+            if (error.cause && typeof error.cause === 'object' && 'code' in error.cause) {
+                const code = error.cause.code;
+                if (code === 'ENOTFOUND') {
+                    throw new Error(`DNS resolution failed for ${url}: host not found`);
+                }
+                if (code === 'ECONNREFUSED') {
+                    throw new Error(`Connection refused for ${url}`);
+                }
+                if (code === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE' || code === 'ERR_TLS_CERT_ALTNAME_INVALID') {
+                    throw new Error(`TLS certificate error for ${url}: ${code}`);
+                }
+            }
+            // TLS errors often show up in the message
+            if (error.message.includes('SSL') || error.message.includes('TLS') || error.message.includes('certificate')) {
+                throw new Error(`TLS error fetching ${url}: ${error.message}`);
+            }
+            throw new Error(`Failed to fetch ${url}: ${error.message}`);
+        }
+        throw new Error(`Failed to fetch ${url}: unknown error`);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+    // Check HTTP status
+    if (!response.ok) {
+        const status = response.status;
+        if (status >= 400 && status < 500) {
+            throw new Error(`HTTP ${status} Client Error for ${url}: ${response.statusText}`);
+        }
+        if (status >= 500) {
+            throw new Error(`HTTP ${status} Server Error for ${url}: ${response.statusText}`);
+        }
+        throw new Error(`HTTP ${status} for ${url}: ${response.statusText}`);
+    }
+    // Verify content type is HTML-like
+    const contentType = response.headers.get('content-type') ?? '';
+    const isHtml = contentType.includes('text/html') ||
+        contentType.includes('application/xhtml+xml') ||
+        contentType.includes('application/xml');
+    if (!isHtml) {
+        throw new Error(`Non-HTML content type "${contentType}" for ${url}. Only text/html is supported.`);
+    }
+    const html = await response.text();
+    const finalUrl = response.url || url;
+    return { html, finalUrl, contentType };
+}
+//# sourceMappingURL=fetch.js.map

package/dist/pipeline/fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../pipeline/fetch.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,kEAAkE;AAClE,MAAM,QAAQ,GAA0E;IACtF,cAAc,EAAE,MAAM;IACtB,YAAY,EAAE,CAAC;IACf,SAAS,EAAE,aAAa;CACzB,CAAC;AAYF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,MAAgC;IAEhC,MAAM,SAAS,GAAG,MAAM,EAAE,cAAc,IAAI,QAAQ,CAAC,cAAc,CAAC;IACpE,MAAM,SAAS,GAAG,MAAM,EAAE,SAAS,IAAI,QAAQ,CAAC,SAAS,CAAC;IAE1D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC1B,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,YAAY,EAAE,SAAS;gBACvB,MAAM,EAAE,iEAAiE;aAC1E;YACD,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,yBAAyB,SAAS,OAAO,GAAG,EAAE,CAAC,CAAC;YAClE,CAAC;YACD,0BAA0B;YAC1B,IAAI,KAAK,CAAC,KAAK,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC5E,MAAM,IAAI,GAAI,KAAK,CAAC,KAA2B,CAAC,IAAI,CAAC;gBACrD,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;oBACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,kBAAkB,CAAC,CAAC;gBACtE,CAAC;gBACD,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;oBAC5B,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;gBACnD,CAAC;gBACD,IAAI,IAAI,KAAK,iCAAiC,IAAI,IAAI,KAAK,8BAA8B,EAAE,CAAC;oBAC1F,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,IAAI,EAAE,CAAC,CAAC;gBAC/D,CAAC;YACH,CAAC;YACD,0CAA0C;YAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC5G,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,iBAAiB,CAAC,CAAC;IAC3D,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAED,oBAAoB;IACpB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC/B,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,qBAAqB,GAAG,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACpF,CAAC;QACD,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,qBAAqB,GAAG,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACpF,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,QAAQ,GAAG,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,mCAAmC;IACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,MAAM,GACV,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;QACjC,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC;QAC7C,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE1C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,0BAA0B,WAAW,SAAS,GAAG,gCAAgC,CAClF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC;IAErC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;AACzC,CAAC"}

package/dist/pipeline/index.d.ts

@@ -0,0 +1,10 @@
+export * from './types.js';
+export { fetchUrl } from './fetch.js';
+export { extractContent } from './extract.js';
+export { chunkText } from './chunker.js';
+export { countTokens } from './tokenizer.js';
+export { Embedder } from './embedder.js';
+export { ModelManager } from './model-manager.js';
+export { ContentPipeline } from './content-pipeline.js';
+export { cosineSimilarity, searchEmbeddings } from './search.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/pipeline/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../pipeline/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC"}

package/dist/pipeline/index.js

@@ -0,0 +1,10 @@
+export * from './types.js';
+export { fetchUrl } from './fetch.js';
+export { extractContent } from './extract.js';
+export { chunkText } from './chunker.js';
+export { countTokens } from './tokenizer.js';
+export { Embedder } from './embedder.js';
+export { ModelManager } from './model-manager.js';
+export { ContentPipeline } from './content-pipeline.js';
+export { cosineSimilarity, searchEmbeddings } from './search.js';
+//# sourceMappingURL=index.js.map

package/dist/pipeline/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../pipeline/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC"}

package/dist/pipeline/model-manager.d.ts

@@ -0,0 +1,57 @@
+/**
+ * ONNX model download and verification manager.
+ *
+ * Handles downloading the snowflake-arctic-embed-s ONNX model from
+ * HuggingFace, verifying its SHA-256 hash, and managing the local
+ * model cache directory.
+ */
+import type { DownloadProgressCallback, IModelManager } from './types.js';
+/**
+ * Manages downloading and verifying the ONNX embedding model.
+ */
+export declare class ModelManager implements IModelManager {
+    private readonly modelsDir;
+    /**
+     * @param modelsDir - Directory to store model files.
+     *                    Defaults to `~/.chaoskb/models/`.
+     */
+    constructor(modelsDir?: string);
+    /**
+     * Get the expected path to the model file.
+     */
+    getModelPath(): string;
+    /**
+     * Get the expected path to the vocabulary file.
+     */
+    getVocabPath(): string;
+    /**
+     * Check if the model file exists and its SHA-256 matches the stored hash.
+     */
+    isModelReady(): Promise<boolean>;
+    /**
+     * Ensure the model is downloaded and verified. Downloads if missing
+     * or hash mismatch. Also downloads the vocabulary file.
+     * Returns the path to the model file.
+     *
+     * @param onProgress - Optional callback for download progress.
+     * @returns Absolute path to the verified model file.
+     */
+    ensureModel(onProgress?: DownloadProgressCallback): Promise<string>;
+    /**
+     * Check if the vocabulary file exists and its hash matches.
+     */
+    isVocabReady(): Promise<boolean>;
+    /**
+     * Ensure the vocabulary file is downloaded and verified.
+     */
+    ensureVocab(): Promise<string>;
+    /**
+     * Download the model file with progress reporting.
+     */
+    private downloadModel;
+    /**
+     * Download a file from a URL with optional progress reporting and resume support.
+     */
+    private downloadFile;
+}
+//# sourceMappingURL=model-manager.d.ts.map

package/dist/pipeline/model-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-manager.d.ts","sourceRoot":"","sources":["../../pipeline/model-manager.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AASH,OAAO,KAAK,EAAE,wBAAwB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAyB1E;;GAEG;AACH,qBAAa,YAAa,YAAW,aAAa;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC;;;OAGG;gBACS,SAAS,CAAC,EAAE,MAAM;IAI9B;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;IAoBtC;;;;;;;OAOG;IACG,WAAW,CAAC,UAAU,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,MAAM,CAAC;IA6BzE;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;IAoBtC;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAmBpC;;OAEG;YACW,aAAa;IAO3B;;OAEG;YACW,YAAY;CA6E3B"}

package/dist/pipeline/model-manager.js

@@ -0,0 +1,234 @@
+/**
+ * ONNX model download and verification manager.
+ *
+ * Handles downloading the snowflake-arctic-embed-s ONNX model from
+ * HuggingFace, verifying its SHA-256 hash, and managing the local
+ * model cache directory.
+ */
+import { createHash } from 'node:crypto';
+import { createReadStream, createWriteStream } from 'node:fs';
+import { access, mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import { Readable } from 'node:stream';
+/** Default model download URL. */
+const MODEL_URL = 'https://huggingface.co/Snowflake/snowflake-arctic-embed-s/resolve/main/onnx/model.onnx';
+/** Vocabulary file download URL. */
+const VOCAB_URL = 'https://huggingface.co/Snowflake/snowflake-arctic-embed-s/resolve/main/vocab.txt';
+/** Model filename. */
+const MODEL_FILENAME = 'model.onnx';
+/** Vocabulary filename. */
+const VOCAB_FILENAME = 'vocab.txt';
+/** SHA-256 sidecar filename. */
+const HASH_FILENAME = 'model.onnx.sha256';
+/** Vocabulary SHA-256 sidecar filename. */
+const VOCAB_HASH_FILENAME = 'vocab.txt.sha256';
+/** Temporary download suffix. */
+const TEMP_SUFFIX = '.download';
+/**
+ * Manages downloading and verifying the ONNX embedding model.
+ */
+export class ModelManager {
+    modelsDir;
+    /**
+     * @param modelsDir - Directory to store model files.
+     *                    Defaults to `~/.chaoskb/models/`.
+     */
+    constructor(modelsDir) {
+        this.modelsDir = modelsDir ?? join(homedir(), '.chaoskb', 'models');
+    }
+    /**
+     * Get the expected path to the model file.
+     */
+    getModelPath() {
+        return join(this.modelsDir, MODEL_FILENAME);
+    }
+    /**
+     * Get the expected path to the vocabulary file.
+     */
+    getVocabPath() {
+        return join(this.modelsDir, VOCAB_FILENAME);
+    }
+    /**
+     * Check if the model file exists and its SHA-256 matches the stored hash.
+     */
+    async isModelReady() {
+        const modelPath = this.getModelPath();
+        const hashPath = join(this.modelsDir, HASH_FILENAME);
+        try {
+            await access(modelPath);
+            await access(hashPath);
+        }
+        catch {
+            return false;
+        }
+        try {
+            const storedHash = (await readFile(hashPath, 'utf-8')).trim();
+            const actualHash = await computeFileHash(modelPath);
+            return storedHash === actualHash;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Ensure the model is downloaded and verified. Downloads if missing
+     * or hash mismatch. Also downloads the vocabulary file.
+     * Returns the path to the model file.
+     *
+     * @param onProgress - Optional callback for download progress.
+     * @returns Absolute path to the verified model file.
+     */
+    async ensureModel(onProgress) {
+        const modelPath = this.getModelPath();
+        // Create directory
+        await mkdir(this.modelsDir, { recursive: true });
+        // Ensure vocab is downloaded (small file, do first)
+        await this.ensureVocab();
+        if (await this.isModelReady()) {
+            return modelPath;
+        }
+        // Download to temporary file
+        const tempPath = modelPath + TEMP_SUFFIX;
+        await this.downloadModel(tempPath, onProgress);
+        // Compute hash
+        const hash = await computeFileHash(tempPath);
+        // Move to final location
+        await rename(tempPath, modelPath);
+        // Write hash sidecar
+        await writeFile(join(this.modelsDir, HASH_FILENAME), hash + '\n', 'utf-8');
+        return modelPath;
+    }
+    /**
+     * Check if the vocabulary file exists and its hash matches.
+     */
+    async isVocabReady() {
+        const vocabPath = this.getVocabPath();
+        const hashPath = join(this.modelsDir, VOCAB_HASH_FILENAME);
+        try {
+            await access(vocabPath);
+            await access(hashPath);
+        }
+        catch {
+            return false;
+        }
+        try {
+            const storedHash = (await readFile(hashPath, 'utf-8')).trim();
+            const actualHash = await computeFileHash(vocabPath);
+            return storedHash === actualHash;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Ensure the vocabulary file is downloaded and verified.
+     */
+    async ensureVocab() {
+        const vocabPath = this.getVocabPath();
+        if (await this.isVocabReady()) {
+            return vocabPath;
+        }
+        await mkdir(this.modelsDir, { recursive: true });
+        const tempPath = vocabPath + TEMP_SUFFIX;
+        await this.downloadFile(VOCAB_URL, tempPath);
+        const hash = await computeFileHash(tempPath);
+        await rename(tempPath, vocabPath);
+        await writeFile(join(this.modelsDir, VOCAB_HASH_FILENAME), hash + '\n', 'utf-8');
+        return vocabPath;
+    }
+    /**
+     * Download the model file with progress reporting.
+     */
+    async downloadModel(destPath, onProgress) {
+        return this.downloadFile(MODEL_URL, destPath, onProgress);
+    }
+    /**
+     * Download a file from a URL with optional progress reporting and resume support.
+     */
+    async downloadFile(url, destPath, onProgress) {
+        // Check for existing partial download for potential resume
+        let existingSize = 0;
+        try {
+            const { stat } = await import('node:fs/promises');
+            const stats = await stat(destPath);
+            existingSize = stats.size;
+        }
+        catch {
+            // No partial download exists
+        }
+        const headers = {
+            'User-Agent': 'ChaosKB/0.1',
+        };
+        // Attempt resume if partial file exists
+        if (existingSize > 0) {
+            headers['Range'] = `bytes=${existingSize}-`;
+        }
+        let response;
+        try {
+            response = await fetch(url, { headers, redirect: 'follow' });
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            throw new Error(`Failed to download ${url}: ${msg}`);
+        }
+        // If server doesn't support range or returned full content, start fresh
+        if (response.status === 200) {
+            existingSize = 0;
+        }
+        else if (response.status === 206) {
+            // Partial content — resume supported
+        }
+        else if (!response.ok) {
+            throw new Error(`Failed to download ${url}: HTTP ${response.status} ${response.statusText}`);
+        }
+        const contentLength = response.headers.get('content-length');
+        const totalSize = contentLength
+            ? existingSize + parseInt(contentLength, 10)
+            : 0;
+        if (!response.body) {
+            throw new Error('Response body is null');
+        }
+        // Convert web ReadableStream to Node.js Readable
+        const nodeStream = Readable.fromWeb(response.body);
+        const writeStream = createWriteStream(destPath, {
+            flags: existingSize > 0 && response.status === 206 ? 'a' : 'w',
+        });
+        let downloaded = existingSize;
+        nodeStream.on('data', (chunk) => {
+            downloaded += chunk.length;
+            onProgress?.(downloaded, totalSize);
+        });
+        try {
+            await pipeline(nodeStream, writeStream);
+        }
+        catch (error) {
+            // Clean up partial download on error
+            try {
+                await unlink(destPath);
+            }
+            catch {
+                // Ignore cleanup errors
+            }
+            const msg = error instanceof Error ? error.message : String(error);
+            throw new Error(`Download interrupted: ${msg}`);
+        }
+    }
+}
+/**
+ * Compute the SHA-256 hash of a file.
+ *
+ * @param filePath - Path to the file.
+ * @returns Hex-encoded SHA-256 hash string.
+ */
+async function computeFileHash(filePath) {
+    return new Promise((resolve, reject) => {
+        const hash = createHash('sha256');
+        const stream = createReadStream(filePath);
+        stream.on('data', (chunk) => hash.update(chunk));
+        stream.on('end', () => resolve(hash.digest('hex')));
+        stream.on('error', reject);
+    });
+}
+//# sourceMappingURL=model-manager.js.map

package/dist/pipeline/model-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-manager.js","sourceRoot":"","sources":["../../pipeline/model-manager.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACtF,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,kCAAkC;AAClC,MAAM,SAAS,GACb,wFAAwF,CAAC;AAE3F,oCAAoC;AACpC,MAAM,SAAS,GACb,kFAAkF,CAAC;AAErF,sBAAsB;AACtB,MAAM,cAAc,GAAG,YAAY,CAAC;AAEpC,2BAA2B;AAC3B,MAAM,cAAc,GAAG,WAAW,CAAC;AAEnC,gCAAgC;AAChC,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAE1C,2CAA2C;AAC3C,MAAM,mBAAmB,GAAG,kBAAkB,CAAC;AAE/C,iCAAiC;AACjC,MAAM,WAAW,GAAG,WAAW,CAAC;AAEhC;;GAEG;AACH,MAAM,OAAO,YAAY;IACN,SAAS,CAAS;IAEnC;;;OAGG;IACH,YAAY,SAAkB;QAC5B,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QAErD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;YACpD,OAAO,UAAU,KAAK,UAAU,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,WAAW,CAAC,UAAqC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAEtC,mBAAmB;QACnB,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEjD,oDAAoD;QACpD,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAEzB,IAAI,MAAM,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YAC9B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,6BAA6B;QAC7B,MAAM,QAAQ,GAAG,SAAS,GAAG,WAAW,CAAC;QACzC,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE/C,eAAe;QACf,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC;QAE7C,yBAAyB;QACzB,MAAM,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAElC,qBAAqB;QACrB,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;QAE3E,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;QAE3D,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;YACpD,OAAO,UAAU,KAAK,UAAU,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAEtC,IAAI,MAAM,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YAC9B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEjD,MAAM,QAAQ,GAAG,SAAS,GAAG,WAAW,CAAC;QACzC,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAE7C,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAClC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;QAEjF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,QAAgB,EAChB,UAAqC;QAErC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CACxB,GAAW,EACX,QAAgB,EAChB,UAAqC;QAErC,2DAA2D;QAC3D,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YACnC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,6BAA6B;QAC/B,CAAC;QAED,MAAM,OAAO,GAA2B;YACtC,YAAY,EAAE,aAAa;SAC5B,CAAC;QAEF,wCAAwC;QACxC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,YAAY,GAAG,CAAC;QAC9C,CAAC;QAED,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnE,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;QAED,wEAAwE;QACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,YAAY,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACnC,qCAAqC;QACvC,CAAC;aAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,UAAU,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,aAAa;YAC7B,CAAC,CAAC,YAAY,GAAG,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC;YAC5C,CAAC,CAAC,CAAC,CAAC;QAEN,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QAED,iDAAiD;QACjD,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAgD,CAAC,CAAC;QAE/F,MAAM,WAAW,GAAG,iBAAiB,CAAC,QAAQ,EAAE;YAC9C,KAAK,EAAE,YAAY,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;SAC/D,CAAC,CAAC;QAEH,IAAI,UAAU,GAAG,YAAY,CAAC;QAE9B,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACtC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;YAC3B,UAAU,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,qCAAqC;YACrC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;YACD,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnE,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;CACF;AAED;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAAC,QAAgB;IAC7C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/pipeline/search.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Brute-force cosine similarity search over embedding vectors.
+ *
+ * Designed for in-memory search of up to ~50k 384-dimensional embeddings.
+ * At that scale, brute-force cosine similarity is fast enough (<50ms)
+ * and avoids the complexity of approximate nearest-neighbor indices.
+ */
+/**
+ * Compute the cosine similarity between two vectors.
+ *
+ * cosine_similarity = (a . b) / (|a| * |b|)
+ *
+ * Optimized for Float32Array — no intermediate allocations.
+ *
+ * @param a - First vector.
+ * @param b - Second vector (must be same length as `a`).
+ * @returns Cosine similarity in the range [-1, 1]. Returns 0 if either
+ *          vector has zero magnitude.
+ */
+export declare function cosineSimilarity(a: Float32Array, b: Float32Array): number;
+/** A search result with index and similarity score. */
+export interface ScoredResult {
+    /** Index of the embedding in the input array. */
+    index: number;
+    /** Cosine similarity score. */
+    score: number;
+}
+/**
+ * Search a collection of embeddings for the top-K most similar to a query.
+ *
+ * @param query - The query embedding vector.
+ * @param embeddings - Array of embedding vectors to search.
+ * @param topK - Number of top results to return.
+ * @returns Array of `{ index, score }` sorted by score descending, length <= topK.
+ */
+export declare function searchEmbeddings(query: Float32Array, embeddings: Float32Array[], topK: number): ScoredResult[];
+//# sourceMappingURL=search.d.ts.map

package/dist/pipeline/search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../pipeline/search.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;;;;;;;;;;GAWG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAwBzE;AAED,uDAAuD;AACvD,MAAM,WAAW,YAAY;IAC3B,iDAAiD;IACjD,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,YAAY,EACnB,UAAU,EAAE,YAAY,EAAE,EAC1B,IAAI,EAAE,MAAM,GACX,YAAY,EAAE,CAmBhB"}

package/dist/pipeline/search.js

@@ -0,0 +1,65 @@
+/**
+ * Brute-force cosine similarity search over embedding vectors.
+ *
+ * Designed for in-memory search of up to ~50k 384-dimensional embeddings.
+ * At that scale, brute-force cosine similarity is fast enough (<50ms)
+ * and avoids the complexity of approximate nearest-neighbor indices.
+ */
+/**
+ * Compute the cosine similarity between two vectors.
+ *
+ * cosine_similarity = (a . b) / (|a| * |b|)
+ *
+ * Optimized for Float32Array — no intermediate allocations.
+ *
+ * @param a - First vector.
+ * @param b - Second vector (must be same length as `a`).
+ * @returns Cosine similarity in the range [-1, 1]. Returns 0 if either
+ *          vector has zero magnitude.
+ */
+export function cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+        throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+    }
+    const len = a.length;
+    let dot = 0;
+    let magA = 0;
+    let magB = 0;
+    for (let i = 0; i < len; i++) {
+        const ai = a[i];
+        const bi = b[i];
+        dot += ai * bi;
+        magA += ai * ai;
+        magB += bi * bi;
+    }
+    const denom = Math.sqrt(magA) * Math.sqrt(magB);
+    if (denom === 0) {
+        return 0;
+    }
+    return dot / denom;
+}
+/**
+ * Search a collection of embeddings for the top-K most similar to a query.
+ *
+ * @param query - The query embedding vector.
+ * @param embeddings - Array of embedding vectors to search.
+ * @param topK - Number of top results to return.
+ * @returns Array of `{ index, score }` sorted by score descending, length <= topK.
+ */
+export function searchEmbeddings(query, embeddings, topK) {
+    if (embeddings.length === 0 || topK <= 0) {
+        return [];
+    }
+    const k = Math.min(topK, embeddings.length);
+    // Compute all similarities
+    const scored = new Array(embeddings.length);
+    for (let i = 0; i < embeddings.length; i++) {
+        scored[i] = { index: i, score: cosineSimilarity(query, embeddings[i]) };
+    }
+    // Partial sort: only need top-K. For small K relative to N,
+    // a selection algorithm is faster than full sort, but for simplicity
+    // and correctness we sort and slice. At 50k embeddings this is <10ms.
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, k);
+}
+//# sourceMappingURL=search.js.map

package/dist/pipeline/search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.js","sourceRoot":"","sources":["../../pipeline/search.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,gBAAgB,CAAC,CAAe,EAAE,CAAe;IAC/D,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;IACrB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAChB,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;QACf,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,GAAG,GAAG,KAAK,CAAC;AACrB,CAAC;AAUD;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAC9B,KAAmB,EACnB,UAA0B,EAC1B,IAAY;IAEZ,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACzC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAE5C,2BAA2B;IAC3B,MAAM,MAAM,GAAmB,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1E,CAAC;IAED,4DAA4D;IAC5D,qEAAqE;IACrE,sEAAsE;IACtE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAEzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5B,CAAC"}

package/dist/pipeline/tokenizer.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Token counting for chunking decisions.
+ *
+ * Uses the real WordPiece tokenizer with the model vocabulary when available,
+ * falling back to a lightweight heuristic when the vocab hasn't been loaded.
+ */
+/**
+ * Set the vocabulary for accurate token counting.
+ * Call this once after the model/vocab has been downloaded.
+ *
+ * @param vocabPath - Absolute path to vocab.txt.
+ */
+export declare function setTokenizerVocab(vocabPath: string): void;
+/**
+ * Clear the tokenizer vocabulary (for testing).
+ */
+export declare function clearTokenizerVocab(): void;
+/**
+ * Count the number of tokens in a text string.
+ *
+ * If a vocabulary has been loaded via `setTokenizerVocab()`, returns
+ * an accurate WordPiece token count. Otherwise, falls back to the
+ * heuristic estimate (suitable for chunking decisions).
+ *
+ * @param text - The input text to count tokens for.
+ * @returns Token count (always >= 0).
+ */
+export declare function countTokens(text: string): number;
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/pipeline/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../pipeline/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAEzD;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,IAAI,CAE1C;AAYD;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAYhD"}

package/dist/pipeline/tokenizer.js

@@ -0,0 +1,54 @@
+/**
+ * Token counting for chunking decisions.
+ *
+ * Uses the real WordPiece tokenizer with the model vocabulary when available,
+ * falling back to a lightweight heuristic when the vocab hasn't been loaded.
+ */
+import { countWordPieceTokens, loadVocabulary } from './wordpiece-tokenizer.js';
+/** Cached vocabulary reference for token counting. */
+let cachedVocab = null;
+/**
+ * Set the vocabulary for accurate token counting.
+ * Call this once after the model/vocab has been downloaded.
+ *
+ * @param vocabPath - Absolute path to vocab.txt.
+ */
+export function setTokenizerVocab(vocabPath) {
+    cachedVocab = loadVocabulary(vocabPath);
+}
+/**
+ * Clear the tokenizer vocabulary (for testing).
+ */
+export function clearTokenizerVocab() {
+    cachedVocab = null;
+}
+/** Punctuation characters that act as token boundaries (for heuristic fallback). */
+const PUNCTUATION_RE = /[\s.,;:!?()\[\]{}'"\-\u2014\u2013\u2026]+/;
+/**
+ * Approximate multiplier to account for sub-word splitting.
+ * WordPiece / BPE tokenizers typically produce ~1.3x the number of
+ * whitespace-delimited words.
+ */
+const SUBWORD_MULTIPLIER = 1.3;
+/**
+ * Count the number of tokens in a text string.
+ *
+ * If a vocabulary has been loaded via `setTokenizerVocab()`, returns
+ * an accurate WordPiece token count. Otherwise, falls back to the
+ * heuristic estimate (suitable for chunking decisions).
+ *
+ * @param text - The input text to count tokens for.
+ * @returns Token count (always >= 0).
+ */
+export function countTokens(text) {
+    if (!text || text.trim().length === 0) {
+        return 0;
+    }
+    if (cachedVocab) {
+        return countWordPieceTokens(text, cachedVocab);
+    }
+    // Heuristic fallback when vocab is not loaded
+    const words = text.split(PUNCTUATION_RE).filter((w) => w.length > 0);
+    return Math.ceil(words.length * SUBWORD_MULTIPLIER);
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/pipeline/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../pipeline/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAmB,MAAM,0BAA0B,CAAC;AAEjG,sDAAsD;AACtD,IAAI,WAAW,GAAsB,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAAiB;IACjD,WAAW,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB;IACjC,WAAW,GAAG,IAAI,CAAC;AACrB,CAAC;AAED,oFAAoF;AACpF,MAAM,cAAc,GAAG,2CAA2C,CAAC;AAEnE;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B;;;;;;;;;GASG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,oBAAoB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACjD,CAAC;IAED,8CAA8C;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrE,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,kBAAkB,CAAC,CAAC;AACtD,CAAC"}