causantic 0.2.0

package/dist/models/embedder.js

@@ -0,0 +1,171 @@
+/**
+ * Embedding pipeline wrapper around @huggingface/transformers.
+ *
+ * Loads models sequentially and provides a simple embed() interface.
+ * Handles task prefixes for models that need them (nomic).
+ * Tracks memory usage for benchmark reporting.
+ * Supports true batch embedding for performance.
+ */
+import { pipeline } from '@huggingface/transformers';
+import { detectDevice } from './device-detector.js';
+import { createLogger } from '../utils/logger.js';
+const log = createLogger('embedder');
+/** Default batch size for true batch embedding to avoid OOM.
+ *  Keep small: attention memory scales O(batch * seq_len²). */
+const DEFAULT_BATCH_SIZE = 4;
+export class Embedder {
+    pipe = null;
+    config = null;
+    _device = null;
+    /**
+     * Load a model. Disposes any previously loaded model first.
+     */
+    async load(config, options = {}) {
+        await this.dispose();
+        const detection = detectDevice(options.device);
+        this._device = detection;
+        const heapBefore = process.memoryUsage().heapUsed;
+        const start = performance.now();
+        // Build pipeline options. All accelerated backends use onnxruntime-node
+        // (device: 'cpu' in transformers.js) with execution providers overridden.
+        // WASM uses device: undefined to let transformers.js pick its WASM backend.
+        const isWasm = detection.device === 'wasm';
+        const pipelineOptions = {
+            dtype: 'fp32',
+            ...(isWasm ? {} : { device: 'cpu' }),
+        };
+        // Inject execution providers for hardware acceleration
+        if (detection.executionProviders.length > 0) {
+            pipelineOptions.session_options = {
+                executionProviders: detection.executionProviders,
+            };
+        }
+        try {
+            this.pipe = await pipeline('feature-extraction', config.hfId, pipelineOptions);
+        }
+        catch (epError) {
+            // If accelerated EP fails, retry with plain CPU (no execution providers)
+            if (detection.device !== 'cpu' && detection.device !== 'wasm') {
+                log.warn(`${detection.label} failed, falling back to CPU`, {
+                    error: epError.message,
+                });
+                this._device = {
+                    device: 'cpu',
+                    executionProviders: ['cpu'],
+                    label: 'CPU (native, fallback)',
+                    source: detection.source,
+                    notes: `${detection.label} failed: ${epError.message}`,
+                };
+                this.pipe = await pipeline('feature-extraction', config.hfId, { dtype: 'fp32', device: 'cpu' });
+            }
+            else {
+                throw epError;
+            }
+        }
+        this.config = config;
+        const loadTimeMs = performance.now() - start;
+        const heapAfter = process.memoryUsage().heapUsed;
+        const heapUsedMB = (heapAfter - heapBefore) / (1024 * 1024);
+        log.info(`Loaded ${config.id} on ${this._device.label} in ${loadTimeMs.toFixed(0)}ms`);
+        return {
+            modelId: config.id,
+            loadTimeMs,
+            heapUsedMB: Math.max(0, heapUsedMB),
+        };
+    }
+    /**
+     * Embed a single text. Applies document prefix if the model needs it.
+     */
+    async embed(text, isQuery = false) {
+        if (!this.pipe || !this.config) {
+            throw new Error('No model loaded. Call load() first.');
+        }
+        const prefixed = this.config.usesPrefix
+            ? (isQuery ? this.config.queryPrefix : this.config.documentPrefix) + text
+            : text;
+        const start = performance.now();
+        const output = await this.pipe(prefixed, {
+            pooling: 'mean',
+            normalize: true,
+        });
+        const inferenceMs = performance.now() - start;
+        // Output is a Tensor — convert to number[]
+        const embedding = Array.from(output.data).slice(0, this.config.dims);
+        return { embedding, inferenceMs };
+    }
+    /**
+     * Embed multiple texts. Returns embeddings in the same order.
+     * @deprecated Use embedBatchTrue() for better performance.
+     */
+    async embedBatch(texts, isQuery = false) {
+        const results = [];
+        for (const text of texts) {
+            results.push(await this.embed(text, isQuery));
+        }
+        return results;
+    }
+    /**
+     * Embed multiple texts using true batch processing.
+     * Passes arrays directly to the HuggingFace pipeline for better performance.
+     * Processes in configurable batch sizes to avoid OOM on large batches.
+     *
+     * @param texts - Array of texts to embed
+     * @param isQuery - Whether these are query texts (affects prefix for some models)
+     * @param batchSize - Maximum texts per batch (default 32)
+     */
+    async embedBatchTrue(texts, isQuery = false, batchSize = DEFAULT_BATCH_SIZE) {
+        if (!this.pipe || !this.config) {
+            throw new Error('No model loaded. Call load() first.');
+        }
+        if (texts.length === 0) {
+            return [];
+        }
+        const results = [];
+        const dims = this.config.dims;
+        // Process in batches to avoid OOM
+        for (let i = 0; i < texts.length; i += batchSize) {
+            const batch = texts.slice(i, i + batchSize);
+            const prefixed = batch.map((t) => this.config.usesPrefix
+                ? (isQuery ? this.config.queryPrefix : this.config.documentPrefix) + t
+                : t);
+            const start = performance.now();
+            const output = await this.pipe(prefixed, {
+                pooling: 'mean',
+                normalize: true,
+            });
+            const inferenceMs = performance.now() - start;
+            // Extract embeddings from flattened tensor
+            const data = output.data;
+            for (let j = 0; j < batch.length; j++) {
+                const embedding = Array.from(data.slice(j * dims, (j + 1) * dims));
+                results.push({ embedding, inferenceMs: inferenceMs / batch.length });
+            }
+            // Free WASM tensor memory (not GC'd automatically)
+            if (typeof output.dispose === 'function') {
+                output.dispose();
+            }
+        }
+        return results;
+    }
+    /**
+     * Dispose the current model to free memory.
+     */
+    async dispose() {
+        if (this.pipe) {
+            await this.pipe.dispose();
+            this.pipe = null;
+            this.config = null;
+            this._device = null;
+            // Hint to GC
+            if (global.gc)
+                global.gc();
+        }
+    }
+    get currentModel() {
+        return this.config;
+    }
+    get currentDevice() {
+        return this._device;
+    }
+}
+//# sourceMappingURL=embedder.js.map

package/dist/models/embedder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/models/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,QAAQ,EAAkC,MAAM,2BAA2B,CAAC;AAErF,OAAO,EAAE,YAAY,EAA8B,MAAM,sBAAsB,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;AAErC;+DAC+D;AAC/D,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAoB7B,MAAM,OAAO,QAAQ;IACX,IAAI,GAAqC,IAAI,CAAC;IAC9C,MAAM,GAAuB,IAAI,CAAC;IAClC,OAAO,GAAiC,IAAI,CAAC;IAErD;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,MAAmB,EAAE,UAA+B,EAAE;QAC/D,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAErB,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAEzB,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;QAClD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEhC,wEAAwE;QACxE,0EAA0E;QAC1E,4EAA4E;QAC5E,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,KAAK,MAAM,CAAC;QAC3C,MAAM,eAAe,GAA4B;YAC/C,KAAK,EAAE,MAAM;YACb,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;SACrC,CAAC;QAEF,uDAAuD;QACvD,IAAI,SAAS,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,eAAe,CAAC,eAAe,GAAG;gBAChC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;aACjD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YAEH,IAAI,CAAC,IAAI,GAAG,MAAO,QAAgB,CACjC,oBAAoB,EAAE,MAAM,CAAC,IAAI,EAAE,eAAe,CACtB,CAAC;QACjC,CAAC;QAAC,OAAO,OAAO,EAAE,CAAC;YACjB,yEAAyE;YACzE,IAAI,SAAS,CAAC,MAAM,KAAK,KAAK,IAAI,SAAS,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC9D,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,KAAK,8BAA8B,EAAE;oBACzD,KAAK,EAAG,OAAiB,CAAC,OAAO;iBAClC,CAAC,CAAC;gBACH,IAAI,CAAC,OAAO,GAAG;oBACb,MAAM,EAAE,KAAK;oBACb,kBAAkB,EAAE,CAAC,KAAK,CAAC;oBAC3B,KAAK,EAAE,wBAAwB;oBAC/B,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,KAAK,EAAE,GAAG,SAAS,CAAC,KAAK,YAAa,OAAiB,CAAC,OAAO,EAAE;iBAClE,CAAC;gBAEF,IAAI,CAAC,IAAI,GAAG,MAAO,QAAgB,CACjC,oBAAoB,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CACvC,CAAC;YACjC,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,CAAC;YAChB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;QACjD,MAAM,UAAU,GAAG,CAAC,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;QAE5D,GAAG,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,EAAE,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEvF,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,EAAE;YAClB,UAAU;YACV,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC;SACpC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAmB,KAAK;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU;YACrC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,IAAI;YACzE,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;YACvC,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE9C,2CAA2C;QAC3C,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAoB,CAAC,CAAC,KAAK,CAC7D,CAAC,EACD,IAAI,CAAC,MAAM,CAAC,IAAI,CACjB,CAAC;QAEF,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,CACd,KAAe,EACf,UAAmB,KAAK;QAExB,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QAChD,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,cAAc,CAClB,KAAe,EACf,UAAmB,KAAK,EACxB,YAAoB,kBAAkB;QAEtC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAkB,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAE9B,kCAAkC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC5C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/B,IAAI,CAAC,MAAO,CAAC,UAAU;gBACrB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,MAAO,CAAC,cAAc,CAAC,GAAG,CAAC;gBACxE,CAAC,CAAC,CAAC,CACN,CAAC;YAEF,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBACvC,OAAO,EAAE,MAAM;gBACf,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YACH,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YAE9C,2CAA2C;YAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAoB,CAAC;YACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBACnE,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACvE,CAAC;YAED,mDAAmD;YACnD,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;gBACzC,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,aAAa;YACb,IAAI,MAAM,CAAC,EAAE;gBAAE,MAAM,CAAC,EAAE,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF"}

package/dist/models/model-registry.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Model configurations for embedding model candidates.
+ */
+export interface ModelConfig {
+    /** Short identifier. */
+    id: string;
+    /** HuggingFace model ID. */
+    hfId: string;
+    /** Embedding dimensions. */
+    dims: number;
+    /** Context window in tokens. */
+    contextTokens: number;
+    /** Whether the model uses task prefixes (e.g. nomic). */
+    usesPrefix: boolean;
+    /** Prefix for document embedding (if usesPrefix). */
+    documentPrefix: string;
+    /** Prefix for query embedding (if usesPrefix). */
+    queryPrefix: string;
+    /** Notes about the model. */
+    notes: string;
+}
+export declare const MODEL_REGISTRY: Record<string, ModelConfig>;
+export declare function getModel(id: string): ModelConfig;
+export declare function getAllModelIds(): string[];
+//# sourceMappingURL=model-registry.d.ts.map

package/dist/models/model-registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-registry.d.ts","sourceRoot":"","sources":["../../src/models/model-registry.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,WAAW;IAC1B,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,yDAAyD;IACzD,UAAU,EAAE,OAAO,CAAC;IACpB,qDAAqD;IACrD,cAAc,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAyCtD,CAAC;AAEF,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,CAQhD;AAED,wBAAgB,cAAc,IAAI,MAAM,EAAE,CAEzC"}

package/dist/models/model-registry.js

@@ -0,0 +1,56 @@
+/**
+ * Model configurations for embedding model candidates.
+ */
+export const MODEL_REGISTRY = {
+    'nomic-v1.5': {
+        id: 'nomic-v1.5',
+        hfId: 'nomic-ai/nomic-embed-text-v1.5',
+        dims: 768,
+        contextTokens: 8192,
+        usesPrefix: true,
+        documentPrefix: 'search_document: ',
+        queryPrefix: 'search_query: ',
+        notes: 'Matryoshka dimensions (768/512/384/256/128). Needs task prefixes.',
+    },
+    'jina-code': {
+        id: 'jina-code',
+        hfId: 'jinaai/jina-embeddings-v2-base-code',
+        dims: 768,
+        contextTokens: 8192,
+        usesPrefix: false,
+        documentPrefix: '',
+        queryPrefix: '',
+        notes: 'Code-specialized. ONNX availability needs validation.',
+    },
+    'jina-small': {
+        id: 'jina-small',
+        hfId: 'Xenova/jina-embeddings-v2-small-en',
+        dims: 512,
+        contextTokens: 8192,
+        usesPrefix: false,
+        documentPrefix: '',
+        queryPrefix: '',
+        notes: 'Confirmed ONNX via Xenova.',
+    },
+    'bge-small': {
+        id: 'bge-small',
+        hfId: 'Xenova/bge-small-en-v1.5',
+        dims: 384,
+        contextTokens: 512,
+        usesPrefix: false,
+        documentPrefix: '',
+        queryPrefix: '',
+        notes: 'Baseline, smallest model. 512 token context limit.',
+    },
+};
+export function getModel(id) {
+    const config = MODEL_REGISTRY[id];
+    if (!config) {
+        throw new Error(`Unknown model: ${id}. Available: ${Object.keys(MODEL_REGISTRY).join(', ')}`);
+    }
+    return config;
+}
+export function getAllModelIds() {
+    return Object.keys(MODEL_REGISTRY);
+}
+//# sourceMappingURL=model-registry.js.map

package/dist/models/model-registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-registry.js","sourceRoot":"","sources":["../../src/models/model-registry.ts"],"names":[],"mappings":"AAAA;;GAEG;AAqBH,MAAM,CAAC,MAAM,cAAc,GAAgC;IACzD,YAAY,EAAE;QACZ,EAAE,EAAE,YAAY;QAChB,IAAI,EAAE,gCAAgC;QACtC,IAAI,EAAE,GAAG;QACT,aAAa,EAAE,IAAI;QACnB,UAAU,EAAE,IAAI;QAChB,cAAc,EAAE,mBAAmB;QACnC,WAAW,EAAE,gBAAgB;QAC7B,KAAK,EAAE,mEAAmE;KAC3E;IACD,WAAW,EAAE;QACX,EAAE,EAAE,WAAW;QACf,IAAI,EAAE,qCAAqC;QAC3C,IAAI,EAAE,GAAG;QACT,aAAa,EAAE,IAAI;QACnB,UAAU,EAAE,KAAK;QACjB,cAAc,EAAE,EAAE;QAClB,WAAW,EAAE,EAAE;QACf,KAAK,EAAE,uDAAuD;KAC/D;IACD,YAAY,EAAE;QACZ,EAAE,EAAE,YAAY;QAChB,IAAI,EAAE,oCAAoC;QAC1C,IAAI,EAAE,GAAG;QACT,aAAa,EAAE,IAAI;QACnB,UAAU,EAAE,KAAK;QACjB,cAAc,EAAE,EAAE;QAClB,WAAW,EAAE,EAAE;QACf,KAAK,EAAE,4BAA4B;KACpC;IACD,WAAW,EAAE;QACX,EAAE,EAAE,WAAW;QACf,IAAI,EAAE,0BAA0B;QAChC,IAAI,EAAE,GAAG;QACT,aAAa,EAAE,GAAG;QAClB,UAAU,EAAE,KAAK;QACjB,cAAc,EAAE,EAAE;QAClB,WAAW,EAAE,EAAE;QACf,KAAK,EAAE,oDAAoD;KAC5D;CACF,CAAC;AAEF,MAAM,UAAU,QAAQ,CAAC,EAAU;IACjC,MAAM,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,kBAAkB,EAAE,gBAAgB,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;AACrC,CAAC"}

package/dist/parser/chunker.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Turn-based, code-block-aware chunking.
+ *
+ * Strategy:
+ * 1. Primary boundary: turns (each turn = one user prompt + response cycle)
+ * 2. Oversized turns split between tool exchanges or code blocks (never mid-code-block)
+ * 3. Trivially small turns merged with neighbors
+ * 4. Text rendering with structure markers for embedding
+ */
+import type { Chunk, ChunkMetadata, RenderMode, Turn } from './types.js';
+import type { VectorClock } from '../temporal/vector-clock.js';
+export interface ChunkerOptions {
+    /** Target max tokens per chunk. Default: 4096. */
+    maxTokens?: number;
+    /** Minimum tokens before a turn gets merged. Default: 64. */
+    minTokens?: number;
+    /** Include thinking blocks in rendered text. Default: true. */
+    includeThinking?: boolean;
+    /** Render mode. Default: 'full'. */
+    renderMode?: RenderMode;
+    /** Session ID for chunk metadata. */
+    sessionId: string;
+    /** Session slug for chunk metadata. */
+    sessionSlug: string;
+}
+/**
+ * Extended options for clock-aware chunking.
+ */
+export interface ChunkerOptionsWithClock extends ChunkerOptions {
+    /** Agent ID for this session (default: MAIN_AGENT_ID). */
+    agentId?: string;
+    /** Initial vector clock state. */
+    initialClock?: VectorClock;
+    /** Callback invoked when clock ticks (each turn = one tick). */
+    onTick?: (clock: VectorClock) => void;
+    /** Spawn depth for sub-agent chunks. Default: 0. */
+    spawnDepth?: number;
+}
+/**
+ * Extended chunk metadata with vector clock info.
+ */
+export interface ChunkMetadataWithClock extends ChunkMetadata {
+    /** Agent that created this chunk. */
+    agentId?: string;
+    /** Vector clock at chunk creation time. */
+    vectorClock?: VectorClock;
+    /** Spawn depth: 0 = main, 1 = sub-agent, etc. */
+    spawnDepth?: number;
+}
+/**
+ * Extended chunk with clock metadata.
+ */
+export interface ChunkWithClock extends Chunk {
+    metadata: ChunkMetadataWithClock;
+}
+/**
+ * Render a turn to flat text with structure markers.
+ */
+export declare function renderTurn(turn: Turn, mode?: RenderMode, includeThinking?: boolean): string;
+/**
+ * Chunk turns into embeddable text chunks.
+ */
+export declare function chunkTurns(turns: Turn[], options: ChunkerOptions): Chunk[];
+/** Reset chunk counter (for testing). */
+export declare function resetChunkCounter(): void;
+/**
+ * Chunk turns with vector clock tracking.
+ * Each turn = one D-T-D cycle = one tick on the agent's clock.
+ *
+ * The human also contributes to the clock - each user message
+ * ticks the HUMAN_AGENT_ID before the main agent ticks.
+ */
+export declare function chunkTurnsWithClock(turns: Turn[], options: ChunkerOptionsWithClock): ChunkWithClock[];
+//# sourceMappingURL=chunker.d.ts.map

package/dist/parser/chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/parser/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EACV,KAAK,EACL,aAAa,EAEb,UAAU,EAKV,IAAI,EACL,MAAM,YAAY,CAAC;AAEpB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAG/D,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oCAAoC;IACpC,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,cAAc;IAC7D,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kCAAkC;IAClC,YAAY,CAAC,EAAE,WAAW,CAAC;IAC3B,gEAAgE;IAChE,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;IACtC,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,aAAa;IAC3D,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAe,SAAQ,KAAK;IAC3C,QAAQ,EAAE,sBAAsB,CAAC;CAClC;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,IAAI,EACV,IAAI,GAAE,UAAmB,EACzB,eAAe,GAAE,OAAc,GAC9B,MAAM,CAsCR;AAuFD;;GAEG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,cAAc,GAAG,KAAK,EAAE,CAoF1E;AAsBD,yCAAyC;AACzC,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,uBAAuB,GAAG,cAAc,EAAE,CAyHrG"}

package/dist/parser/chunker.js

@@ -0,0 +1,330 @@
+/**
+ * Turn-based, code-block-aware chunking.
+ *
+ * Strategy:
+ * 1. Primary boundary: turns (each turn = one user prompt + response cycle)
+ * 2. Oversized turns split between tool exchanges or code blocks (never mid-code-block)
+ * 3. Trivially small turns merged with neighbors
+ * 4. Text rendering with structure markers for embedding
+ */
+import { approximateTokens } from '../utils/token-counter.js';
+import { tick, MAIN_AGENT_ID, HUMAN_AGENT_ID } from '../temporal/vector-clock.js';
+/**
+ * Render a turn to flat text with structure markers.
+ */
+export function renderTurn(turn, mode = 'full', includeThinking = true) {
+    const parts = [];
+    // User message
+    if (turn.userText) {
+        parts.push(`[User]\n${turn.userText}`);
+    }
+    // Assistant content
+    for (const block of turn.assistantBlocks) {
+        switch (block.type) {
+            case 'thinking':
+                if (includeThinking) {
+                    parts.push(`[Thinking]\n${block.thinking}`);
+                }
+                break;
+            case 'text':
+                parts.push(`[Assistant]\n${block.text}`);
+                break;
+            case 'tool_use':
+                if (mode === 'full') {
+                    const tu = block;
+                    parts.push(`[Tool:${tu.name}]\n${summarizeToolInput(tu)}`);
+                }
+                break;
+        }
+    }
+    // Tool results
+    if (mode === 'full' || mode === 'code-focused') {
+        for (const ex of turn.toolExchanges) {
+            if (mode === 'code-focused' && !isCodeTool(ex.toolName))
+                continue;
+            const resultPreview = truncateResult(ex.result, 500);
+            parts.push(`[Result:${ex.toolName}]\n${resultPreview}`);
+        }
+    }
+    return parts.join('\n\n');
+}
+function summarizeToolInput(tu) {
+    const input = tu.input;
+    // Common patterns
+    if ('command' in input)
+        return String(input.command);
+    if ('pattern' in input)
+        return String(input.pattern);
+    if ('file_path' in input)
+        return String(input.file_path);
+    if ('query' in input)
+        return String(input.query);
+    if ('url' in input)
+        return String(input.url);
+    return JSON.stringify(input).slice(0, 200);
+}
+function isCodeTool(name) {
+    return ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'].includes(name);
+}
+function truncateResult(text, maxChars) {
+    if (text.length <= maxChars)
+        return text;
+    return text.slice(0, maxChars) + '\n...[truncated]';
+}
+/**
+ * Count code blocks in a turn's text content.
+ */
+function countCodeBlocks(turn) {
+    let count = 0;
+    for (const block of turn.assistantBlocks) {
+        if (block.type === 'text') {
+            const matches = block.text.match(/```/g);
+            count += matches ? Math.floor(matches.length / 2) : 0;
+        }
+    }
+    return count;
+}
+/**
+ * Split an oversized turn's rendered text at natural boundaries.
+ * Splits between tool exchanges or between paragraphs, never mid-code-block.
+ */
+function splitRenderedText(text, maxTokens) {
+    const tokens = approximateTokens(text);
+    if (tokens <= maxTokens)
+        return [text];
+    // Try splitting at section markers first
+    const sections = text.split(/\n\n(?=\[(?:User|Assistant|Tool|Result|Thinking)\])/);
+    if (sections.length <= 1) {
+        // Can't split at markers, split at paragraph boundaries
+        return splitAtParagraphs(text, maxTokens);
+    }
+    const chunks = [];
+    let current = '';
+    for (const section of sections) {
+        const combined = current ? current + '\n\n' + section : section;
+        if (approximateTokens(combined) > maxTokens && current) {
+            chunks.push(current);
+            current = section;
+        }
+        else {
+            current = combined;
+        }
+    }
+    if (current)
+        chunks.push(current);
+    return chunks;
+}
+function splitAtParagraphs(text, maxTokens) {
+    const paragraphs = text.split('\n\n');
+    const chunks = [];
+    let current = '';
+    for (const para of paragraphs) {
+        const combined = current ? current + '\n\n' + para : para;
+        if (approximateTokens(combined) > maxTokens && current) {
+            chunks.push(current);
+            current = para;
+        }
+        else {
+            current = combined;
+        }
+    }
+    if (current)
+        chunks.push(current);
+    return chunks;
+}
+/**
+ * Chunk turns into embeddable text chunks.
+ */
+export function chunkTurns(turns, options) {
+    const { maxTokens = 4096, minTokens = 64, includeThinking = true, renderMode = 'full', sessionId, sessionSlug, } = options;
+    // First pass: render each turn
+    const rendered = turns.map((turn) => ({
+        turn,
+        text: renderTurn(turn, renderMode, includeThinking),
+    }));
+    // Second pass: merge small turns, split large turns
+    const chunks = [];
+    let mergeBuffer = { turns: [], text: '' };
+    function flushBuffer() {
+        if (!mergeBuffer.text)
+            return;
+        const texts = splitRenderedText(mergeBuffer.text, maxTokens);
+        for (const text of texts) {
+            const turnIndices = mergeBuffer.turns.map((t) => t.index);
+            const firstTurn = mergeBuffer.turns[0];
+            const lastTurn = mergeBuffer.turns[mergeBuffer.turns.length - 1];
+            chunks.push(makeChunk(text, {
+                sessionId,
+                sessionSlug,
+                turnIndices,
+                startTime: firstTurn.startTime,
+                endTime: lastTurn.startTime,
+                codeBlockCount: mergeBuffer.turns.reduce((n, t) => n + countCodeBlocks(t), 0),
+                toolUseCount: mergeBuffer.turns.reduce((n, t) => n + t.toolExchanges.length, 0),
+                hasThinking: mergeBuffer.turns.some((t) => t.hasThinking),
+                renderMode,
+                approxTokens: approximateTokens(text),
+            }));
+        }
+        mergeBuffer = { turns: [], text: '' };
+    }
+    for (const { turn, text } of rendered) {
+        const tokens = approximateTokens(text);
+        if (tokens < minTokens) {
+            // Merge with buffer
+            const combined = mergeBuffer.text
+                ? mergeBuffer.text + '\n\n' + text
+                : text;
+            if (approximateTokens(combined) > maxTokens) {
+                flushBuffer();
+                mergeBuffer = { turns: [turn], text };
+            }
+            else {
+                mergeBuffer.turns.push(turn);
+                mergeBuffer.text = combined;
+            }
+        }
+        else if (tokens > maxTokens) {
+            // Flush buffer first, then split this turn
+            flushBuffer();
+            mergeBuffer = { turns: [turn], text };
+            flushBuffer();
+        }
+        else {
+            // Normal sized turn
+            if (mergeBuffer.text) {
+                const combined = mergeBuffer.text + '\n\n' + text;
+                if (approximateTokens(combined) <= maxTokens) {
+                    mergeBuffer.turns.push(turn);
+                    mergeBuffer.text = combined;
+                    continue;
+                }
+                flushBuffer();
+            }
+            mergeBuffer = { turns: [turn], text };
+        }
+    }
+    flushBuffer();
+    return chunks;
+}
+let chunkCounter = 0;
+function makeChunk(text, metadata) {
+    chunkCounter++;
+    return {
+        id: `${metadata.sessionId}-chunk-${chunkCounter}`,
+        text,
+        metadata,
+    };
+}
+function makeChunkWithClock(text, metadata) {
+    chunkCounter++;
+    return {
+        id: `${metadata.sessionId}-chunk-${chunkCounter}`,
+        text,
+        metadata,
+    };
+}
+/** Reset chunk counter (for testing). */
+export function resetChunkCounter() {
+    chunkCounter = 0;
+}
+/**
+ * Chunk turns with vector clock tracking.
+ * Each turn = one D-T-D cycle = one tick on the agent's clock.
+ *
+ * The human also contributes to the clock - each user message
+ * ticks the HUMAN_AGENT_ID before the main agent ticks.
+ */
+export function chunkTurnsWithClock(turns, options) {
+    const { maxTokens = 4096, minTokens = 64, includeThinking = true, renderMode = 'full', sessionId, sessionSlug, agentId = MAIN_AGENT_ID, initialClock = {}, onTick, spawnDepth = 0, } = options;
+    let clock = { ...initialClock };
+    // First pass: render each turn with clock tracking
+    const rendered = [];
+    for (const turn of turns) {
+        // Human contributed a message - tick their clock
+        if (turn.userText) {
+            clock = tick(clock, HUMAN_AGENT_ID);
+        }
+        // Agent processed this turn - tick their clock
+        clock = tick(clock, agentId);
+        // Notify callback of clock update
+        if (onTick) {
+            onTick(clock);
+        }
+        rendered.push({
+            turn,
+            text: renderTurn(turn, renderMode, includeThinking),
+            clock: { ...clock }, // Snapshot the clock at this turn
+        });
+    }
+    // Second pass: merge small turns, split large turns
+    const chunks = [];
+    let mergeBuffer = { turns: [], text: '', clock: {} };
+    function flushBuffer() {
+        if (!mergeBuffer.text)
+            return;
+        const texts = splitRenderedText(mergeBuffer.text, maxTokens);
+        for (const text of texts) {
+            const turnIndices = mergeBuffer.turns.map((t) => t.index);
+            const firstTurn = mergeBuffer.turns[0];
+            const lastTurn = mergeBuffer.turns[mergeBuffer.turns.length - 1];
+            chunks.push(makeChunkWithClock(text, {
+                sessionId,
+                sessionSlug,
+                turnIndices,
+                startTime: firstTurn.startTime,
+                endTime: lastTurn.startTime,
+                codeBlockCount: mergeBuffer.turns.reduce((n, t) => n + countCodeBlocks(t), 0),
+                toolUseCount: mergeBuffer.turns.reduce((n, t) => n + t.toolExchanges.length, 0),
+                hasThinking: mergeBuffer.turns.some((t) => t.hasThinking),
+                renderMode,
+                approxTokens: approximateTokens(text),
+                agentId,
+                vectorClock: mergeBuffer.clock,
+                spawnDepth,
+            }));
+        }
+        mergeBuffer = { turns: [], text: '', clock: {} };
+    }
+    for (const { turn, text, clock: turnClock } of rendered) {
+        const tokens = approximateTokens(text);
+        if (tokens < minTokens) {
+            // Merge with buffer
+            const combined = mergeBuffer.text
+                ? mergeBuffer.text + '\n\n' + text
+                : text;
+            if (approximateTokens(combined) > maxTokens) {
+                flushBuffer();
+                mergeBuffer = { turns: [turn], text, clock: turnClock };
+            }
+            else {
+                mergeBuffer.turns.push(turn);
+                mergeBuffer.text = combined;
+                mergeBuffer.clock = turnClock; // Use latest clock
+            }
+        }
+        else if (tokens > maxTokens) {
+            // Flush buffer first, then split this turn
+            flushBuffer();
+            mergeBuffer = { turns: [turn], text, clock: turnClock };
+            flushBuffer();
+        }
+        else {
+            // Normal sized turn
+            if (mergeBuffer.text) {
+                const combined = mergeBuffer.text + '\n\n' + text;
+                if (approximateTokens(combined) <= maxTokens) {
+                    mergeBuffer.turns.push(turn);
+                    mergeBuffer.text = combined;
+                    mergeBuffer.clock = turnClock;
+                    continue;
+                }
+                flushBuffer();
+            }
+            mergeBuffer = { turns: [turn], text, clock: turnClock };
+        }
+    }
+    flushBuffer();
+    return chunks;
+}
+//# sourceMappingURL=chunker.js.map