causantic 0.2.0

package/dist/report/reporter.js

@@ -0,0 +1,121 @@
+/**
+ * Console table + JSON output for benchmark results.
+ */
+import { writeFile } from 'node:fs/promises';
+/**
+ * Print a comparison table to the console.
+ */
+export function printComparisonTable(result) {
+    console.log('\n' + '='.repeat(100));
+    console.log('  EMBEDDING MODEL BENCHMARK RESULTS');
+    console.log('='.repeat(100));
+    console.log(`  Corpus: ${result.corpus.chunkCount} chunks, ${result.corpus.pairCount} labeled pairs`);
+    console.log('='.repeat(100));
+    // Header
+    const cols = [
+        pad('Model', 14),
+        pad('Dims', 5),
+        pad('Context', 8),
+        pad('ROC AUC', 8),
+        pad('Clusters', 9),
+        pad('Noise%', 7),
+        pad('Silh.', 7),
+        pad('CodeNL', 7),
+        pad('ms/chunk', 9),
+        pad('Load(s)', 8),
+        pad('Heap(MB)', 9),
+    ];
+    console.log(cols.join(' | '));
+    console.log('-'.repeat(100));
+    // Rows
+    for (const m of result.models) {
+        const row = [
+            pad(m.modelId, 14),
+            pad(String(m.modelConfig.dims), 5),
+            pad(String(m.modelConfig.contextTokens), 8),
+            pad(m.rocAuc.toFixed(3), 8),
+            pad(String(m.clusterCount), 9),
+            pad((m.noiseRatio * 100).toFixed(1) + '%', 7),
+            pad(m.silhouetteScore.toFixed(3), 7),
+            pad(m.codeNLAlignment.alignmentRatio.toFixed(3), 7),
+            pad(m.meanInferenceMs.toFixed(1), 9),
+            pad((m.loadStats.loadTimeMs / 1000).toFixed(1), 8),
+            pad(m.loadStats.heapUsedMB.toFixed(0), 9),
+        ];
+        console.log(row.join(' | '));
+    }
+    console.log('='.repeat(100));
+    // Context window comparison
+    if (result.contextWindowComparison) {
+        const cw = result.contextWindowComparison;
+        console.log('\nContext Window Impact:');
+        console.log(`  Long chunks (>512 tokens): ${cw.longChunks} / ${cw.totalChunks}`);
+        console.log(`  Mean drift (long chunks):  ${cw.meanDriftLongChunks.toFixed(4)}`);
+        console.log(`  Mean drift (short chunks): ${cw.meanDriftShortChunks.toFixed(4)}`);
+        if (cw.longChunkDrifts.length > 0) {
+            console.log('  Top 5 drifting chunks:');
+            for (const d of cw.longChunkDrifts.slice(0, 5)) {
+                console.log(`    ${d.chunkId}: ${d.tokens} tokens, drift=${d.drift.toFixed(4)}`);
+            }
+        }
+    }
+    // Per-model cluster details
+    console.log('\nCluster Details:');
+    for (const m of result.models) {
+        console.log(`  ${m.modelId}: ${m.clusterResult.clusterSizes.join(', ')} (${m.clusterResult.numClusters} clusters)`);
+    }
+    console.log('');
+}
+/**
+ * Print cluster membership with chunk text previews.
+ */
+export function printClusterMembership(modelResult, chunks, maxPreviewChars = 80) {
+    console.log(`\n--- Cluster membership for ${modelResult.modelId} ---`);
+    for (const [label, indices] of modelResult.clusterMembership) {
+        const labelStr = label < 0 ? 'NOISE' : `Cluster ${label}`;
+        console.log(`\n  ${labelStr} (${indices.length} chunks):`);
+        for (const idx of indices.slice(0, 10)) {
+            const chunk = chunks[idx];
+            const preview = chunk.text
+                .replace(/\n/g, ' ')
+                .slice(0, maxPreviewChars);
+            console.log(`    [${chunk.metadata.sessionSlug}] ${preview}...`);
+        }
+        if (indices.length > 10) {
+            console.log(`    ... and ${indices.length - 10} more`);
+        }
+    }
+}
+/**
+ * Write benchmark results as JSON.
+ */
+export async function writeJsonReport(result, outputPath) {
+    // Strip non-serializable data (embeddings, Maps)
+    const serializable = {
+        ...result,
+        models: result.models.map((m) => ({
+            modelId: m.modelId,
+            modelConfig: m.modelConfig,
+            loadStats: m.loadStats,
+            rocAuc: m.rocAuc,
+            clusterCount: m.clusterCount,
+            noiseRatio: m.noiseRatio,
+            silhouetteScore: m.silhouetteScore,
+            codeNLAlignment: m.codeNLAlignment,
+            meanInferenceMs: m.meanInferenceMs,
+            totalInferenceMs: m.totalInferenceMs,
+            clusterResult: {
+                numClusters: m.clusterResult.numClusters,
+                noiseRatio: m.clusterResult.noiseRatio,
+                clusterSizes: m.clusterResult.clusterSizes,
+            },
+            clusterMembership: Object.fromEntries(m.clusterMembership),
+        })),
+    };
+    await writeFile(outputPath, JSON.stringify(serializable, null, 2));
+    console.log(`Report written to ${outputPath}`);
+}
+function pad(str, width) {
+    return str.padEnd(width).slice(0, width);
+}
+//# sourceMappingURL=reporter.js.map

package/dist/report/reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/report/reporter.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7C;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAuB;IAC1D,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CACT,aAAa,MAAM,CAAC,MAAM,CAAC,UAAU,YAAY,MAAM,CAAC,MAAM,CAAC,SAAS,gBAAgB,CACzF,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE7B,SAAS;IACT,MAAM,IAAI,GAAG;QACX,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;QAChB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACd,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACjB,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACjB,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAClB,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAChB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACf,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAChB,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAClB,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACjB,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;KACnB,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE7B,OAAO;IACP,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG;YACV,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YAClB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAC3C,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3B,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAC9B,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;YAC7C,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpC,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnD,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAClD,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SAC1C,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE7B,4BAA4B;IAC5B,IAAI,MAAM,CAAC,uBAAuB,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,MAAM,CAAC,uBAAuB,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CACT,gCAAgC,EAAE,CAAC,UAAU,MAAM,EAAE,CAAC,WAAW,EAAE,CACpE,CAAC;QACF,OAAO,CAAC,GAAG,CACT,gCAAgC,EAAE,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACpE,CAAC;QACF,OAAO,CAAC,GAAG,CACT,gCAAgC,EAAE,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;QACF,IAAI,EAAE,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;YACxC,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC/C,OAAO,CAAC,GAAG,CACT,OAAO,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,kBAAkB,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACpE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,aAAa,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,CAAC,WAAW,YAAY,CACvG,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CACpC,WAAiC,EACjC,MAAe,EACf,kBAA0B,EAAE;IAE5B,OAAO,CAAC,GAAG,CAAC,gCAAgC,WAAW,CAAC,OAAO,MAAM,CAAC,CAAC;IAEvE,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,WAAW,CAAC,iBAAiB,EAAE,CAAC;QAC7D,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,KAAK,EAAE,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,OAAO,QAAQ,KAAK,OAAO,CAAC,MAAM,WAAW,CAAC,CAAC;QAC3D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI;iBACvB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;iBACnB,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;YAC7B,OAAO,CAAC,GAAG,CACT,QAAQ,KAAK,CAAC,QAAQ,CAAC,WAAW,KAAK,OAAO,KAAK,CACpD,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAuB,EACvB,UAAkB;IAElB,iDAAiD;IACjD,MAAM,YAAY,GAAG;QACnB,GAAG,MAAM;QACT,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChC,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;YACpC,aAAa,EAAE;gBACb,WAAW,EAAE,CAAC,CAAC,aAAa,CAAC,WAAW;gBACxC,UAAU,EAAE,CAAC,CAAC,aAAa,CAAC,UAAU;gBACtC,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,YAAY;aAC3C;YACD,iBAAiB,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC;SAC3D,CAAC,CAAC;KACJ,CAAC;IAEF,MAAM,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,GAAG,CAAC,GAAW,EAAE,KAAa;IACrC,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAC3C,CAAC"}

package/dist/retrieval/cluster-expander.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Post-fusion cluster expansion.
+ *
+ * Expands search results by finding sibling chunks within the same HDBSCAN clusters.
+ * This surfaces topically related chunks that neither vector nor keyword search found.
+ */
+import type { RankedItem } from './rrf.js';
+export interface ClusterExpansionConfig {
+    /** Max clusters to expand from (default: 3) */
+    maxClusters: number;
+    /** Max sibling chunks per cluster (default: 5) */
+    maxSiblings: number;
+    /** Score multiplier for siblings (default: 0.3) */
+    boostFactor: number;
+}
+export declare const DEFAULT_CLUSTER_EXPANSION: ClusterExpansionConfig;
+/**
+ * Expand hits with cluster siblings.
+ *
+ * For each hit, looks up its cluster assignments, then fetches sibling chunks
+ * from the top clusters. Siblings get scored based on the hit's score,
+ * the boost factor, and their distance to the cluster centroid.
+ *
+ * @param hits - Ranked items from RRF fusion
+ * @param config - Expansion parameters
+ * @param projectFilter - Optional project filter to restrict siblings
+ * @returns Original hits + cluster sibling items (deduplicated)
+ */
+export declare function expandViaClusters(hits: RankedItem[], config?: ClusterExpansionConfig, projectFilter?: string | string[]): RankedItem[];
+//# sourceMappingURL=cluster-expander.d.ts.map

package/dist/retrieval/cluster-expander.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cluster-expander.d.ts","sourceRoot":"","sources":["../../src/retrieval/cluster-expander.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C,MAAM,WAAW,sBAAsB;IACrC,+CAA+C;IAC/C,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,yBAAyB,EAAE,sBAIvC,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,UAAU,EAAE,EAClB,MAAM,GAAE,sBAAkD,EAC1D,aAAa,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAChC,UAAU,EAAE,CAkEd"}

package/dist/retrieval/cluster-expander.js

@@ -0,0 +1,87 @@
+/**
+ * Post-fusion cluster expansion.
+ *
+ * Expands search results by finding sibling chunks within the same HDBSCAN clusters.
+ * This surfaces topically related chunks that neither vector nor keyword search found.
+ */
+import { getChunkClusterAssignments, getClusterChunkIds, } from '../storage/cluster-store.js';
+import { vectorStore } from '../storage/vector-store.js';
+export const DEFAULT_CLUSTER_EXPANSION = {
+    maxClusters: 3,
+    maxSiblings: 5,
+    boostFactor: 0.3,
+};
+/**
+ * Expand hits with cluster siblings.
+ *
+ * For each hit, looks up its cluster assignments, then fetches sibling chunks
+ * from the top clusters. Siblings get scored based on the hit's score,
+ * the boost factor, and their distance to the cluster centroid.
+ *
+ * @param hits - Ranked items from RRF fusion
+ * @param config - Expansion parameters
+ * @param projectFilter - Optional project filter to restrict siblings
+ * @returns Original hits + cluster sibling items (deduplicated)
+ */
+export function expandViaClusters(hits, config = DEFAULT_CLUSTER_EXPANSION, projectFilter) {
+    if (hits.length === 0)
+        return [];
+    const existingIds = new Set(hits.map(h => h.chunkId));
+    const siblingItems = [];
+    // Build project filter set
+    const projectSet = projectFilter
+        ? new Set(Array.isArray(projectFilter) ? projectFilter : [projectFilter])
+        : null;
+    // Track clusters we've already expanded to avoid duplicates
+    const expandedClusters = new Set();
+    let clusterCount = 0;
+    for (const hit of hits) {
+        if (clusterCount >= config.maxClusters)
+            break;
+        // Get cluster assignments for this chunk (sorted by distance)
+        const assignments = getChunkClusterAssignments(hit.chunkId);
+        if (assignments.length === 0)
+            continue;
+        for (const assignment of assignments) {
+            if (clusterCount >= config.maxClusters)
+                break;
+            if (expandedClusters.has(assignment.clusterId))
+                continue;
+            expandedClusters.add(assignment.clusterId);
+            clusterCount++;
+            // Get all chunk IDs in this cluster
+            const clusterChunkIds = getClusterChunkIds(assignment.clusterId);
+            let addedSiblings = 0;
+            for (const siblingId of clusterChunkIds) {
+                if (addedSiblings >= config.maxSiblings)
+                    break;
+                if (existingIds.has(siblingId))
+                    continue;
+                // Filter by project if needed
+                if (projectSet) {
+                    const project = vectorStore.getChunkProject(siblingId);
+                    if (!project || !projectSet.has(project))
+                        continue;
+                }
+                // Look up this sibling's distance in the cluster for scoring
+                const siblingAssignments = getChunkClusterAssignments(siblingId);
+                const siblingAssignment = siblingAssignments.find(a => a.clusterId === assignment.clusterId);
+                const distance = siblingAssignment?.distance ?? 0.5;
+                // Score: best hit score * boost * (1 - distance)
+                const score = hit.score * config.boostFactor * (1 - distance);
+                if (score > 0) {
+                    siblingItems.push({
+                        chunkId: siblingId,
+                        score,
+                        source: 'cluster',
+                    });
+                    existingIds.add(siblingId);
+                    addedSiblings++;
+                }
+            }
+        }
+    }
+    // Combine original hits with siblings
+    return [...hits, ...siblingItems];
+}
+//# sourceMappingURL=cluster-expander.js.map

package/dist/retrieval/cluster-expander.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cluster-expander.js","sourceRoot":"","sources":["../../src/retrieval/cluster-expander.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAYzD,MAAM,CAAC,MAAM,yBAAyB,GAA2B;IAC/D,WAAW,EAAE,CAAC;IACd,WAAW,EAAE,CAAC;IACd,WAAW,EAAE,GAAG;CACjB,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAC/B,IAAkB,EAClB,SAAiC,yBAAyB,EAC1D,aAAiC;IAEjC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACtD,MAAM,YAAY,GAAiB,EAAE,CAAC;IAEtC,2BAA2B;IAC3B,MAAM,UAAU,GAAG,aAAa;QAC9B,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;QACzE,CAAC,CAAC,IAAI,CAAC;IAET,4DAA4D;IAC5D,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC3C,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,YAAY,IAAI,MAAM,CAAC,WAAW;YAAE,MAAM;QAE9C,8DAA8D;QAC9D,MAAM,WAAW,GAAG,0BAA0B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC5D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEvC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,IAAI,YAAY,IAAI,MAAM,CAAC,WAAW;gBAAE,MAAM;YAC9C,IAAI,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;gBAAE,SAAS;YAEzD,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAC3C,YAAY,EAAE,CAAC;YAEf,oCAAoC;YACpC,MAAM,eAAe,GAAG,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YACjE,IAAI,aAAa,GAAG,CAAC,CAAC;YAEtB,KAAK,MAAM,SAAS,IAAI,eAAe,EAAE,CAAC;gBACxC,IAAI,aAAa,IAAI,MAAM,CAAC,WAAW;oBAAE,MAAM;gBAC/C,IAAI,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC;oBAAE,SAAS;gBAEzC,8BAA8B;gBAC9B,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,GAAG,WAAW,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;oBACvD,IAAI,CAAC,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC;wBAAE,SAAS;gBACrD,CAAC;gBAED,6DAA6D;gBAC7D,MAAM,kBAAkB,GAAG,0BAA0B,CAAC,SAAS,CAAC,CAAC;gBACjE,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,UAAU,CAAC,SAAS,CAAC,CAAC;gBAC7F,MAAM,QAAQ,GAAG,iBAAiB,EAAE,QAAQ,IAAI,GAAG,CAAC;gBAEpD,iDAAiD;gBACjD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAE9D,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;oBACd,YAAY,CAAC,IAAI,CAAC;wBAChB,OAAO,EAAE,SAAS;wBAClB,KAAK;wBACL,MAAM,EAAE,SAAS;qBAClB,CAAC,CAAC;oBACH,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;oBAC3B,aAAa,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,OAAO,CAAC,GAAG,IAAI,EAAE,GAAG,YAAY,CAAC,CAAC;AACpC,CAAC"}

package/dist/retrieval/context-assembler.d.ts

@@ -0,0 +1,89 @@
+/**
+ * Context assembly for memory retrieval.
+ * Combines vector search with graph traversal to assemble relevant context.
+ */
+/**
+ * Retrieval mode determines traversal direction and ranking.
+ */
+export type RetrievalMode = 'recall' | 'explain' | 'predict';
+/**
+ * Range hint for retrieval - lets Claude choose appropriate decay model.
+ * - 'short': Recent context (15min hold) - best for immediate follow-ups
+ * - 'long': Distant context (60min hold) - best for cross-session/historical
+ * - 'auto': System decides based on query characteristics
+ */
+export type RetrievalRange = 'short' | 'long' | 'auto';
+/**
+ * Request for context retrieval.
+ */
+export interface RetrievalRequest {
+    /** Query text to find relevant context for */
+    query: string;
+    /** Current session ID (optional, for recency boost) */
+    currentSessionId?: string;
+    /** Project slug for clock lookup (optional, enables vector clock decay) */
+    projectSlug?: string;
+    /** Filter results to specific project(s). Omit to search all projects. */
+    projectFilter?: string | string[];
+    /** Query time for decay calculation (default: now) */
+    queryTime?: number;
+    /** Maximum tokens in response */
+    maxTokens?: number;
+    /** Retrieval mode */
+    mode: RetrievalMode;
+    /** Range hint: 'short' for recent, 'long' for historical, 'auto' to decide */
+    range?: RetrievalRange;
+    /** Number of vector search results to start from */
+    vectorSearchLimit?: number;
+    /** Skip graph traversal (for benchmarking vector-only retrieval) */
+    skipGraph?: boolean;
+    /** Skip cluster expansion (for benchmarking vector-only retrieval) */
+    skipClusters?: boolean;
+}
+/**
+ * Retrieval response with assembled context.
+ */
+export interface RetrievalResponse {
+    /** Assembled context text */
+    text: string;
+    /** Approximate token count */
+    tokenCount: number;
+    /** Chunks included in response */
+    chunks: Array<{
+        id: string;
+        sessionSlug: string;
+        weight: number;
+        preview: string;
+        source?: 'vector' | 'keyword' | 'cluster' | 'graph';
+    }>;
+    /** Total chunks considered */
+    totalConsidered: number;
+    /** Time taken in milliseconds */
+    durationMs: number;
+    /** Number of chunks boosted by graph agreement (found by both vector and graph) */
+    graphBoosted: number;
+}
+/**
+ * Assemble context from memory based on a query.
+ *
+ * Pipeline: embed → [vector search, keyword search] → RRF fusion → cluster expansion
+ *           → graph traverse → combine → dedupe → recency → budget
+ */
+export declare function assembleContext(request: RetrievalRequest): Promise<RetrievalResponse>;
+/**
+ * Recall: retrieve context relevant to a query.
+ */
+export declare function recall(query: string, options?: Partial<RetrievalRequest>): Promise<RetrievalResponse>;
+/**
+ * Explain: retrieve context that explains how we got to current state.
+ */
+export declare function explain(topic: string, options?: Partial<RetrievalRequest>): Promise<RetrievalResponse>;
+/**
+ * Predict: retrieve context that might be relevant next.
+ */
+export declare function predict(context: string, options?: Partial<RetrievalRequest>): Promise<RetrievalResponse>;
+/**
+ * Cleanup shared resources.
+ */
+export declare function disposeRetrieval(): Promise<void>;
+//# sourceMappingURL=context-assembler.d.ts.map

package/dist/retrieval/context-assembler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-assembler.d.ts","sourceRoot":"","sources":["../../src/retrieval/context-assembler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAkBH;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;AAE7D;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;AAEvD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0EAA0E;IAC1E,aAAa,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAClC,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qBAAqB;IACrB,IAAI,EAAE,aAAa,CAAC;IACpB,8EAA8E;IAC9E,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,oDAAoD;IACpD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oEAAoE;IACpE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,sEAAsE;IACtE,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,MAAM,EAAE,KAAK,CAAC;QACZ,EAAE,EAAE,MAAM,CAAC;QACX,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;KACrD,CAAC,CAAC;IACH,8BAA8B;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,mFAAmF;IACnF,YAAY,EAAE,MAAM,CAAC;CACtB;AAiCD;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAgM3F;AAyGD;;GAEG;AACH,wBAAsB,MAAM,CAC1B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,OAAO,CAAC,gBAAgB,CAAM,GACtC,OAAO,CAAC,iBAAiB,CAAC,CAM5B;AAED;;GAEG;AACH,wBAAsB,OAAO,CAC3B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,OAAO,CAAC,gBAAgB,CAAM,GACtC,OAAO,CAAC,iBAAiB,CAAC,CAM5B;AAED;;GAEG;AACH,wBAAsB,OAAO,CAC3B,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,OAAO,CAAC,gBAAgB,CAAM,GACtC,OAAO,CAAC,iBAAiB,CAAC,CAM5B;AAED;;GAEG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAKtD"}

package/dist/retrieval/context-assembler.js

@@ -0,0 +1,313 @@
+/**
+ * Context assembly for memory retrieval.
+ * Combines vector search with graph traversal to assemble relevant context.
+ */
+import { vectorStore } from '../storage/vector-store.js';
+import { getChunkById } from '../storage/chunk-store.js';
+import { Embedder } from '../models/embedder.js';
+import { getModel } from '../models/model-registry.js';
+import { getConfig } from '../config/memory-config.js';
+import { approximateTokens } from '../utils/token-counter.js';
+import { traverseMultiple, dedupeAndRank } from './traverser.js';
+import { getReferenceClock } from '../storage/clock-store.js';
+import { KeywordStore } from '../storage/keyword-store.js';
+import { fuseRRF } from './rrf.js';
+import { expandViaClusters } from './cluster-expander.js';
+import { createLogger } from '../utils/logger.js';
+const log = createLogger('context-assembler');
+/**
+ * Shared embedder instance for retrieval.
+ */
+let sharedEmbedder = null;
+/**
+ * Get or create shared embedder.
+ */
+async function getEmbedder() {
+    if (!sharedEmbedder) {
+        sharedEmbedder = new Embedder();
+        await sharedEmbedder.load(getModel('jina-small'));
+    }
+    return sharedEmbedder;
+}
+/**
+ * Shared keyword store instance for retrieval.
+ */
+let sharedKeywordStore = null;
+/**
+ * Get or create shared keyword store.
+ */
+function getKeywordStore() {
+    if (!sharedKeywordStore) {
+        sharedKeywordStore = new KeywordStore();
+    }
+    return sharedKeywordStore;
+}
+/**
+ * Assemble context from memory based on a query.
+ *
+ * Pipeline: embed → [vector search, keyword search] → RRF fusion → cluster expansion
+ *           → graph traverse → combine → dedupe → recency → budget
+ */
+export async function assembleContext(request) {
+    const startTime = Date.now();
+    const config = getConfig();
+    const { query, currentSessionId, projectSlug, projectFilter, queryTime = Date.now(), maxTokens = config.mcpMaxResponseTokens, mode, range = 'auto', vectorSearchLimit = 20, skipGraph = false, skipClusters = false, } = request;
+    const { hybridSearch, clusterExpansion } = config;
+    // If projectFilter is a single string, also use it for clock lookup
+    const effectiveProjectSlug = projectSlug ??
+        (typeof projectFilter === 'string' ? projectFilter : undefined);
+    // Get reference clock for vector clock-based decay (if project slug provided)
+    const referenceClock = effectiveProjectSlug ? getReferenceClock(effectiveProjectSlug) : undefined;
+    // 1. Embed query
+    const embedder = await getEmbedder();
+    const queryResult = await embedder.embed(query, true); // isQuery = true
+    // 2. Run vector search and keyword search in parallel
+    const vectorSearchPromise = projectFilter
+        ? vectorStore.searchByProject(queryResult.embedding, projectFilter, vectorSearchLimit)
+        : vectorStore.search(queryResult.embedding, vectorSearchLimit);
+    let keywordResults = [];
+    try {
+        const keywordStore = getKeywordStore();
+        keywordResults = projectFilter
+            ? keywordStore.searchByProject(query, projectFilter, hybridSearch.keywordSearchLimit)
+            : keywordStore.search(query, hybridSearch.keywordSearchLimit);
+    }
+    catch (error) {
+        // Graceful fallback: keyword search unavailable (FTS5 missing, table corrupted, etc.)
+        log.warn('Keyword search unavailable, falling back to vector-only', {
+            error: error.message,
+        });
+    }
+    const similar = await vectorSearchPromise;
+    if (similar.length === 0 && keywordResults.length === 0) {
+        return {
+            text: '',
+            tokenCount: 0,
+            chunks: [],
+            totalConsidered: 0,
+            durationMs: Date.now() - startTime,
+            graphBoosted: 0,
+        };
+    }
+    // 3. Convert to RankedItem format and fuse with RRF
+    const vectorItems = similar.map(s => ({
+        chunkId: s.id,
+        score: Math.max(0, 1 - s.distance),
+        source: 'vector',
+    }));
+    const keywordItems = keywordResults.map(r => ({
+        chunkId: r.id,
+        score: r.score,
+        source: 'keyword',
+    }));
+    const fusedResults = fuseRRF([
+        { items: vectorItems, weight: hybridSearch.vectorWeight },
+        ...(keywordItems.length > 0
+            ? [{ items: keywordItems, weight: hybridSearch.keywordWeight }]
+            : []),
+    ], hybridSearch.rrfK);
+    // 4. Cluster expansion (skip if requested for benchmarking)
+    const expandedResults = skipClusters
+        ? fusedResults
+        : expandViaClusters(fusedResults, clusterExpansion, projectFilter);
+    const sourceMap = new Map();
+    for (const item of expandedResults) {
+        if (item.source && !sourceMap.has(item.chunkId)) {
+            sourceMap.set(item.chunkId, item.source);
+        }
+    }
+    // 5. Determine traversal direction and decay model based on mode and range
+    const direction = mode === 'predict' ? 'forward' : 'backward';
+    let decayConfig;
+    if (direction === 'forward') {
+        decayConfig = config.forwardDecay;
+    }
+    else if (range === 'short') {
+        decayConfig = config.shortRangeDecay;
+    }
+    else if (range === 'long') {
+        decayConfig = config.longRangeDecay;
+    }
+    else {
+        decayConfig = mode === 'explain' ? config.longRangeDecay : config.shortRangeDecay;
+    }
+    // 6. Traverse graph from fused+expanded seeds (skip if requested for benchmarking)
+    let traversalResult;
+    if (skipGraph) {
+        traversalResult = { chunks: [], visited: 0 };
+    }
+    else {
+        const startIds = expandedResults.map(r => r.chunkId);
+        const startWeights = expandedResults.map(r => r.score);
+        traversalResult = await traverseMultiple(startIds, startWeights, queryTime, {
+            direction,
+            decayConfig,
+            referenceClock,
+        });
+        // Tag traversal results with 'graph' source
+        for (const tc of traversalResult.chunks) {
+            if (!sourceMap.has(tc.chunkId)) {
+                sourceMap.set(tc.chunkId, 'graph');
+            }
+        }
+    }
+    // 7. Combine direct hits with traversal, boosting intersection
+    const allChunks = [];
+    let graphBoostedCount = 0;
+    // Build graph traversal lookup
+    const graphChunkMap = new Map();
+    for (const tc of traversalResult.chunks) {
+        graphChunkMap.set(tc.chunkId, tc);
+    }
+    // Direct hits: boost if also found via graph traversal
+    const directChunkIds = new Set();
+    for (const item of expandedResults) {
+        directChunkIds.add(item.chunkId);
+        let weight = item.score * config.directHitBoost;
+        const graphEntry = graphChunkMap.get(item.chunkId);
+        if (graphEntry) {
+            weight += graphEntry.weight * config.graphAgreementBoost;
+            graphBoostedCount++;
+        }
+        allChunks.push({ chunkId: item.chunkId, weight, depth: 0 });
+    }
+    // Graph-only results: add unchanged
+    for (const tc of traversalResult.chunks) {
+        if (!directChunkIds.has(tc.chunkId)) {
+            allChunks.push(tc);
+        }
+    }
+    // 8. Dedupe and rank
+    const ranked = dedupeAndRank(allChunks);
+    // 9. Apply recency boost for current session
+    if (currentSessionId) {
+        for (const wc of ranked) {
+            const chunk = getChunkById(wc.chunkId);
+            if (chunk && chunk.sessionId === currentSessionId) {
+                wc.weight *= 1.2; // 20% boost for current session
+            }
+        }
+        ranked.sort((a, b) => b.weight - a.weight);
+    }
+    // 10. Assemble within token budget
+    const assembled = assembleWithinBudget(ranked, maxTokens, sourceMap);
+    return {
+        text: assembled.text,
+        tokenCount: assembled.tokenCount,
+        chunks: assembled.includedChunks,
+        totalConsidered: ranked.length,
+        durationMs: Date.now() - startTime,
+        graphBoosted: graphBoostedCount,
+    };
+}
+/**
+ * Assemble text within token budget.
+ */
+function assembleWithinBudget(ranked, maxTokens, sourceMap) {
+    const parts = [];
+    const includedChunks = [];
+    let totalTokens = 0;
+    for (const wc of ranked) {
+        const chunk = getChunkById(wc.chunkId);
+        if (!chunk)
+            continue;
+        const chunkTokens = chunk.approxTokens || approximateTokens(chunk.content);
+        // Check if adding this chunk would exceed budget
+        if (totalTokens + chunkTokens > maxTokens) {
+            // Try to fit a truncated version
+            const remainingTokens = maxTokens - totalTokens;
+            if (remainingTokens > 100) {
+                const truncated = truncateChunk(chunk.content, remainingTokens);
+                parts.push(formatChunkForOutput(chunk, truncated, wc.weight));
+                totalTokens += approximateTokens(truncated);
+                includedChunks.push({
+                    id: chunk.id,
+                    sessionSlug: chunk.sessionSlug,
+                    weight: wc.weight,
+                    preview: truncated.slice(0, 100) + '...',
+                    source: sourceMap?.get(chunk.id),
+                });
+            }
+            break;
+        }
+        parts.push(formatChunkForOutput(chunk, chunk.content, wc.weight));
+        totalTokens += chunkTokens;
+        includedChunks.push({
+            id: chunk.id,
+            sessionSlug: chunk.sessionSlug,
+            weight: wc.weight,
+            preview: chunk.content.slice(0, 100) + (chunk.content.length > 100 ? '...' : ''),
+            source: sourceMap?.get(chunk.id),
+        });
+    }
+    return {
+        text: parts.join('\n\n---\n\n'),
+        tokenCount: totalTokens,
+        includedChunks,
+    };
+}
+/**
+ * Format a chunk for output with metadata header.
+ */
+function formatChunkForOutput(chunk, content, weight) {
+    const date = new Date(chunk.startTime).toLocaleDateString();
+    const relevance = (weight * 100).toFixed(0);
+    return `[Session: ${chunk.sessionSlug} | Date: ${date} | Relevance: ${relevance}%]\n${content}`;
+}
+/**
+ * Truncate chunk content to fit token budget.
+ */
+function truncateChunk(content, maxTokens) {
+    // Rough estimate: 4 chars per token
+    const maxChars = maxTokens * 4;
+    if (content.length <= maxChars) {
+        return content;
+    }
+    // Try to cut at a paragraph boundary
+    const truncated = content.slice(0, maxChars);
+    const lastNewline = truncated.lastIndexOf('\n\n');
+    if (lastNewline > maxChars * 0.5) {
+        return truncated.slice(0, lastNewline) + '\n\n...[truncated]';
+    }
+    return truncated + '\n...[truncated]';
+}
+/**
+ * Recall: retrieve context relevant to a query.
+ */
+export async function recall(query, options = {}) {
+    return assembleContext({
+        query,
+        mode: 'recall',
+        ...options,
+    });
+}
+/**
+ * Explain: retrieve context that explains how we got to current state.
+ */
+export async function explain(topic, options = {}) {
+    return assembleContext({
+        query: topic,
+        mode: 'explain',
+        ...options,
+    });
+}
+/**
+ * Predict: retrieve context that might be relevant next.
+ */
+export async function predict(context, options = {}) {
+    return assembleContext({
+        query: context,
+        mode: 'predict',
+        ...options,
+    });
+}
+/**
+ * Cleanup shared resources.
+ */
+export async function disposeRetrieval() {
+    if (sharedEmbedder) {
+        await sharedEmbedder.dispose();
+        sharedEmbedder = null;
+    }
+}
+//# sourceMappingURL=context-assembler.js.map