causantic 0.2.0

package/dist/eval/experiments/hdbscan-sweep.js

@@ -0,0 +1,71 @@
+/**
+ * Experiment 2: HDBSCAN minClusterSize Sweep
+ *
+ * Question: Is minClusterSize=3 optimal, or are we leaving structure on the table?
+ *
+ * Reuses jina-small embeddings and sweeps minClusterSize from 2 to 10,
+ * recording cluster count, noise ratio, and silhouette score for each.
+ */
+import { clusterEmbeddings } from '../cluster-evaluator.js';
+import { silhouetteScore } from '../metrics.js';
+import { singleModelRun } from './single-model-run.js';
+const MODEL_ID = 'jina-small';
+const MIN_CLUSTER_SIZES = [2, 3, 4, 5, 6, 7, 8, 9, 10];
+/**
+ * Run the HDBSCAN parameter sweep.
+ *
+ * If baselineResult is provided (with embeddings), reuses them to avoid re-embedding.
+ */
+export async function runHdbscanSweep(chunks, pairs, baselineResult) {
+    console.log('\n=== Experiment 2: HDBSCAN minClusterSize Sweep ===');
+    // Get embeddings (reuse or compute)
+    let embeddingsMap;
+    if (baselineResult) {
+        console.log('  Reusing cached embeddings');
+        embeddingsMap = baselineResult.embeddings;
+    }
+    else {
+        console.log('  Computing embeddings...');
+        const result = await singleModelRun(MODEL_ID, chunks, pairs);
+        embeddingsMap = result.embeddings;
+    }
+    // Build ordered embedding array
+    const embeddingArray = chunks.map((c) => embeddingsMap.get(c.id)).filter(Boolean);
+    // Sweep
+    const rows = [];
+    console.log('');
+    console.log('  ' +
+        pad('minSize', 8) +
+        pad('Clusters', 10) +
+        pad('Noise%', 10) +
+        pad('Silhouette', 12));
+    console.log('  ' + '-'.repeat(40));
+    for (const minClusterSize of MIN_CLUSTER_SIZES) {
+        const clusterResult = clusterEmbeddings(embeddingArray, { minClusterSize });
+        const silhouette = silhouetteScore(embeddingArray, clusterResult.labels);
+        const row = {
+            minClusterSize,
+            clusterCount: clusterResult.numClusters,
+            noiseRatio: clusterResult.noiseRatio,
+            silhouetteScore: silhouette,
+        };
+        rows.push(row);
+        console.log('  ' +
+            pad(String(minClusterSize), 8) +
+            pad(String(clusterResult.numClusters), 10) +
+            pad((clusterResult.noiseRatio * 100).toFixed(1) + '%', 10) +
+            pad(silhouette.toFixed(3), 12));
+    }
+    // Find optimal
+    const best = rows.reduce((a, b) => b.silhouetteScore > a.silhouetteScore ? b : a);
+    console.log(`\n  Best silhouette: minClusterSize=${best.minClusterSize} (silhouette=${best.silhouetteScore.toFixed(3)})`);
+    return {
+        name: 'hdbscan-sweep',
+        description: `HDBSCAN minClusterSize sweep from ${MIN_CLUSTER_SIZES[0]} to ${MIN_CLUSTER_SIZES[MIN_CLUSTER_SIZES.length - 1]}. Best silhouette at minClusterSize=${best.minClusterSize}.`,
+        rows,
+    };
+}
+function pad(str, width) {
+    return str.padEnd(width);
+}
+//# sourceMappingURL=hdbscan-sweep.js.map

package/dist/eval/experiments/hdbscan-sweep.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hdbscan-sweep.js","sourceRoot":"","sources":["../../../src/eval/experiments/hdbscan-sweep.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGhD,OAAO,EAAE,cAAc,EAA0B,MAAM,uBAAuB,CAAC;AAG/E,MAAM,QAAQ,GAAG,YAAY,CAAC;AAC9B,MAAM,iBAAiB,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAEvD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAe,EACf,KAAoB,EACpB,cAAkC;IAElC,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;IAEpE,oCAAoC;IACpC,IAAI,aAAoC,CAAC;IACzC,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,aAAa,GAAG,cAAc,CAAC,UAAU,CAAC;IAC5C,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7D,aAAa,GAAG,MAAM,CAAC,UAAU,CAAC;IACpC,CAAC;IAED,gCAAgC;IAChC,MAAM,cAAc,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnF,QAAQ;IACR,MAAM,IAAI,GAAe,EAAE,CAAC;IAE5B,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CACT,IAAI;QACJ,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACjB,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC;QACnB,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC;QACjB,GAAG,CAAC,YAAY,EAAE,EAAE,CAAC,CACtB,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAEnC,KAAK,MAAM,cAAc,IAAI,iBAAiB,EAAE,CAAC;QAC/C,MAAM,aAAa,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,eAAe,CAAC,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;QAEzE,MAAM,GAAG,GAAa;YACpB,cAAc;YACd,YAAY,EAAE,aAAa,CAAC,WAAW;YACvC,UAAU,EAAE,aAAa,CAAC,UAAU;YACpC,eAAe,EAAE,UAAU;SAC5B,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEf,OAAO,CAAC,GAAG,CACT,IAAI;YACJ,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YAC9B,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;YAC1C,GAAG,CAAC,CAAC,aAAa,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,EAAE,CAAC;YAC1D,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAC/B,CAAC;IACJ,CAAC;IAED,eAAe;IACf,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAChC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC9C,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,uCAAuC,IAAI,CAAC,cAAc,gBAAgB,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAE1H,OAAO;QACL,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,qCAAqC,iBAAiB,CAAC,CAAC,CAAC,OAAO,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,uCAAuC,IAAI,CAAC,cAAc,GAAG;QACzL,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,GAAG,CAAC,GAAW,EAAE,KAAa;IACrC,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC"}

package/dist/eval/experiments/single-model-run.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Reusable single-model embedding + scoring + clustering pipeline.
+ *
+ * Extracts the per-model logic from benchmark-runner.ts so each
+ * experiment can run it without duplicating the pipeline.
+ */
+import { type ClusterResult } from '../cluster-evaluator.js';
+import { type AlignmentResult } from '../code-nl-alignment.js';
+import type { LabeledPair } from '../annotation-schema.js';
+import type { Chunk } from '../../parser/types.js';
+import type { MetricSnapshot } from './types.js';
+export interface SingleModelResult {
+    rocAuc: number;
+    silhouetteScore: number;
+    clusterCount: number;
+    noiseRatio: number;
+    clusterResult: ClusterResult;
+    clusterMembership: Map<number, number[]>;
+    embeddings: Map<string, number[]>;
+    codeNLAlignment: AlignmentResult;
+    meanInferenceMs: number;
+}
+export interface SingleModelRunOptions {
+    minClusterSize?: number;
+}
+/**
+ * Embed chunks with a model, score pairs, and cluster.
+ *
+ * Loads the model, embeds all chunks, scores labeled pairs,
+ * runs HDBSCAN clustering, computes silhouette, and returns
+ * a complete result.
+ */
+export declare function singleModelRun(modelId: string, chunks: Chunk[], pairs: LabeledPair[], options?: SingleModelRunOptions): Promise<SingleModelResult>;
+/**
+ * Embed chunk texts directly (for experiments that modify text before embedding).
+ *
+ * Takes an array of {id, text} instead of Chunk objects, allowing experiments
+ * to pass modified text (e.g. truncated, filtered) while keeping original chunk IDs.
+ */
+export declare function embedTextsAndScore(modelId: string, texts: {
+    id: string;
+    text: string;
+}[], pairs: LabeledPair[], options?: SingleModelRunOptions): Promise<SingleModelResult>;
+/**
+ * Score and cluster from pre-computed embeddings (no model loading needed).
+ */
+export declare function scoreFromEmbeddings(chunks: Chunk[], pairs: LabeledPair[], embeddings: Map<string, number[]>, options?: SingleModelRunOptions): Omit<SingleModelResult, 'meanInferenceMs'> & {
+    meanInferenceMs: number;
+};
+/**
+ * Extract a MetricSnapshot from a SingleModelResult.
+ */
+export declare function toSnapshot(result: SingleModelResult, chunkCount: number): MetricSnapshot;
+//# sourceMappingURL=single-model-run.d.ts.map

package/dist/eval/experiments/single-model-run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"single-model-run.d.ts","sourceRoot":"","sources":["../../../src/eval/experiments/single-model-run.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAA2C,KAAK,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACtG,OAAO,EAA2B,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAMxF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,aAAa,CAAC;IAC7B,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACzC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAClC,eAAe,EAAE,eAAe,CAAC;IACjC,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,qBAAqB;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,KAAK,EAAE,EACf,KAAK,EAAE,WAAW,EAAE,EACpB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,iBAAiB,CAAC,CA4B5B;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,EACrC,KAAK,EAAE,WAAW,EAAE,EACpB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,iBAAiB,CAAC,CA6B5B;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,KAAK,EAAE,EACf,KAAK,EAAE,WAAW,EAAE,EACpB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EACjC,OAAO,GAAE,qBAA0B,GAClC,IAAI,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,GAAG;IAAE,eAAe,EAAE,MAAM,CAAA;CAAE,CAE1E;AAwCD;;GAEG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,GAAG,cAAc,CAQxF"}

package/dist/eval/experiments/single-model-run.js

@@ -0,0 +1,120 @@
+/**
+ * Reusable single-model embedding + scoring + clustering pipeline.
+ *
+ * Extracts the per-model logic from benchmark-runner.ts so each
+ * experiment can run it without duplicating the pipeline.
+ */
+import { Embedder } from '../../models/embedder.js';
+import { getModel } from '../../models/model-registry.js';
+import { clusterEmbeddings, getClusterMembership } from '../cluster-evaluator.js';
+import { evaluateCodeNLAlignment } from '../code-nl-alignment.js';
+import { scorePairs, rocAuc, silhouetteScore, } from '../metrics.js';
+/**
+ * Embed chunks with a model, score pairs, and cluster.
+ *
+ * Loads the model, embeds all chunks, scores labeled pairs,
+ * runs HDBSCAN clustering, computes silhouette, and returns
+ * a complete result.
+ */
+export async function singleModelRun(modelId, chunks, pairs, options = {}) {
+    const { minClusterSize = 3 } = options;
+    const config = getModel(modelId);
+    const embedder = new Embedder();
+    try {
+        console.log(`  Loading ${modelId}...`);
+        await embedder.load(config);
+        // Embed all chunks
+        console.log(`  Embedding ${chunks.length} chunks...`);
+        const embeddings = new Map();
+        let totalInferenceMs = 0;
+        for (let i = 0; i < chunks.length; i++) {
+            const result = await embedder.embed(chunks[i].text);
+            embeddings.set(chunks[i].id, result.embedding);
+            totalInferenceMs += result.inferenceMs;
+            if ((i + 1) % 50 === 0 || i === chunks.length - 1) {
+                console.log(`    ${i + 1}/${chunks.length} chunks embedded`);
+            }
+        }
+        return buildResult(chunks, pairs, embeddings, totalInferenceMs, minClusterSize);
+    }
+    finally {
+        await embedder.dispose();
+    }
+}
+/**
+ * Embed chunk texts directly (for experiments that modify text before embedding).
+ *
+ * Takes an array of {id, text} instead of Chunk objects, allowing experiments
+ * to pass modified text (e.g. truncated, filtered) while keeping original chunk IDs.
+ */
+export async function embedTextsAndScore(modelId, texts, pairs, options = {}) {
+    const { minClusterSize = 3 } = options;
+    const config = getModel(modelId);
+    const embedder = new Embedder();
+    try {
+        console.log(`  Loading ${modelId}...`);
+        await embedder.load(config);
+        console.log(`  Embedding ${texts.length} texts...`);
+        const embeddings = new Map();
+        let totalInferenceMs = 0;
+        for (let i = 0; i < texts.length; i++) {
+            const result = await embedder.embed(texts[i].text);
+            embeddings.set(texts[i].id, result.embedding);
+            totalInferenceMs += result.inferenceMs;
+            if ((i + 1) % 50 === 0 || i === texts.length - 1) {
+                console.log(`    ${i + 1}/${texts.length} texts embedded`);
+            }
+        }
+        // Build fake chunks array for clustering (just needs id ordering)
+        const fakeChunks = texts.map((t) => ({ id: t.id }));
+        return buildResult(fakeChunks, pairs, embeddings, totalInferenceMs, minClusterSize);
+    }
+    finally {
+        await embedder.dispose();
+    }
+}
+/**
+ * Score and cluster from pre-computed embeddings (no model loading needed).
+ */
+export function scoreFromEmbeddings(chunks, pairs, embeddings, options = {}) {
+    return buildResult(chunks, pairs, embeddings, 0, options.minClusterSize ?? 3);
+}
+function buildResult(chunks, pairs, embeddings, totalInferenceMs, minClusterSize) {
+    // Score labeled pairs
+    const scored = scorePairs(pairs, embeddings);
+    const auc = rocAuc(scored);
+    // Cluster
+    const orderedIds = chunks.map((c) => c.id);
+    const embeddingArray = orderedIds.map((id) => embeddings.get(id)).filter(Boolean);
+    const clusterResult = clusterEmbeddings(embeddingArray, { minClusterSize });
+    const membership = getClusterMembership(clusterResult.labels);
+    // Silhouette
+    const silhouette = silhouetteScore(embeddingArray, clusterResult.labels);
+    // Code-NL alignment
+    const alignment = evaluateCodeNLAlignment(pairs, embeddings);
+    const count = chunks.length;
+    return {
+        rocAuc: auc,
+        silhouetteScore: silhouette,
+        clusterCount: clusterResult.numClusters,
+        noiseRatio: clusterResult.noiseRatio,
+        clusterResult,
+        clusterMembership: membership,
+        embeddings,
+        codeNLAlignment: alignment,
+        meanInferenceMs: count > 0 ? totalInferenceMs / count : 0,
+    };
+}
+/**
+ * Extract a MetricSnapshot from a SingleModelResult.
+ */
+export function toSnapshot(result, chunkCount) {
+    return {
+        rocAuc: result.rocAuc,
+        silhouetteScore: result.silhouetteScore,
+        clusterCount: result.clusterCount,
+        noiseRatio: result.noiseRatio,
+        chunkCount,
+    };
+}
+//# sourceMappingURL=single-model-run.js.map

package/dist/eval/experiments/single-model-run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"single-model-run.js","sourceRoot":"","sources":["../../../src/eval/experiments/single-model-run.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAsB,MAAM,yBAAyB,CAAC;AACtG,OAAO,EAAE,uBAAuB,EAAwB,MAAM,yBAAyB,CAAC;AACxF,OAAO,EACL,UAAU,EACV,MAAM,EACN,eAAe,GAChB,MAAM,eAAe,CAAC;AAqBvB;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,MAAe,EACf,KAAoB,EACpB,UAAiC,EAAE;IAEnC,MAAM,EAAE,cAAc,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,KAAK,CAAC,CAAC;QACvC,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE5B,mBAAmB;QACnB,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,MAAM,YAAY,CAAC,CAAC;QACtD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC/C,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpD,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAC/C,gBAAgB,IAAI,MAAM,CAAC,WAAW,CAAC;YAEvC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAC;IAClF,CAAC;YAAS,CAAC;QACT,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,OAAe,EACf,KAAqC,EACrC,KAAoB,EACpB,UAAiC,EAAE;IAEnC,MAAM,EAAE,cAAc,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,KAAK,CAAC,CAAC;QACvC,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE5B,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,CAAC,MAAM,WAAW,CAAC,CAAC;QACpD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC/C,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACnD,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAC9C,gBAAgB,IAAI,MAAM,CAAC,WAAW,CAAC;YAEvC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,iBAAiB,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,kEAAkE;QAClE,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAY,CAAC;QAC/D,OAAO,WAAW,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAC;IACtF,CAAC;YAAS,CAAC;QACT,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAe,EACf,KAAoB,EACpB,UAAiC,EACjC,UAAiC,EAAE;IAEnC,OAAO,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,WAAW,CAClB,MAAkC,EAClC,KAAoB,EACpB,UAAiC,EACjC,gBAAwB,EACxB,cAAsB;IAEtB,sBAAsB;IACtB,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAE3B,UAAU;IACV,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnF,MAAM,aAAa,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAG,oBAAoB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAE9D,aAAa;IACb,MAAM,UAAU,GAAG,eAAe,CAAC,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IAEzE,oBAAoB;IACpB,MAAM,SAAS,GAAG,uBAAuB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAE7D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAE5B,OAAO;QACL,MAAM,EAAE,GAAG;QACX,eAAe,EAAE,UAAU;QAC3B,YAAY,EAAE,aAAa,CAAC,WAAW;QACvC,UAAU,EAAE,aAAa,CAAC,UAAU;QACpC,aAAa;QACb,iBAAiB,EAAE,UAAU;QAC7B,UAAU;QACV,eAAe,EAAE,SAAS;QAC1B,eAAe,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;KAC1D,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,MAAyB,EAAE,UAAkB;IACtE,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,UAAU;KACX,CAAC;AACJ,CAAC"}

package/dist/eval/experiments/thinking-ablation.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Experiment 4: Thinking Block Ablation
+ *
+ * Question: Do thinking blocks help or hurt embedding quality?
+ *
+ * Re-chunks the corpus with includeThinking:false (default is true),
+ * re-embeds with jina-small, and compares ROC AUC and silhouette
+ * against the default (thinking included).
+ */
+import type { LabeledPair } from '../annotation-schema.js';
+import type { Corpus } from '../corpus-builder.js';
+import { type SingleModelResult } from './single-model-run.js';
+import { type ExperimentResult } from './types.js';
+/**
+ * Run the thinking ablation experiment.
+ *
+ * Requires the original corpus config to rebuild with different settings.
+ * The original corpus should have been built with includeThinking:true.
+ */
+export declare function runThinkingAblation(originalCorpus: Corpus, originalPairs: LabeledPair[], baselineResult?: SingleModelResult): Promise<ExperimentResult>;
+//# sourceMappingURL=thinking-ablation.d.ts.map

package/dist/eval/experiments/thinking-ablation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"thinking-ablation.d.ts","sourceRoot":"","sources":["../../../src/eval/experiments/thinking-ablation.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAGnD,OAAO,EAGL,KAAK,iBAAiB,EACvB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAgB,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAIjE;;;;;GAKG;AACH,wBAAsB,mBAAmB,CACvC,cAAc,EAAE,MAAM,EACtB,aAAa,EAAE,WAAW,EAAE,EAC5B,cAAc,CAAC,EAAE,iBAAiB,GACjC,OAAO,CAAC,gBAAgB,CAAC,CAwD3B"}

package/dist/eval/experiments/thinking-ablation.js

@@ -0,0 +1,68 @@
+/**
+ * Experiment 4: Thinking Block Ablation
+ *
+ * Question: Do thinking blocks help or hurt embedding quality?
+ *
+ * Re-chunks the corpus with includeThinking:false (default is true),
+ * re-embeds with jina-small, and compares ROC AUC and silhouette
+ * against the default (thinking included).
+ */
+import { buildCorpus } from '../corpus-builder.js';
+import { generateLabeledPairs } from '../annotation-schema.js';
+import { singleModelRun, toSnapshot, } from './single-model-run.js';
+import { computeDelta } from './types.js';
+const MODEL_ID = 'jina-small';
+/**
+ * Run the thinking ablation experiment.
+ *
+ * Requires the original corpus config to rebuild with different settings.
+ * The original corpus should have been built with includeThinking:true.
+ */
+export async function runThinkingAblation(originalCorpus, originalPairs, baselineResult) {
+    console.log('\n=== Experiment 4: Thinking Block Ablation ===');
+    // Baseline: thinking included (original corpus)
+    let baseline;
+    if (baselineResult) {
+        console.log('  Reusing cached baseline embeddings');
+        baseline = baselineResult;
+    }
+    else {
+        console.log('  Running baseline (thinking included)...');
+        baseline = await singleModelRun(MODEL_ID, originalCorpus.chunks, originalPairs);
+    }
+    // Variant: rebuild corpus without thinking blocks
+    console.log('  Rebuilding corpus with includeThinking=false...');
+    const variantCorpus = await buildCorpus({
+        ...originalCorpus.config,
+        includeThinking: false,
+    });
+    console.log(`  Variant corpus: ${variantCorpus.chunks.length} chunks ` +
+        `(original: ${originalCorpus.chunks.length})`);
+    // Generate new labeled pairs for the variant corpus
+    // (chunk IDs will differ, so we need fresh pairs)
+    const variantAnnotations = generateLabeledPairs(variantCorpus.chunks, {
+        adjacentPairs: 60,
+        crossSessionPairs: 40,
+        crossProjectPairs: 80,
+        codeNLPairs: 20,
+    });
+    console.log(`  Variant pairs: ${variantAnnotations.pairs.length}`);
+    // Embed and score variant
+    console.log('  Running variant (no thinking)...');
+    const variant = await singleModelRun(MODEL_ID, variantCorpus.chunks, variantAnnotations.pairs);
+    const baselineSnap = toSnapshot(baseline, originalCorpus.chunks.length);
+    const variantSnap = toSnapshot(variant, variantCorpus.chunks.length);
+    const delta = computeDelta(baselineSnap, variantSnap);
+    console.log(`  Baseline ROC AUC: ${baselineSnap.rocAuc.toFixed(3)} (${baselineSnap.chunkCount} chunks)`);
+    console.log(`  No-thinking ROC AUC: ${variantSnap.rocAuc.toFixed(3)} (${variantSnap.chunkCount} chunks) (delta: ${delta.rocAuc >= 0 ? '+' : ''}${delta.rocAuc.toFixed(3)})`);
+    console.log(`  Baseline Silhouette: ${baselineSnap.silhouetteScore.toFixed(3)}`);
+    console.log(`  No-thinking Silhouette: ${variantSnap.silhouetteScore.toFixed(3)} (delta: ${delta.silhouetteScore >= 0 ? '+' : ''}${delta.silhouetteScore.toFixed(3)})`);
+    return {
+        name: 'thinking-ablation',
+        description: `Rebuilt corpus with includeThinking=false. Baseline: ${originalCorpus.chunks.length} chunks, variant: ${variantCorpus.chunks.length} chunks.`,
+        baseline: baselineSnap,
+        variant: variantSnap,
+        delta,
+    };
+}
+//# sourceMappingURL=thinking-ablation.js.map

package/dist/eval/experiments/thinking-ablation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"thinking-ablation.js","sourceRoot":"","sources":["../../../src/eval/experiments/thinking-ablation.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EACL,cAAc,EACd,UAAU,GAEX,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAyB,MAAM,YAAY,CAAC;AAEjE,MAAM,QAAQ,GAAG,YAAY,CAAC;AAE9B;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,cAAsB,EACtB,aAA4B,EAC5B,cAAkC;IAElC,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IAE/D,gDAAgD;IAChD,IAAI,QAA2B,CAAC;IAChC,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;QACpD,QAAQ,GAAG,cAAc,CAAC;IAC5B,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;QACzD,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAClF,CAAC;IAED,kDAAkD;IAClD,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;IACjE,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC;QACtC,GAAG,cAAc,CAAC,MAAM;QACxB,eAAe,EAAE,KAAK;KACvB,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CACT,qBAAqB,aAAa,CAAC,MAAM,CAAC,MAAM,UAAU;QAC1D,cAAc,cAAc,CAAC,MAAM,CAAC,MAAM,GAAG,CAC9C,CAAC;IAEF,oDAAoD;IACpD,kDAAkD;IAClD,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,aAAa,CAAC,MAAM,EAAE;QACpE,aAAa,EAAE,EAAE;QACjB,iBAAiB,EAAE,EAAE;QACrB,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE,EAAE;KAChB,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,oBAAoB,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAEnE,0BAA0B;IAC1B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,aAAa,CAAC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAE/F,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,EAAE,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxE,MAAM,WAAW,GAAG,UAAU,CAAC,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,KAAK,GAAG,YAAY,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAEtD,OAAO,CAAC,GAAG,CAAC,uBAAuB,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,UAAU,UAAU,CAAC,CAAC;IACzG,OAAO,CAAC,GAAG,CAAC,0BAA0B,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,UAAU,oBAAoB,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7K,OAAO,CAAC,GAAG,CAAC,0BAA0B,YAAY,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjF,OAAO,CAAC,GAAG,CAAC,6BAA6B,WAAW,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAExK,OAAO;QACL,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,wDAAwD,cAAc,CAAC,MAAM,CAAC,MAAM,qBAAqB,aAAa,CAAC,MAAM,CAAC,MAAM,UAAU;QAC3J,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,WAAW;QACpB,KAAK;KACN,CAAC;AACJ,CAAC"}

package/dist/eval/experiments/topic-continuity/embedding-classifier.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Embedding-based classifier for topic continuity.
+ *
+ * Uses angular distance between user text embedding and previous
+ * assistant output embedding to predict topic continuation.
+ */
+import type { TurnTransition, ClassificationResult } from './types.js';
+/**
+ * Result of embedding a single transition.
+ */
+export interface EmbeddedTransition {
+    transition: TurnTransition;
+    userEmbedding: number[];
+    assistantEmbedding: number[];
+    embeddingDistanceMin: number;
+    embeddingDistanceMean: number;
+}
+/**
+ * Embed all transitions using a specified model.
+ */
+export declare function embedTransitions(modelId: string, transitions: TurnTransition[]): Promise<EmbeddedTransition[]>;
+/**
+ * Compute continuation score from embedding distance.
+ * Lower distance = more similar = more likely continuation.
+ * Returns score in [0, 1] where 1 = continuation.
+ */
+export declare function embeddingDistanceToScore(distance: number): number;
+/**
+ * Classify transitions using only embedding distance.
+ */
+export declare function classifyWithEmbeddings(embeddedTransitions: EmbeddedTransition[]): ClassificationResult[];
+/**
+ * Get just the embedding distances for a model without full classification.
+ * Useful for feature ablation where we want to test embedding contribution.
+ */
+export declare function getEmbeddingDistances(modelId: string, transitions: TurnTransition[]): Promise<Map<string, {
+    min: number;
+    mean: number;
+}>>;
+/**
+ * Pre-computed embeddings cache for reuse across experiments.
+ */
+export interface EmbeddingCache {
+    modelId: string;
+    transitions: Map<string, EmbeddedTransition>;
+}
+/**
+ * Create an embedding cache from embedded transitions.
+ */
+export declare function createEmbeddingCache(modelId: string, embeddedTransitions: EmbeddedTransition[]): EmbeddingCache;
+/**
+ * Classify from cache without re-embedding.
+ */
+export declare function classifyFromCache(cache: EmbeddingCache): ClassificationResult[];
+//# sourceMappingURL=embedding-classifier.d.ts.map

package/dist/eval/experiments/topic-continuity/embedding-classifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-classifier.d.ts","sourceRoot":"","sources":["../../../../src/eval/experiments/topic-continuity/embedding-classifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,cAAc,EAAE,oBAAoB,EAAsB,MAAM,YAAY,CAAC;AAG3F;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,cAAc,CAAC;IAC3B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,cAAc,EAAE,GAC5B,OAAO,CAAC,kBAAkB,EAAE,CAAC,CA6C/B;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAIjE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CACpC,mBAAmB,EAAE,kBAAkB,EAAE,GACxC,oBAAoB,EAAE,CAqBxB;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,cAAc,EAAE,GAC5B,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAYrD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;CAC9C;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,EACf,mBAAmB,EAAE,kBAAkB,EAAE,GACxC,cAAc,CAMhB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,cAAc,GAAG,oBAAoB,EAAE,CAyB/E"}

package/dist/eval/experiments/topic-continuity/embedding-classifier.js

@@ -0,0 +1,126 @@
+/**
+ * Embedding-based classifier for topic continuity.
+ *
+ * Uses angular distance between user text embedding and previous
+ * assistant output embedding to predict topic continuation.
+ */
+import { Embedder } from '../../../models/embedder.js';
+import { getModel } from '../../../models/model-registry.js';
+import { angularDistance } from '../../../utils/angular-distance.js';
+import { extractLexicalFeatures } from './lexical-features.js';
+/**
+ * Embed all transitions using a specified model.
+ */
+export async function embedTransitions(modelId, transitions) {
+    const config = getModel(modelId);
+    const embedder = new Embedder();
+    try {
+        console.log(`  Loading ${modelId}...`);
+        await embedder.load(config);
+        const results = [];
+        for (let i = 0; i < transitions.length; i++) {
+            const t = transitions[i];
+            // Skip transitions without previous assistant text
+            if (!t.prevAssistantText.trim()) {
+                continue;
+            }
+            // Embed user text (as query since it's seeking information)
+            const userResult = await embedder.embed(t.nextUserText, true);
+            // Embed assistant text (as document since it's providing information)
+            const assistantResult = await embedder.embed(t.prevAssistantText, false);
+            // For now, we treat the entire assistant text as one chunk
+            // In a more sophisticated version, we could chunk it and compute min/mean
+            const distance = angularDistance(userResult.embedding, assistantResult.embedding);
+            results.push({
+                transition: t,
+                userEmbedding: userResult.embedding,
+                assistantEmbedding: assistantResult.embedding,
+                embeddingDistanceMin: distance,
+                embeddingDistanceMean: distance,
+            });
+            if ((i + 1) % 50 === 0 || i === transitions.length - 1) {
+                console.log(`    ${i + 1}/${transitions.length} transitions embedded`);
+            }
+        }
+        return results;
+    }
+    finally {
+        await embedder.dispose();
+    }
+}
+/**
+ * Compute continuation score from embedding distance.
+ * Lower distance = more similar = more likely continuation.
+ * Returns score in [0, 1] where 1 = continuation.
+ */
+export function embeddingDistanceToScore(distance) {
+    // Angular distance is in [0, 1], with 0 = identical
+    // Invert so higher score = more likely continuation
+    return 1 - distance;
+}
+/**
+ * Classify transitions using only embedding distance.
+ */
+export function classifyWithEmbeddings(embeddedTransitions) {
+    return embeddedTransitions.map((et) => {
+        const lexicalFeatures = extractLexicalFeatures(et.transition.prevAssistantText, et.transition.nextUserText, et.transition.timeGapMs);
+        const features = {
+            ...lexicalFeatures,
+            embeddingDistanceMin: et.embeddingDistanceMin,
+            embeddingDistanceMean: et.embeddingDistanceMean,
+        };
+        return {
+            transitionId: et.transition.id,
+            groundTruth: et.transition.label,
+            continuationScore: embeddingDistanceToScore(et.embeddingDistanceMin),
+            features,
+        };
+    });
+}
+/**
+ * Get just the embedding distances for a model without full classification.
+ * Useful for feature ablation where we want to test embedding contribution.
+ */
+export async function getEmbeddingDistances(modelId, transitions) {
+    const embedded = await embedTransitions(modelId, transitions);
+    const distances = new Map();
+    for (const et of embedded) {
+        distances.set(et.transition.id, {
+            min: et.embeddingDistanceMin,
+            mean: et.embeddingDistanceMean,
+        });
+    }
+    return distances;
+}
+/**
+ * Create an embedding cache from embedded transitions.
+ */
+export function createEmbeddingCache(modelId, embeddedTransitions) {
+    const transitions = new Map();
+    for (const et of embeddedTransitions) {
+        transitions.set(et.transition.id, et);
+    }
+    return { modelId, transitions };
+}
+/**
+ * Classify from cache without re-embedding.
+ */
+export function classifyFromCache(cache) {
+    const results = [];
+    for (const et of cache.transitions.values()) {
+        const lexicalFeatures = extractLexicalFeatures(et.transition.prevAssistantText, et.transition.nextUserText, et.transition.timeGapMs);
+        const features = {
+            ...lexicalFeatures,
+            embeddingDistanceMin: et.embeddingDistanceMin,
+            embeddingDistanceMean: et.embeddingDistanceMean,
+        };
+        results.push({
+            transitionId: et.transition.id,
+            groundTruth: et.transition.label,
+            continuationScore: embeddingDistanceToScore(et.embeddingDistanceMin),
+            features,
+        });
+    }
+    return results;
+}
+//# sourceMappingURL=embedding-classifier.js.map

package/dist/eval/experiments/topic-continuity/embedding-classifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-classifier.js","sourceRoot":"","sources":["../../../../src/eval/experiments/topic-continuity/embedding-classifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,mCAAmC,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,oCAAoC,CAAC;AAErE,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAa/D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,WAA6B;IAE7B,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,KAAK,CAAC,CAAC;QACvC,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE5B,MAAM,OAAO,GAAyB,EAAE,CAAC;QAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YAEzB,mDAAmD;YACnD,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC;gBAChC,SAAS;YACX,CAAC;YAED,4DAA4D;YAC5D,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;YAE9D,sEAAsE;YACtE,MAAM,eAAe,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;YAEzE,2DAA2D;YAC3D,0EAA0E;YAC1E,MAAM,QAAQ,GAAG,eAAe,CAAC,UAAU,CAAC,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC,CAAC;YAElF,OAAO,CAAC,IAAI,CAAC;gBACX,UAAU,EAAE,CAAC;gBACb,aAAa,EAAE,UAAU,CAAC,SAAS;gBACnC,kBAAkB,EAAE,eAAe,CAAC,SAAS;gBAC7C,oBAAoB,EAAE,QAAQ;gBAC9B,qBAAqB,EAAE,QAAQ;aAChC,CAAC,CAAC;YAEH,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,uBAAuB,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;YAAS,CAAC;QACT,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,QAAgB;IACvD,oDAAoD;IACpD,oDAAoD;IACpD,OAAO,CAAC,GAAG,QAAQ,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CACpC,mBAAyC;IAEzC,OAAO,mBAAmB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACpC,MAAM,eAAe,GAAG,sBAAsB,CAC5C,EAAE,CAAC,UAAU,CAAC,iBAAiB,EAC/B,EAAE,CAAC,UAAU,CAAC,YAAY,EAC1B,EAAE,CAAC,UAAU,CAAC,SAAS,CACxB,CAAC;QAEF,MAAM,QAAQ,GAAuB;YACnC,GAAG,eAAe;YAClB,oBAAoB,EAAE,EAAE,CAAC,oBAAoB;YAC7C,qBAAqB,EAAE,EAAE,CAAC,qBAAqB;SAChD,CAAC;QAEF,OAAO;YACL,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE;YAC9B,WAAW,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK;YAChC,iBAAiB,EAAE,wBAAwB,CAAC,EAAE,CAAC,oBAAoB,CAAC;YACpE,QAAQ;SACT,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,OAAe,EACf,WAA6B;IAE7B,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyC,CAAC;IAEnE,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE;YAC9B,GAAG,EAAE,EAAE,CAAC,oBAAoB;YAC5B,IAAI,EAAE,EAAE,CAAC,qBAAqB;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAUD;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAe,EACf,mBAAyC;IAEzC,MAAM,WAAW,GAAG,IAAI,GAAG,EAA8B,CAAC;IAC1D,KAAK,MAAM,EAAE,IAAI,mBAAmB,EAAE,CAAC;QACrC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;AAClC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAqB;IACrD,MAAM,OAAO,GAA2B,EAAE,CAAC;IAE3C,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;QAC5C,MAAM,eAAe,GAAG,sBAAsB,CAC5C,EAAE,CAAC,UAAU,CAAC,iBAAiB,EAC/B,EAAE,CAAC,UAAU,CAAC,YAAY,EAC1B,EAAE,CAAC,UAAU,CAAC,SAAS,CACxB,CAAC;QAEF,MAAM,QAAQ,GAAuB;YACnC,GAAG,eAAe;YAClB,oBAAoB,EAAE,EAAE,CAAC,oBAAoB;YAC7C,qBAAqB,EAAE,EAAE,CAAC,qBAAqB;SAChD,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC;YACX,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE;YAC9B,WAAW,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK;YAChC,iBAAiB,EAAE,wBAAwB,CAAC,EAAE,CAAC,oBAAoB,CAAC;YACpE,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/eval/experiments/topic-continuity/hybrid-classifier.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Hybrid classifier combining embedding distance with lexical features.
+ *
+ * Uses tunable weights to combine multiple signals for topic continuity
+ * prediction.
+ */
+import type { TurnTransition, TransitionFeatures, ClassificationResult } from './types.js';
+import { type EmbeddedTransition } from './embedding-classifier.js';
+/**
+ * Weight configuration for hybrid classifier.
+ */
+export interface HybridWeights {
+    /** Weight for embedding distance signal. Default: 0.5 */
+    embedding: number;
+    /** Weight for topic shift marker. Default: 0.2 */
+    topicShiftMarker: number;
+    /** Weight for continuation marker. Default: 0.15 */
+    continuationMarker: number;
+    /** Weight for time gap signal. Default: 0.05 */
+    timeGap: number;
+    /** Weight for file path overlap. Default: 0.05 */
+    filePathOverlap: number;
+    /** Weight for keyword overlap. Default: 0.05 */
+    keywordOverlap: number;
+}
+export declare const DEFAULT_WEIGHTS: HybridWeights;
+/**
+ * Compute hybrid continuation score from features.
+ */
+export declare function computeHybridScore(features: TransitionFeatures, weights?: HybridWeights): number;
+/**
+ * Classify transitions using hybrid approach (requires pre-embedded data).
+ */
+export declare function classifyWithHybrid(embeddedTransitions: EmbeddedTransition[], weights?: HybridWeights): ClassificationResult[];
+/**
+ * Classify using lexical features only (no embedding).
+ */
+export declare function classifyWithLexicalOnly(transitions: TurnTransition[]): ClassificationResult[];
+/**
+ * Feature ablation: classify with only a subset of features enabled.
+ */
+export interface FeatureFlags {
+    useEmbedding: boolean;
+    useTopicShiftMarker: boolean;
+    useContinuationMarker: boolean;
+    useTimeGap: boolean;
+    useFilePathOverlap: boolean;
+    useKeywordOverlap: boolean;
+}
+export declare const ALL_FEATURES: FeatureFlags;
+export declare const EMBEDDING_ONLY: FeatureFlags;
+export declare const LEXICAL_ONLY: FeatureFlags;
+/**
+ * Compute score with specific features enabled/disabled.
+ */
+export declare function computeAblatedScore(features: TransitionFeatures, flags: FeatureFlags, baseWeights?: HybridWeights): number;
+/**
+ * Classify with specific features enabled for ablation study.
+ */
+export declare function classifyWithAblation(embeddedTransitions: EmbeddedTransition[], flags: FeatureFlags): ClassificationResult[];
+/**
+ * Named ablation configurations for experiment.
+ */
+export declare const ABLATION_CONFIGS: {
+    name: string;
+    flags: FeatureFlags;
+}[];
+//# sourceMappingURL=hybrid-classifier.d.ts.map

package/dist/eval/experiments/topic-continuity/hybrid-classifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-classifier.d.ts","sourceRoot":"","sources":["../../../../src/eval/experiments/topic-continuity/hybrid-classifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,KAAK,kBAAkB,EAExB,MAAM,2BAA2B,CAAC;AAEnC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,yDAAyD;IACzD,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,gBAAgB,EAAE,MAAM,CAAC;IACzB,oDAAoD;IACpD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,eAAe,EAAE,MAAM,CAAC;IACxB,gDAAgD;IAChD,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,eAAO,MAAM,eAAe,EAAE,aAO7B,CAAC;AAEF;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,GAAE,aAA+B,GACvC,MAAM,CAkCR;AAaD;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,mBAAmB,EAAE,kBAAkB,EAAE,EACzC,OAAO,GAAE,aAA+B,GACvC,oBAAoB,EAAE,CAuBxB;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,cAAc,EAAE,GAC5B,oBAAoB,EAAE,CAyBxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,OAAO,CAAC;IACtB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,qBAAqB,EAAE,OAAO,CAAC;IAC/B,UAAU,EAAE,OAAO,CAAC;IACpB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,iBAAiB,EAAE,OAAO,CAAC;CAC5B;AAED,eAAO,MAAM,YAAY,EAAE,YAO1B,CAAC;AAEF,eAAO,MAAM,cAAc,EAAE,YAO5B,CAAC;AAEF,eAAO,MAAM,YAAY,EAAE,YAO1B,CAAC;AAEF;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,kBAAkB,EAC5B,KAAK,EAAE,YAAY,EACnB,WAAW,GAAE,aAA+B,GAC3C,MAAM,CAYR;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,mBAAmB,EAAE,kBAAkB,EAAE,EACzC,KAAK,EAAE,YAAY,GAClB,oBAAoB,EAAE,CAuBxB;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,YAAY,CAAA;CAAE,EAanE,CAAC"}