causantic 0.9.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -56
- package/dist/cli/skill-templates.d.ts.map +1 -1
- package/dist/cli/skill-templates.js +23 -18
- package/dist/cli/skill-templates.js.map +1 -1
- package/dist/clusters/cluster-manager.d.ts +16 -0
- package/dist/clusters/cluster-manager.d.ts.map +1 -1
- package/dist/clusters/cluster-manager.js +119 -1
- package/dist/clusters/cluster-manager.js.map +1 -1
- package/dist/config/loader.d.ts +16 -0
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +51 -0
- package/dist/config/loader.js.map +1 -1
- package/dist/config/memory-config.d.ts +26 -0
- package/dist/config/memory-config.d.ts.map +1 -1
- package/dist/config/memory-config.js +22 -0
- package/dist/config/memory-config.js.map +1 -1
- package/dist/eval/experiments/embedding-model-comparison/run-experiment.d.ts +20 -0
- package/dist/eval/experiments/embedding-model-comparison/run-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/embedding-model-comparison/run-experiment.js +289 -0
- package/dist/eval/experiments/embedding-model-comparison/run-experiment.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/alignment-analysis.d.ts +53 -0
- package/dist/eval/experiments/index-differentiation/alignment-analysis.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/alignment-analysis.js +91 -0
- package/dist/eval/experiments/index-differentiation/alignment-analysis.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/discrimination-test.d.ts +24 -0
- package/dist/eval/experiments/index-differentiation/discrimination-test.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/discrimination-test.js +79 -0
- package/dist/eval/experiments/index-differentiation/discrimination-test.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/index.d.ts +11 -0
- package/dist/eval/experiments/index-differentiation/index.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/index.js +8 -0
- package/dist/eval/experiments/index-differentiation/index.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/refinement-test.d.ts +32 -0
- package/dist/eval/experiments/index-differentiation/refinement-test.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/refinement-test.js +203 -0
- package/dist/eval/experiments/index-differentiation/refinement-test.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/run-experiment.d.ts +20 -0
- package/dist/eval/experiments/index-differentiation/run-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/run-experiment.js +338 -0
- package/dist/eval/experiments/index-differentiation/run-experiment.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/similarity-analysis.d.ts +31 -0
- package/dist/eval/experiments/index-differentiation/similarity-analysis.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/similarity-analysis.js +60 -0
- package/dist/eval/experiments/index-differentiation/similarity-analysis.js.map +1 -0
- package/dist/eval/experiments/index-differentiation/types.d.ts +114 -0
- package/dist/eval/experiments/index-differentiation/types.d.ts.map +1 -0
- package/dist/eval/experiments/index-differentiation/types.js +8 -0
- package/dist/eval/experiments/index-differentiation/types.js.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.d.ts +19 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js +328 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-experiment.js.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.d.ts +27 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.d.ts.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.js +154 -0
- package/dist/eval/experiments/index-vs-chunk/jeopardy-generator.js.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/query-generator.d.ts +23 -0
- package/dist/eval/experiments/index-vs-chunk/query-generator.d.ts.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/query-generator.js +113 -0
- package/dist/eval/experiments/index-vs-chunk/query-generator.js.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/run-experiment.d.ts +17 -0
- package/dist/eval/experiments/index-vs-chunk/run-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/run-experiment.js +341 -0
- package/dist/eval/experiments/index-vs-chunk/run-experiment.js.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/types.d.ts +71 -0
- package/dist/eval/experiments/index-vs-chunk/types.d.ts.map +1 -0
- package/dist/eval/experiments/index-vs-chunk/types.js +8 -0
- package/dist/eval/experiments/index-vs-chunk/types.js.map +1 -0
- package/dist/eval/experiments/pipeline-dropout/run-experiment.d.ts +18 -0
- package/dist/eval/experiments/pipeline-dropout/run-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/pipeline-dropout/run-experiment.js +347 -0
- package/dist/eval/experiments/pipeline-dropout/run-experiment.js.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/analyze-misses.d.ts +17 -0
- package/dist/eval/experiments/rescorer-ceiling/analyze-misses.d.ts.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js +247 -0
- package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.d.ts +18 -0
- package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.d.ts.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js +443 -0
- package/dist/eval/experiments/rescorer-ceiling/benchmark-rescorers.js.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/run-experiment.d.ts +16 -0
- package/dist/eval/experiments/rescorer-ceiling/run-experiment.d.ts.map +1 -0
- package/dist/eval/experiments/rescorer-ceiling/run-experiment.js +226 -0
- package/dist/eval/experiments/rescorer-ceiling/run-experiment.js.map +1 -0
- package/dist/index-entries/index-generator.d.ts +74 -0
- package/dist/index-entries/index-generator.d.ts.map +1 -0
- package/dist/index-entries/index-generator.js +323 -0
- package/dist/index-entries/index-generator.js.map +1 -0
- package/dist/index-entries/index-refresher.d.ts +54 -0
- package/dist/index-entries/index-refresher.d.ts.map +1 -0
- package/dist/index-entries/index-refresher.js +203 -0
- package/dist/index-entries/index-refresher.js.map +1 -0
- package/dist/index-entries/index.d.ts +6 -0
- package/dist/index-entries/index.d.ts.map +1 -0
- package/dist/index-entries/index.js +6 -0
- package/dist/index-entries/index.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/ingest/index-entry-hook.d.ts +15 -0
- package/dist/ingest/index-entry-hook.d.ts.map +1 -0
- package/dist/ingest/index-entry-hook.js +84 -0
- package/dist/ingest/index-entry-hook.js.map +1 -0
- package/dist/ingest/ingest-session.d.ts.map +1 -1
- package/dist/ingest/ingest-session.js +72 -18
- package/dist/ingest/ingest-session.js.map +1 -1
- package/dist/ingest/session-state.d.ts +49 -0
- package/dist/ingest/session-state.d.ts.map +1 -0
- package/dist/ingest/session-state.js +158 -0
- package/dist/ingest/session-state.js.map +1 -0
- package/dist/maintenance/scheduler.d.ts.map +1 -1
- package/dist/maintenance/scheduler.js +25 -0
- package/dist/maintenance/scheduler.js.map +1 -1
- package/dist/maintenance/tasks/backfill-index.d.ts +27 -0
- package/dist/maintenance/tasks/backfill-index.d.ts.map +1 -0
- package/dist/maintenance/tasks/backfill-index.js +44 -0
- package/dist/maintenance/tasks/backfill-index.js.map +1 -0
- package/dist/mcp/tools.d.ts +4 -0
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +115 -7
- package/dist/mcp/tools.js.map +1 -1
- package/dist/models/embedder.js +2 -2
- package/dist/models/embedder.js.map +1 -1
- package/dist/models/model-registry.d.ts +2 -0
- package/dist/models/model-registry.d.ts.map +1 -1
- package/dist/models/model-registry.js +15 -0
- package/dist/models/model-registry.js.map +1 -1
- package/dist/repomap/cache.d.ts +58 -0
- package/dist/repomap/cache.d.ts.map +1 -0
- package/dist/repomap/cache.js +101 -0
- package/dist/repomap/cache.js.map +1 -0
- package/dist/repomap/graph.d.ts +54 -0
- package/dist/repomap/graph.d.ts.map +1 -0
- package/dist/repomap/graph.js +113 -0
- package/dist/repomap/graph.js.map +1 -0
- package/dist/repomap/index.d.ts +83 -0
- package/dist/repomap/index.d.ts.map +1 -0
- package/dist/repomap/index.js +99 -0
- package/dist/repomap/index.js.map +1 -0
- package/dist/repomap/parser.d.ts +43 -0
- package/dist/repomap/parser.d.ts.map +1 -0
- package/dist/repomap/parser.js +994 -0
- package/dist/repomap/parser.js.map +1 -0
- package/dist/repomap/regex-parser.d.ts +24 -0
- package/dist/repomap/regex-parser.d.ts.map +1 -0
- package/dist/repomap/regex-parser.js +190 -0
- package/dist/repomap/regex-parser.js.map +1 -0
- package/dist/repomap/renderer.d.ts +40 -0
- package/dist/repomap/renderer.d.ts.map +1 -0
- package/dist/repomap/renderer.js +163 -0
- package/dist/repomap/renderer.js.map +1 -0
- package/dist/repomap/scanner.d.ts +32 -0
- package/dist/repomap/scanner.d.ts.map +1 -0
- package/dist/repomap/scanner.js +171 -0
- package/dist/repomap/scanner.js.map +1 -0
- package/dist/retrieval/chain-assembler.d.ts.map +1 -1
- package/dist/retrieval/chain-assembler.js +22 -3
- package/dist/retrieval/chain-assembler.js.map +1 -1
- package/dist/retrieval/index.d.ts +2 -0
- package/dist/retrieval/index.d.ts.map +1 -1
- package/dist/retrieval/index.js +2 -0
- package/dist/retrieval/index.js.map +1 -1
- package/dist/retrieval/mmr.d.ts +1 -0
- package/dist/retrieval/mmr.d.ts.map +1 -1
- package/dist/retrieval/mmr.js +35 -1
- package/dist/retrieval/mmr.js.map +1 -1
- package/dist/retrieval/search-assembler.d.ts +10 -1
- package/dist/retrieval/search-assembler.d.ts.map +1 -1
- package/dist/retrieval/search-assembler.js +249 -81
- package/dist/retrieval/search-assembler.js.map +1 -1
- package/dist/retrieval/session-reconstructor.d.ts +36 -0
- package/dist/retrieval/session-reconstructor.d.ts.map +1 -1
- package/dist/retrieval/session-reconstructor.js +126 -0
- package/dist/retrieval/session-reconstructor.js.map +1 -1
- package/dist/storage/db.d.ts.map +1 -1
- package/dist/storage/db.js +15 -0
- package/dist/storage/db.js.map +1 -1
- package/dist/storage/index-entry-store.d.ts +71 -0
- package/dist/storage/index-entry-store.d.ts.map +1 -0
- package/dist/storage/index-entry-store.js +275 -0
- package/dist/storage/index-entry-store.js.map +1 -0
- package/dist/storage/index.d.ts +5 -2
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +5 -1
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/migrations.d.ts.map +1 -1
- package/dist/storage/migrations.js +102 -0
- package/dist/storage/migrations.js.map +1 -1
- package/dist/storage/schema.sql +68 -2
- package/dist/storage/session-state-store.d.ts +61 -0
- package/dist/storage/session-state-store.d.ts.map +1 -0
- package/dist/storage/session-state-store.js +119 -0
- package/dist/storage/session-state-store.js.map +1 -0
- package/dist/storage/types.d.ts +50 -0
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/storage/vector-store.d.ts +17 -2
- package/dist/storage/vector-store.d.ts.map +1 -1
- package/dist/storage/vector-store.js +96 -36
- package/dist/storage/vector-store.js.map +1 -1
- package/package.json +4 -2
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index Entry Differentiation Experiment
|
|
3
|
+
*
|
|
4
|
+
* Tests whether semantic index entries within the same topic cluster are
|
|
5
|
+
* too similar to each other, reducing the search system's ability to
|
|
6
|
+
* discriminate between chunks covering the same topic.
|
|
7
|
+
*
|
|
8
|
+
* Three phases:
|
|
9
|
+
* 1. Similarity analysis — do entries converge more than raw chunks?
|
|
10
|
+
* 2. Discrimination test — can we find the right entry among siblings?
|
|
11
|
+
* 3. Refinement simulation — does cluster-aware regeneration help?
|
|
12
|
+
*
|
|
13
|
+
* Usage:
|
|
14
|
+
* npx tsx src/eval/experiments/index-differentiation/run-experiment.ts [--refine] [--max-clusters N]
|
|
15
|
+
*
|
|
16
|
+
* Requires: populated index entries and chunk cluster assignments.
|
|
17
|
+
* Run `npx causantic maintenance run backfill-index` first if needed.
|
|
18
|
+
*/
|
|
19
|
+
import { getDb } from '../../../storage/db.js';
|
|
20
|
+
import { vectorStore, indexVectorStore } from '../../../storage/vector-store.js';
|
|
21
|
+
import { getClusterChunkIds, getAllClusters } from '../../../storage/cluster-store.js';
|
|
22
|
+
import { getIndexEntriesForChunk, getIndexEntryCount, } from '../../../storage/index-entry-store.js';
|
|
23
|
+
import { getChunkById } from '../../../storage/chunk-store.js';
|
|
24
|
+
import { runSimilarityAnalysis } from './similarity-analysis.js';
|
|
25
|
+
import { runDiscriminationTest } from './discrimination-test.js';
|
|
26
|
+
import { runAlignmentAnalysis } from './alignment-analysis.js';
|
|
27
|
+
import { runRefinementTest } from './refinement-test.js';
|
|
28
|
+
const MIN_ENTRIES_PER_CLUSTER = 3;
|
|
29
|
+
/**
|
|
30
|
+
* Build the analysis dataset: for each cluster with enough index entries,
|
|
31
|
+
* gather the entry + chunk embeddings.
|
|
32
|
+
*/
|
|
33
|
+
async function buildClustersForAnalysis() {
|
|
34
|
+
// Force DB initialisation (runs migrations, loads config)
|
|
35
|
+
getDb();
|
|
36
|
+
const allClusters = getAllClusters();
|
|
37
|
+
console.log(` Total clusters: ${allClusters.length}`);
|
|
38
|
+
// Load all embeddings
|
|
39
|
+
const chunkVectors = await vectorStore.getAllVectors();
|
|
40
|
+
const indexVectors = await indexVectorStore.getAllVectors();
|
|
41
|
+
const chunkEmbMap = new Map(chunkVectors.map((v) => [v.id, v.embedding]));
|
|
42
|
+
const indexEmbMap = new Map(indexVectors.map((v) => [v.id, v.embedding]));
|
|
43
|
+
console.log(` Chunk vectors: ${chunkEmbMap.size}`);
|
|
44
|
+
console.log(` Index vectors: ${indexEmbMap.size}`);
|
|
45
|
+
const clusters = [];
|
|
46
|
+
const allEntries = [];
|
|
47
|
+
const chunkContents = new Map();
|
|
48
|
+
for (const cluster of allClusters) {
|
|
49
|
+
const chunkIds = getClusterChunkIds(cluster.id);
|
|
50
|
+
// Find index entries for chunks in this cluster
|
|
51
|
+
const clusterEntries = [];
|
|
52
|
+
const seenEntryIds = new Set();
|
|
53
|
+
for (const chunkId of chunkIds) {
|
|
54
|
+
const entries = getIndexEntriesForChunk(chunkId);
|
|
55
|
+
for (const entry of entries) {
|
|
56
|
+
if (seenEntryIds.has(entry.id))
|
|
57
|
+
continue;
|
|
58
|
+
seenEntryIds.add(entry.id);
|
|
59
|
+
const entryEmbedding = indexEmbMap.get(entry.id);
|
|
60
|
+
if (!entryEmbedding)
|
|
61
|
+
continue;
|
|
62
|
+
// Get chunk embeddings for this entry
|
|
63
|
+
const chunkEmbeddings = [];
|
|
64
|
+
for (const cid of entry.chunkIds) {
|
|
65
|
+
const emb = chunkEmbMap.get(cid);
|
|
66
|
+
if (emb)
|
|
67
|
+
chunkEmbeddings.push(emb);
|
|
68
|
+
// Collect chunk content for refinement phase
|
|
69
|
+
if (!chunkContents.has(cid)) {
|
|
70
|
+
const chunk = getChunkById(cid);
|
|
71
|
+
if (chunk)
|
|
72
|
+
chunkContents.set(cid, chunk.content);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
clusterEntries.push({
|
|
76
|
+
entryId: entry.id,
|
|
77
|
+
entryEmbedding,
|
|
78
|
+
chunkEmbeddings,
|
|
79
|
+
});
|
|
80
|
+
allEntries.push(entry);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
if (clusterEntries.length >= MIN_ENTRIES_PER_CLUSTER) {
|
|
84
|
+
clusters.push({
|
|
85
|
+
clusterId: cluster.id,
|
|
86
|
+
clusterName: cluster.name,
|
|
87
|
+
entries: clusterEntries,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return { clusters, allEntries, chunkContents };
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Format a number for display.
|
|
95
|
+
*/
|
|
96
|
+
function fmt(n, decimals = 3) {
|
|
97
|
+
return n.toFixed(decimals);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Run the full experiment.
|
|
101
|
+
*/
|
|
102
|
+
async function runExperiment() {
|
|
103
|
+
const args = process.argv.slice(2);
|
|
104
|
+
const doRefine = args.includes('--refine');
|
|
105
|
+
const maxClustersArg = args.find((a) => a.startsWith('--max-clusters='));
|
|
106
|
+
const maxClusters = maxClustersArg
|
|
107
|
+
? parseInt(maxClustersArg.split('=')[1], 10)
|
|
108
|
+
: 5;
|
|
109
|
+
console.log('=== Index Entry Differentiation Experiment ===\n');
|
|
110
|
+
const totalEntries = getIndexEntryCount();
|
|
111
|
+
console.log(`Total index entries: ${totalEntries}`);
|
|
112
|
+
if (totalEntries === 0) {
|
|
113
|
+
console.log('\nNo index entries found. Run backfill first:\n npx causantic maintenance run backfill-index');
|
|
114
|
+
process.exit(1);
|
|
115
|
+
}
|
|
116
|
+
// Build dataset
|
|
117
|
+
console.log('\nBuilding analysis dataset...');
|
|
118
|
+
const { clusters, allEntries, chunkContents } = await buildClustersForAnalysis();
|
|
119
|
+
console.log(` Eligible clusters (≥${MIN_ENTRIES_PER_CLUSTER} entries): ${clusters.length}`);
|
|
120
|
+
console.log(` Total entries in eligible clusters: ${clusters.reduce((s, c) => s + c.entries.length, 0)}`);
|
|
121
|
+
if (clusters.length === 0) {
|
|
122
|
+
console.log('\nNo clusters with enough index entries. Need at least ' +
|
|
123
|
+
`${MIN_ENTRIES_PER_CLUSTER} entries per cluster. ` +
|
|
124
|
+
'Run clustering and backfill first.');
|
|
125
|
+
process.exit(1);
|
|
126
|
+
}
|
|
127
|
+
// ── Phase 1: Similarity Analysis ──────────────────────────────────────────
|
|
128
|
+
console.log('\n── Phase 1: Intra-Cluster Similarity Analysis ──\n');
|
|
129
|
+
const similarityResults = runSimilarityAnalysis(clusters);
|
|
130
|
+
const meanCompression = similarityResults.reduce((s, r) => s + r.compressionRatio, 0) /
|
|
131
|
+
similarityResults.length;
|
|
132
|
+
const homogenized = similarityResults.filter((r) => r.compressionRatio > 1.0).length;
|
|
133
|
+
console.log(' Cluster Entries Entry Sim Chunk Sim Ratio');
|
|
134
|
+
console.log(' ' + '─'.repeat(75));
|
|
135
|
+
for (const r of similarityResults.slice(0, 20)) {
|
|
136
|
+
const name = (r.clusterName ?? r.clusterId).slice(0, 30).padEnd(32);
|
|
137
|
+
const ratio = r.compressionRatio > 1 ? `${fmt(r.compressionRatio)} ▲` : `${fmt(r.compressionRatio)} ▼`;
|
|
138
|
+
console.log(` ${name} ${String(r.entryCount).padStart(4)} ${fmt(r.meanEntryPairSim)} ${fmt(r.meanChunkPairSim)} ${ratio}`);
|
|
139
|
+
}
|
|
140
|
+
if (similarityResults.length > 20) {
|
|
141
|
+
console.log(` ... and ${similarityResults.length - 20} more clusters`);
|
|
142
|
+
}
|
|
143
|
+
console.log(`\n Mean compression ratio: ${fmt(meanCompression)}`);
|
|
144
|
+
console.log(` Homogenized clusters (ratio > 1): ${homogenized}/${similarityResults.length} (${fmt((100 * homogenized) / similarityResults.length, 1)}%)`);
|
|
145
|
+
// ── Phase 2: Discrimination Test ──────────────────────────────────────────
|
|
146
|
+
console.log('\n── Phase 2: Discrimination Test ──\n');
|
|
147
|
+
const discriminationResults = runDiscriminationTest(clusters);
|
|
148
|
+
let totalRR = 0;
|
|
149
|
+
let totalHits = 0;
|
|
150
|
+
let totalTests = 0;
|
|
151
|
+
for (const r of discriminationResults) {
|
|
152
|
+
totalRR += r.perEntry.reduce((s, e) => s + 1 / e.rankAmongSiblings, 0);
|
|
153
|
+
totalHits += r.perEntry.filter((e) => e.rankAmongSiblings === 1).length;
|
|
154
|
+
totalTests += r.perEntry.length;
|
|
155
|
+
}
|
|
156
|
+
const overallMRR = totalRR / totalTests;
|
|
157
|
+
const overallHitRate = totalHits / totalTests;
|
|
158
|
+
console.log(' Cluster Entries MRR Hit Rate');
|
|
159
|
+
console.log(' ' + '─'.repeat(65));
|
|
160
|
+
for (const r of discriminationResults.slice(0, 20)) {
|
|
161
|
+
const name = (r.clusterName ?? r.clusterId).slice(0, 30).padEnd(32);
|
|
162
|
+
console.log(` ${name} ${String(r.entryCount).padStart(4)} ${fmt(r.meanReciprocalRank)} ${fmt(r.hitRate * 100, 1)}%`);
|
|
163
|
+
}
|
|
164
|
+
if (discriminationResults.length > 20) {
|
|
165
|
+
console.log(` ... and ${discriminationResults.length - 20} more clusters`);
|
|
166
|
+
}
|
|
167
|
+
console.log(`\n Overall MRR: ${fmt(overallMRR)}`);
|
|
168
|
+
console.log(` Overall hit rate: ${fmt(overallHitRate * 100, 1)}%`);
|
|
169
|
+
// ── Phase 2b: Alignment Analysis ─────────────────────────────────────────
|
|
170
|
+
console.log('\n── Phase 2b: Entry-to-Chunk Alignment Analysis ──\n');
|
|
171
|
+
const alignmentResults = runAlignmentAnalysis(clusters);
|
|
172
|
+
let totalSelfAlign = 0;
|
|
173
|
+
let totalSibAlign = 0;
|
|
174
|
+
let totalGap = 0;
|
|
175
|
+
let totalUnique = 0;
|
|
176
|
+
let totalAlignEntries = 0;
|
|
177
|
+
for (const r of alignmentResults) {
|
|
178
|
+
const validCount = r.perEntry.filter((e) => e.selfAlignment > 0).length;
|
|
179
|
+
totalSelfAlign += r.meanSelfAlignment * validCount;
|
|
180
|
+
totalSibAlign += r.meanSiblingAlignment * validCount;
|
|
181
|
+
totalGap += r.meanAlignmentGap * validCount;
|
|
182
|
+
totalUnique += r.uniquelyAlignedFraction * validCount;
|
|
183
|
+
totalAlignEntries += validCount;
|
|
184
|
+
}
|
|
185
|
+
const overallSelfAlign = totalAlignEntries > 0 ? totalSelfAlign / totalAlignEntries : 0;
|
|
186
|
+
const overallSibAlign = totalAlignEntries > 0 ? totalSibAlign / totalAlignEntries : 0;
|
|
187
|
+
const overallGap = totalAlignEntries > 0 ? totalGap / totalAlignEntries : 0;
|
|
188
|
+
const overallUniquelyAligned = totalAlignEntries > 0 ? totalUnique / totalAlignEntries : 0;
|
|
189
|
+
console.log(' Cluster Entries Self Sibling Gap Unique%');
|
|
190
|
+
console.log(' ' + '─'.repeat(80));
|
|
191
|
+
for (const r of alignmentResults.slice(0, 20)) {
|
|
192
|
+
const name = (r.clusterName ?? r.clusterId).slice(0, 30).padEnd(32);
|
|
193
|
+
console.log(` ${name} ${String(r.entryCount).padStart(4)} ${fmt(r.meanSelfAlignment)} ${fmt(r.meanSiblingAlignment)} ${fmt(r.meanAlignmentGap)} ${fmt(r.uniquelyAlignedFraction * 100, 1)}%`);
|
|
194
|
+
}
|
|
195
|
+
if (alignmentResults.length > 20) {
|
|
196
|
+
console.log(` ... and ${alignmentResults.length - 20} more clusters`);
|
|
197
|
+
}
|
|
198
|
+
console.log(`\n Overall self-alignment (entry ↔ own chunk): ${fmt(overallSelfAlign)}`);
|
|
199
|
+
console.log(` Overall sibling alignment (entry ↔ other chunks): ${fmt(overallSibAlign)}`);
|
|
200
|
+
console.log(` Overall alignment gap (self - sibling): ${fmt(overallGap)}`);
|
|
201
|
+
console.log(` Uniquely aligned (self > max sibling): ${fmt(overallUniquelyAligned * 100, 1)}%`);
|
|
202
|
+
// ── Phase 3: Refinement (optional) ────────────────────────────────────────
|
|
203
|
+
let refinementSection;
|
|
204
|
+
if (doRefine) {
|
|
205
|
+
console.log(`\n── Phase 3: Cluster-Aware Refinement (top ${maxClusters} worst clusters) ──\n`);
|
|
206
|
+
const discriminationScores = new Map(discriminationResults.map((r) => [r.clusterId, r.meanReciprocalRank]));
|
|
207
|
+
const refinementResults = await runRefinementTest(clusters, allEntries, chunkContents, discriminationScores, maxClusters);
|
|
208
|
+
if (refinementResults && refinementResults.length > 0) {
|
|
209
|
+
const meanMRRDelta = refinementResults.reduce((s, r) => s + (r.refinedMRR - r.baselineMRR), 0) / refinementResults.length;
|
|
210
|
+
const meanCRDelta = refinementResults.reduce((s, r) => s + r.compressionRatioDelta, 0) /
|
|
211
|
+
refinementResults.length;
|
|
212
|
+
console.log('\n Cluster Baseline MRR Refined MRR Delta');
|
|
213
|
+
console.log(' ' + '─'.repeat(65));
|
|
214
|
+
for (const r of refinementResults) {
|
|
215
|
+
const name = (r.clusterName ?? r.clusterId).slice(0, 27).padEnd(29);
|
|
216
|
+
const delta = r.refinedMRR - r.baselineMRR;
|
|
217
|
+
const arrow = delta > 0.01 ? ' ▲' : delta < -0.01 ? ' ▼' : ' ─';
|
|
218
|
+
console.log(` ${name} ${fmt(r.baselineMRR)} ${fmt(r.refinedMRR)} ${delta > 0 ? '+' : ''}${fmt(delta)}${arrow}`);
|
|
219
|
+
}
|
|
220
|
+
console.log(`\n Mean MRR delta: ${meanMRRDelta > 0 ? '+' : ''}${fmt(meanMRRDelta)}`);
|
|
221
|
+
console.log(` Mean compression ratio delta: ${meanCRDelta > 0 ? '+' : ''}${fmt(meanCRDelta)}`);
|
|
222
|
+
// Show sample refinements
|
|
223
|
+
const withSamples = refinementResults.filter((r) => r.sampleRefinements.length > 0);
|
|
224
|
+
if (withSamples.length > 0) {
|
|
225
|
+
console.log('\n Sample refinements:');
|
|
226
|
+
for (const r of withSamples.slice(0, 2)) {
|
|
227
|
+
console.log(`\n Cluster: ${r.clusterName ?? r.clusterId}`);
|
|
228
|
+
for (const s of r.sampleRefinements.slice(0, 2)) {
|
|
229
|
+
console.log(` Original: ${s.original.slice(0, 120)}...`);
|
|
230
|
+
console.log(` Refined: ${s.refined.slice(0, 120)}...`);
|
|
231
|
+
console.log('');
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
refinementSection = {
|
|
236
|
+
results: refinementResults,
|
|
237
|
+
meanMRRDelta,
|
|
238
|
+
meanCompressionRatioDelta: meanCRDelta,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
else {
|
|
243
|
+
console.log('\n (Skipping Phase 3 refinement — pass --refine to enable)');
|
|
244
|
+
}
|
|
245
|
+
// ── Summary ───────────────────────────────────────────────────────────────
|
|
246
|
+
const summary = [];
|
|
247
|
+
summary.push(`Analysed ${clusters.length} clusters (${clusters.reduce((s, c) => s + c.entries.length, 0)} entries)`);
|
|
248
|
+
if (meanCompression > 1.05) {
|
|
249
|
+
summary.push(`Compression homogenizes: mean ratio ${fmt(meanCompression)} (entries ${fmt((meanCompression - 1) * 100, 1)}% more similar than chunks)`);
|
|
250
|
+
}
|
|
251
|
+
else if (meanCompression < 0.95) {
|
|
252
|
+
summary.push(`LLM naturally differentiates: mean ratio ${fmt(meanCompression)} (entries ${fmt((1 - meanCompression) * 100, 1)}% less similar than chunks)`);
|
|
253
|
+
}
|
|
254
|
+
else {
|
|
255
|
+
summary.push(`Neutral compression: mean ratio ${fmt(meanCompression)} (entries ~same similarity as chunks)`);
|
|
256
|
+
}
|
|
257
|
+
if (overallMRR > 0.8) {
|
|
258
|
+
summary.push(`Strong discrimination: MRR=${fmt(overallMRR)}, hit rate=${fmt(overallHitRate * 100, 1)}% — entries are distinct enough within clusters`);
|
|
259
|
+
}
|
|
260
|
+
else if (overallMRR > 0.5) {
|
|
261
|
+
summary.push(`Moderate discrimination: MRR=${fmt(overallMRR)}, hit rate=${fmt(overallHitRate * 100, 1)}% — some cluster confusion, refinement may help`);
|
|
262
|
+
}
|
|
263
|
+
else {
|
|
264
|
+
summary.push(`Weak discrimination: MRR=${fmt(overallMRR)}, hit rate=${fmt(overallHitRate * 100, 1)}% — entries are too similar within clusters, refinement needed`);
|
|
265
|
+
}
|
|
266
|
+
// Alignment summary
|
|
267
|
+
if (overallGap > 0.05) {
|
|
268
|
+
summary.push(`Good alignment: gap=${fmt(overallGap)} (self=${fmt(overallSelfAlign)}, sibling=${fmt(overallSibAlign)}), ${fmt(overallUniquelyAligned * 100, 1)}% uniquely aligned — entries preserve chunk-specific signal`);
|
|
269
|
+
}
|
|
270
|
+
else if (overallGap > 0) {
|
|
271
|
+
summary.push(`Weak alignment: gap=${fmt(overallGap)} (self=${fmt(overallSelfAlign)}, sibling=${fmt(overallSibAlign)}), ${fmt(overallUniquelyAligned * 100, 1)}% uniquely aligned — LLM summarization drifts toward cluster-generic direction`);
|
|
272
|
+
}
|
|
273
|
+
else {
|
|
274
|
+
summary.push(`No alignment: gap=${fmt(overallGap)} — entries are not aligned to their source chunks, retrieval via chunk query will fail`);
|
|
275
|
+
}
|
|
276
|
+
if (refinementSection) {
|
|
277
|
+
if (refinementSection.meanMRRDelta > 0.05) {
|
|
278
|
+
summary.push(`Refinement helps: +${fmt(refinementSection.meanMRRDelta)} MRR on worst clusters — worth implementing`);
|
|
279
|
+
}
|
|
280
|
+
else if (refinementSection.meanMRRDelta > 0) {
|
|
281
|
+
summary.push(`Refinement marginal: +${fmt(refinementSection.meanMRRDelta)} MRR — may not justify the LLM cost`);
|
|
282
|
+
}
|
|
283
|
+
else {
|
|
284
|
+
summary.push(`Refinement unhelpful: ${fmt(refinementSection.meanMRRDelta)} MRR — cluster-aware regeneration doesn't improve discrimination`);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
console.log('\n══ Summary ══\n');
|
|
288
|
+
for (const line of summary) {
|
|
289
|
+
console.log(` • ${line}`);
|
|
290
|
+
}
|
|
291
|
+
const report = {
|
|
292
|
+
timestamp: new Date().toISOString(),
|
|
293
|
+
totalEntries,
|
|
294
|
+
eligibleClusters: clusters.length,
|
|
295
|
+
analysedClusters: clusters.length,
|
|
296
|
+
similarity: {
|
|
297
|
+
results: similarityResults,
|
|
298
|
+
meanCompressionRatio: meanCompression,
|
|
299
|
+
homogenizedFraction: homogenized / similarityResults.length,
|
|
300
|
+
},
|
|
301
|
+
discrimination: {
|
|
302
|
+
results: discriminationResults,
|
|
303
|
+
overallMRR,
|
|
304
|
+
overallHitRate,
|
|
305
|
+
},
|
|
306
|
+
alignment: {
|
|
307
|
+
results: alignmentResults.map((r) => ({
|
|
308
|
+
clusterId: r.clusterId,
|
|
309
|
+
clusterName: r.clusterName,
|
|
310
|
+
entryCount: r.entryCount,
|
|
311
|
+
meanSelfAlignment: r.meanSelfAlignment,
|
|
312
|
+
meanSiblingAlignment: r.meanSiblingAlignment,
|
|
313
|
+
meanAlignmentGap: r.meanAlignmentGap,
|
|
314
|
+
uniquelyAlignedFraction: r.uniquelyAlignedFraction,
|
|
315
|
+
})),
|
|
316
|
+
overallSelfAlignment: overallSelfAlign,
|
|
317
|
+
overallSiblingAlignment: overallSibAlign,
|
|
318
|
+
overallAlignmentGap: overallGap,
|
|
319
|
+
overallUniquelyAligned: overallUniquelyAligned,
|
|
320
|
+
},
|
|
321
|
+
refinement: refinementSection,
|
|
322
|
+
summary,
|
|
323
|
+
};
|
|
324
|
+
return report;
|
|
325
|
+
}
|
|
326
|
+
// ── CLI entrypoint ──────────────────────────────────────────────────────────
|
|
327
|
+
import { writeFileSync } from 'fs';
|
|
328
|
+
runExperiment()
|
|
329
|
+
.then((report) => {
|
|
330
|
+
const outPath = 'index-differentiation-report.json';
|
|
331
|
+
writeFileSync(outPath, JSON.stringify(report, null, 2));
|
|
332
|
+
console.log(`\nReport written to ${outPath}`);
|
|
333
|
+
})
|
|
334
|
+
.catch((err) => {
|
|
335
|
+
console.error('Experiment failed:', err);
|
|
336
|
+
process.exit(1);
|
|
337
|
+
});
|
|
338
|
+
//# sourceMappingURL=run-experiment.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-experiment.js","sourceRoot":"","sources":["../../../../src/eval/experiments/index-differentiation/run-experiment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACvF,OAAO,EACL,uBAAuB,EACvB,kBAAkB,GACnB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAG/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAGzD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC;;;GAGG;AACH,KAAK,UAAU,wBAAwB;IAKrC,0DAA0D;IAC1D,KAAK,EAAE,CAAC;IAER,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,qBAAqB,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;IAEvD,sBAAsB;IACtB,MAAM,YAAY,GAAG,MAAM,WAAW,CAAC,aAAa,EAAE,CAAC;IACvD,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,aAAa,EAAE,CAAC;IAE5D,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAE1E,OAAO,CAAC,GAAG,CAAC,oBAAoB,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,oBAAoB,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;IAEpD,MAAM,QAAQ,GAAyB,EAAE,CAAC;IAC1C,MAAM,UAAU,GAAiB,EAAE,CAAC;IACpC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEhD,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAEhD,gDAAgD;QAChD,MAAM,cAAc,GAAkC,EAAE,CAAC;QACzD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YACjD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBAAE,SAAS;gBACzC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAE3B,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjD,IAAI,CAAC,cAAc;oBAAE,SAAS;gBAE9B,sCAAsC;gBACtC,MAAM,eAAe,GAAe,EAAE,CAAC;gBACvC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;oBACjC,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACjC,IAAI,GAAG;wBAAE,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAEnC,6CAA6C;oBAC7C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC5B,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;wBAChC,IAAI,KAAK;4BAAE,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;oBACnD,CAAC;gBACH,CAAC;gBAED,cAAc,CAAC,IAAI,CAAC;oBAClB,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,cAAc;oBACd,eAAe;iBAChB,CAAC,CAAC;gBAEH,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,IAAI,cAAc,CAAC,MAAM,IAAI,uBAAuB,EAAE,CAAC;YACrD,QAAQ,CAAC,IAAI,CAAC;gBACZ,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,OAAO,EAAE,cAAc;aACxB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;AACjD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,CAAS,EAAE,QAAQ,GAAG,CAAC;IAClC,OAAO,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa;IAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,cAAc;QAChC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;IAEhE,MAAM,YAAY,GAAG,kBAAkB,EAAE,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,wBAAwB,YAAY,EAAE,CAAC,CAAC;IAEpD,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CACT,+FAA+F,CAChG,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,gBAAgB;IAChB,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAC9C,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,GAAG,MAAM,wBAAwB,EAAE,CAAC;IAEjF,OAAO,CAAC,GAAG,CAAC,yBAAyB,uBAAuB,cAAc,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7F,OAAO,CAAC,GAAG,CACT,yCAAyC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAC9F,CAAC;IAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,CACT,yDAAyD;YACvD,GAAG,uBAAuB,wBAAwB;YAClD,oCAAoC,CACvC,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,6EAA6E;IAE7E,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAE1D,MAAM,eAAe,GACnB,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC7D,iBAAiB,CAAC,MAAM,CAAC;IAC3B,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAC1C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,GAAG,GAAG,CAChC,CAAC,MAAM,CAAC;IAET,OAAO,CAAC,GAAG,CAAC,yEAAyE,CAAC,CAAC;IACvF,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC;QACvG,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,KAAK,EAAE,CAC1H,CAAC;IACJ,CAAC;IACD,IAAI,iBAAiB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,aAAa,iBAAiB,CAAC,MAAM,GAAG,EAAE,gBAAgB,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,+BAA+B,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CACT,uCAAuC,WAAW,IAAI,iBAAiB,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAC9I,CAAC;IAEF,6EAA6E;IAE7E,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;IACtD,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAE9D,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,CAAC,IAAI,qBAAqB,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;QACvE,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACxE,UAAU,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;IAClC,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,GAAG,UAAU,CAAC;IACxC,MAAM,cAAc,GAAG,SAAS,GAAG,UAAU,CAAC;IAE9C,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,qBAAqB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAC9G,CAAC;IACJ,CAAC;IACD,IAAI,qBAAqB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,aAAa,qBAAqB,CAAC,MAAM,GAAG,EAAE,gBAAgB,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,uBAAuB,GAAG,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAEpE,4EAA4E;IAE5E,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IACrE,MAAM,gBAAgB,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAExD,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,KAAK,MAAM,CAAC,IAAI,gBAAgB,EAAE,CAAC;QACjC,MAAM,UAAU,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QACxE,cAAc,IAAI,CAAC,CAAC,iBAAiB,GAAG,UAAU,CAAC;QACnD,aAAa,IAAI,CAAC,CAAC,oBAAoB,GAAG,UAAU,CAAC;QACrD,QAAQ,IAAI,CAAC,CAAC,gBAAgB,GAAG,UAAU,CAAC;QAC5C,WAAW,IAAI,CAAC,CAAC,uBAAuB,GAAG,UAAU,CAAC;QACtD,iBAAiB,IAAI,UAAU,CAAC;IAClC,CAAC;IAED,MAAM,gBAAgB,GAAG,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;IACxF,MAAM,eAAe,GAAG,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;IACtF,MAAM,UAAU,GAAG,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,sBAAsB,GAAG,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;IAE3F,OAAO,CAAC,GAAG,CAAC,4EAA4E,CAAC,CAAC;IAC1F,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,uBAAuB,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CACzL,CAAC;IACJ,CAAC;IACD,IAAI,gBAAgB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,aAAa,gBAAgB,CAAC,MAAM,GAAG,EAAE,gBAAgB,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,qDAAqD,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;IAC1F,OAAO,CAAC,GAAG,CAAC,uDAAuD,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,CAAC,mDAAmD,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,CAAC,mDAAmD,GAAG,CAAC,sBAAsB,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAExG,6EAA6E;IAE7E,IAAI,iBAAkE,CAAC;IAEvE,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,+CAA+C,WAAW,uBAAuB,CAAC,CAAC;QAE/F,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAClC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CACtE,CAAC;QAEF,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAC/C,QAAQ,EACR,UAAU,EACV,aAAa,EACb,oBAAoB,EACpB,WAAW,CACZ,CAAC;QAEF,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,MAAM,YAAY,GAChB,iBAAiB,CAAC,MAAM,CACtB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,WAAW,CAAC,EAC5C,CAAC,CACF,GAAG,iBAAiB,CAAC,MAAM,CAAC;YAC/B,MAAM,WAAW,GACf,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,qBAAqB,EAAE,CAAC,CAAC;gBAClE,iBAAiB,CAAC,MAAM,CAAC;YAE3B,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;YAChF,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YACnC,KAAK,MAAM,CAAC,IAAI,iBAAiB,EAAE,CAAC;gBAClC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACpE,MAAM,KAAK,GAAG,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,WAAW,CAAC;gBAC3C,MAAM,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;gBAChE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,KAAK,EAAE,CAClH,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,uBAAuB,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YACtF,OAAO,CAAC,GAAG,CAAC,mCAAmC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAEhG,0BAA0B;YAC1B,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACpF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;gBACvC,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;oBACxC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;oBAC5D,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;wBAChD,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;wBAC7D,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;wBAC5D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAClB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,iBAAiB,GAAG;gBAClB,OAAO,EAAE,iBAAiB;gBAC1B,YAAY;gBACZ,yBAAyB,EAAE,WAAW;aACvC,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;IAC7E,CAAC;IAED,6EAA6E;IAE7E,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,OAAO,CAAC,IAAI,CACV,YAAY,QAAQ,CAAC,MAAM,cAAc,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CACvG,CAAC;IAEF,IAAI,eAAe,GAAG,IAAI,EAAE,CAAC;QAC3B,OAAO,CAAC,IAAI,CACV,uCAAuC,GAAG,CAAC,eAAe,CAAC,aAAa,GAAG,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,6BAA6B,CACzI,CAAC;IACJ,CAAC;SAAM,IAAI,eAAe,GAAG,IAAI,EAAE,CAAC;QAClC,OAAO,CAAC,IAAI,CACV,4CAA4C,GAAG,CAAC,eAAe,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,GAAG,eAAe,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,6BAA6B,CAC9I,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CACV,mCAAmC,GAAG,CAAC,eAAe,CAAC,uCAAuC,CAC/F,CAAC;IACJ,CAAC;IAED,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CACV,8BAA8B,GAAG,CAAC,UAAU,CAAC,cAAc,GAAG,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC,CAAC,iDAAiD,CACzI,CAAC;IACJ,CAAC;SAAM,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;QAC5B,OAAO,CAAC,IAAI,CACV,gCAAgC,GAAG,CAAC,UAAU,CAAC,cAAc,GAAG,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC,CAAC,iDAAiD,CAC3I,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CACV,4BAA4B,GAAG,CAAC,UAAU,CAAC,cAAc,GAAG,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC,CAAC,gEAAgE,CACtJ,CAAC;IACJ,CAAC;IAED,oBAAoB;IACpB,IAAI,UAAU,GAAG,IAAI,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CACV,uBAAuB,GAAG,CAAC,UAAU,CAAC,UAAU,GAAG,CAAC,gBAAgB,CAAC,aAAa,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,sBAAsB,GAAG,GAAG,EAAE,CAAC,CAAC,6DAA6D,CAC9M,CAAC;IACJ,CAAC;SAAM,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,IAAI,CACV,uBAAuB,GAAG,CAAC,UAAU,CAAC,UAAU,GAAG,CAAC,gBAAgB,CAAC,aAAa,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,sBAAsB,GAAG,GAAG,EAAE,CAAC,CAAC,gFAAgF,CACjO,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CACV,qBAAqB,GAAG,CAAC,UAAU,CAAC,wFAAwF,CAC7H,CAAC;IACJ,CAAC;IAED,IAAI,iBAAiB,EAAE,CAAC;QACtB,IAAI,iBAAiB,CAAC,YAAY,GAAG,IAAI,EAAE,CAAC;YAC1C,OAAO,CAAC,IAAI,CACV,sBAAsB,GAAG,CAAC,iBAAiB,CAAC,YAAY,CAAC,6CAA6C,CACvG,CAAC;QACJ,CAAC;aAAM,IAAI,iBAAiB,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CACV,yBAAyB,GAAG,CAAC,iBAAiB,CAAC,YAAY,CAAC,qCAAqC,CAClG,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CACV,yBAAyB,GAAG,CAAC,iBAAiB,CAAC,YAAY,CAAC,kEAAkE,CAC/H,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,MAAM,GAA0B;QACpC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,YAAY;QACZ,gBAAgB,EAAE,QAAQ,CAAC,MAAM;QACjC,gBAAgB,EAAE,QAAQ,CAAC,MAAM;QACjC,UAAU,EAAE;YACV,OAAO,EAAE,iBAAiB;YAC1B,oBAAoB,EAAE,eAAe;YACrC,mBAAmB,EAAE,WAAW,GAAG,iBAAiB,CAAC,MAAM;SAC5D;QACD,cAAc,EAAE;YACd,OAAO,EAAE,qBAAqB;YAC9B,UAAU;YACV,cAAc;SACf;QACD,SAAS,EAAE;YACT,OAAO,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACpC,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,iBAAiB,EAAE,CAAC,CAAC,iBAAiB;gBACtC,oBAAoB,EAAE,CAAC,CAAC,oBAAoB;gBAC5C,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;gBACpC,uBAAuB,EAAE,CAAC,CAAC,uBAAuB;aACnD,CAAC,CAAC;YACH,oBAAoB,EAAE,gBAAgB;YACtC,uBAAuB,EAAE,eAAe;YACxC,mBAAmB,EAAE,UAAU;YAC/B,sBAAsB,EAAE,sBAAsB;SAC/C;QACD,UAAU,EAAE,iBAAiB;QAC7B,OAAO;KACR,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+EAA+E;AAE/E,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEnC,aAAa,EAAE;KACZ,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;IACf,MAAM,OAAO,GAAG,mCAAmC,CAAC;IACpD,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,uBAAuB,OAAO,EAAE,CAAC,CAAC;AAChD,CAAC,CAAC;KACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACb,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;IACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1: Intra-cluster similarity analysis.
|
|
3
|
+
*
|
|
4
|
+
* For each cluster with 3+ index entries, compute:
|
|
5
|
+
* - Mean pairwise cosine similarity between index entry embeddings
|
|
6
|
+
* - Mean pairwise cosine similarity between raw chunk embeddings
|
|
7
|
+
* - Compression ratio (entry_sim / chunk_sim)
|
|
8
|
+
*
|
|
9
|
+
* If ratio > 1, the LLM compression homogenized the entries (bad for discrimination).
|
|
10
|
+
* If ratio < 1, the LLM naturally differentiates entries (good).
|
|
11
|
+
*/
|
|
12
|
+
import type { ClusterSimilarityResult } from './types.js';
|
|
13
|
+
/** Input: a cluster ready for analysis, with matched embeddings. */
|
|
14
|
+
export interface ClusterForAnalysis {
|
|
15
|
+
clusterId: string;
|
|
16
|
+
clusterName: string | null;
|
|
17
|
+
entries: Array<{
|
|
18
|
+
entryId: string;
|
|
19
|
+
entryEmbedding: number[];
|
|
20
|
+
chunkEmbeddings: number[][];
|
|
21
|
+
}>;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Analyse similarity for a single cluster.
|
|
25
|
+
*/
|
|
26
|
+
export declare function analyseCluster(cluster: ClusterForAnalysis): ClusterSimilarityResult;
|
|
27
|
+
/**
|
|
28
|
+
* Run similarity analysis across all eligible clusters.
|
|
29
|
+
*/
|
|
30
|
+
export declare function runSimilarityAnalysis(clusters: ClusterForAnalysis[]): ClusterSimilarityResult[];
|
|
31
|
+
//# sourceMappingURL=similarity-analysis.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"similarity-analysis.d.ts","sourceRoot":"","sources":["../../../../src/eval/experiments/index-differentiation/similarity-analysis.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAE1D,oEAAoE;AACpE,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,KAAK,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,eAAe,EAAE,MAAM,EAAE,EAAE,CAAC;KAC7B,CAAC,CAAC;CACJ;AAqBD;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,kBAAkB,GAAG,uBAAuB,CAwBnF;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,kBAAkB,EAAE,GAC7B,uBAAuB,EAAE,CAE3B"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 1: Intra-cluster similarity analysis.
|
|
3
|
+
*
|
|
4
|
+
* For each cluster with 3+ index entries, compute:
|
|
5
|
+
* - Mean pairwise cosine similarity between index entry embeddings
|
|
6
|
+
* - Mean pairwise cosine similarity between raw chunk embeddings
|
|
7
|
+
* - Compression ratio (entry_sim / chunk_sim)
|
|
8
|
+
*
|
|
9
|
+
* If ratio > 1, the LLM compression homogenized the entries (bad for discrimination).
|
|
10
|
+
* If ratio < 1, the LLM naturally differentiates entries (good).
|
|
11
|
+
*/
|
|
12
|
+
import { cosineSimilarity } from '../../../utils/angular-distance.js';
|
|
13
|
+
/**
|
|
14
|
+
* Compute pairwise cosine similarity statistics for a set of vectors.
|
|
15
|
+
* Returns mean and standard deviation.
|
|
16
|
+
*/
|
|
17
|
+
function pairwiseStats(vectors) {
|
|
18
|
+
if (vectors.length < 2)
|
|
19
|
+
return { mean: 1.0, stdDev: 0 };
|
|
20
|
+
const sims = [];
|
|
21
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
22
|
+
for (let j = i + 1; j < vectors.length; j++) {
|
|
23
|
+
sims.push(cosineSimilarity(vectors[i], vectors[j]));
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
const mean = sims.reduce((a, b) => a + b, 0) / sims.length;
|
|
27
|
+
const variance = sims.reduce((a, b) => a + (b - mean) ** 2, 0) / sims.length;
|
|
28
|
+
return { mean, stdDev: Math.sqrt(variance) };
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Analyse similarity for a single cluster.
|
|
32
|
+
*/
|
|
33
|
+
export function analyseCluster(cluster) {
|
|
34
|
+
const entryEmbeddings = cluster.entries.map((e) => e.entryEmbedding);
|
|
35
|
+
const entryStats = pairwiseStats(entryEmbeddings);
|
|
36
|
+
// For chunk comparison, take the first chunk embedding per entry
|
|
37
|
+
// (entries are 1:1 with chunks currently)
|
|
38
|
+
const chunkEmbeddings = cluster.entries
|
|
39
|
+
.flatMap((e) => e.chunkEmbeddings)
|
|
40
|
+
.filter((e) => e.length > 0);
|
|
41
|
+
const chunkStats = pairwiseStats(chunkEmbeddings);
|
|
42
|
+
const compressionRatio = chunkStats.mean > 0 ? entryStats.mean / chunkStats.mean : 1.0;
|
|
43
|
+
return {
|
|
44
|
+
clusterId: cluster.clusterId,
|
|
45
|
+
clusterName: cluster.clusterName,
|
|
46
|
+
entryCount: cluster.entries.length,
|
|
47
|
+
chunkCount: chunkEmbeddings.length,
|
|
48
|
+
meanEntryPairSim: entryStats.mean,
|
|
49
|
+
meanChunkPairSim: chunkStats.mean,
|
|
50
|
+
compressionRatio,
|
|
51
|
+
entrySimStdDev: entryStats.stdDev,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Run similarity analysis across all eligible clusters.
|
|
56
|
+
*/
|
|
57
|
+
export function runSimilarityAnalysis(clusters) {
|
|
58
|
+
return clusters.map(analyseCluster);
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=similarity-analysis.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"similarity-analysis.js","sourceRoot":"","sources":["../../../../src/eval/experiments/index-differentiation/similarity-analysis.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AActE;;;GAGG;AACH,SAAS,aAAa,CAAC,OAAmB;IACxC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAExD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;AAC/C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAA2B;IACxD,MAAM,eAAe,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,aAAa,CAAC,eAAe,CAAC,CAAC;IAElD,iEAAiE;IACjE,0CAA0C;IAC1C,MAAM,eAAe,GAAG,OAAO,CAAC,OAAO;SACpC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;SACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,MAAM,UAAU,GAAG,aAAa,CAAC,eAAe,CAAC,CAAC;IAElD,MAAM,gBAAgB,GACpB,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAEhE,OAAO;QACL,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;QAClC,UAAU,EAAE,eAAe,CAAC,MAAM;QAClC,gBAAgB,EAAE,UAAU,CAAC,IAAI;QACjC,gBAAgB,EAAE,UAAU,CAAC,IAAI;QACjC,gBAAgB;QAChB,cAAc,EAAE,UAAU,CAAC,MAAM;KAClC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,QAA8B;IAE9B,OAAO,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;AACtC,CAAC"}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for the index entry differentiation experiment.
|
|
3
|
+
*
|
|
4
|
+
* Tests whether semantic index entries within the same cluster are
|
|
5
|
+
* too similar, and whether cluster-aware refinement improves discrimination.
|
|
6
|
+
*/
|
|
7
|
+
/** Per-cluster similarity analysis. */
|
|
8
|
+
export interface ClusterSimilarityResult {
|
|
9
|
+
clusterId: string;
|
|
10
|
+
clusterName: string | null;
|
|
11
|
+
entryCount: number;
|
|
12
|
+
chunkCount: number;
|
|
13
|
+
/** Mean pairwise cosine similarity between index entry embeddings. */
|
|
14
|
+
meanEntryPairSim: number;
|
|
15
|
+
/** Mean pairwise cosine similarity between raw chunk embeddings. */
|
|
16
|
+
meanChunkPairSim: number;
|
|
17
|
+
/**
|
|
18
|
+
* Ratio of entry similarity to chunk similarity.
|
|
19
|
+
* > 1 = entries more similar (compression homogenized)
|
|
20
|
+
* < 1 = entries more different (LLM differentiates naturally)
|
|
21
|
+
* = 1 = no change
|
|
22
|
+
*/
|
|
23
|
+
compressionRatio: number;
|
|
24
|
+
/** Standard deviation of pairwise entry similarities. */
|
|
25
|
+
entrySimStdDev: number;
|
|
26
|
+
}
|
|
27
|
+
/** Discrimination test: can we find the right entry among cluster siblings? */
|
|
28
|
+
export interface DiscriminationResult {
|
|
29
|
+
clusterId: string;
|
|
30
|
+
clusterName: string | null;
|
|
31
|
+
entryCount: number;
|
|
32
|
+
/** Mean reciprocal rank for finding the correct entry when querying by its description embedding. */
|
|
33
|
+
meanReciprocalRank: number;
|
|
34
|
+
/** Fraction of entries where correct entry was rank 1 among cluster siblings. */
|
|
35
|
+
hitRate: number;
|
|
36
|
+
/** Per-entry discrimination scores (for analysis). */
|
|
37
|
+
perEntry: Array<{
|
|
38
|
+
entryId: string;
|
|
39
|
+
rankAmongSiblings: number;
|
|
40
|
+
correctSimilarity: number;
|
|
41
|
+
bestSiblingSimilarity: number;
|
|
42
|
+
}>;
|
|
43
|
+
}
|
|
44
|
+
/** Refinement comparison: baseline vs cluster-aware regeneration. */
|
|
45
|
+
export interface RefinementResult {
|
|
46
|
+
clusterId: string;
|
|
47
|
+
clusterName: string | null;
|
|
48
|
+
entryCount: number;
|
|
49
|
+
baselineMRR: number;
|
|
50
|
+
refinedMRR: number;
|
|
51
|
+
baselineHitRate: number;
|
|
52
|
+
refinedHitRate: number;
|
|
53
|
+
/** How much compression ratio changed (negative = more differentiated). */
|
|
54
|
+
compressionRatioDelta: number;
|
|
55
|
+
/** Sample refined descriptions for inspection. */
|
|
56
|
+
sampleRefinements: Array<{
|
|
57
|
+
entryId: string;
|
|
58
|
+
original: string;
|
|
59
|
+
refined: string;
|
|
60
|
+
}>;
|
|
61
|
+
}
|
|
62
|
+
/** Full experiment report. */
|
|
63
|
+
export interface DifferentiationReport {
|
|
64
|
+
timestamp: string;
|
|
65
|
+
/** Total index entries in the system. */
|
|
66
|
+
totalEntries: number;
|
|
67
|
+
/** Total clusters with 3+ index entries (experiment-eligible). */
|
|
68
|
+
eligibleClusters: number;
|
|
69
|
+
/** Total clusters analysed. */
|
|
70
|
+
analysedClusters: number;
|
|
71
|
+
similarity: {
|
|
72
|
+
results: ClusterSimilarityResult[];
|
|
73
|
+
/** Mean compression ratio across all eligible clusters. */
|
|
74
|
+
meanCompressionRatio: number;
|
|
75
|
+
/** Fraction of clusters where entries are more similar than chunks. */
|
|
76
|
+
homogenizedFraction: number;
|
|
77
|
+
};
|
|
78
|
+
discrimination: {
|
|
79
|
+
results: DiscriminationResult[];
|
|
80
|
+
/** Overall MRR across all entries in all clusters. */
|
|
81
|
+
overallMRR: number;
|
|
82
|
+
/** Overall hit rate (correct entry at rank 1). */
|
|
83
|
+
overallHitRate: number;
|
|
84
|
+
};
|
|
85
|
+
alignment?: {
|
|
86
|
+
results: Array<{
|
|
87
|
+
clusterId: string;
|
|
88
|
+
clusterName: string | null;
|
|
89
|
+
entryCount: number;
|
|
90
|
+
meanSelfAlignment: number;
|
|
91
|
+
meanSiblingAlignment: number;
|
|
92
|
+
meanAlignmentGap: number;
|
|
93
|
+
uniquelyAlignedFraction: number;
|
|
94
|
+
}>;
|
|
95
|
+
/** Overall mean self-alignment (entry ↔ own chunk). */
|
|
96
|
+
overallSelfAlignment: number;
|
|
97
|
+
/** Overall mean sibling alignment (entry ↔ other chunks). */
|
|
98
|
+
overallSiblingAlignment: number;
|
|
99
|
+
/** Overall mean alignment gap (self - sibling). */
|
|
100
|
+
overallAlignmentGap: number;
|
|
101
|
+
/** Overall fraction of entries uniquely aligned to own chunk. */
|
|
102
|
+
overallUniquelyAligned: number;
|
|
103
|
+
};
|
|
104
|
+
refinement?: {
|
|
105
|
+
results: RefinementResult[];
|
|
106
|
+
/** Mean MRR improvement from refinement. */
|
|
107
|
+
meanMRRDelta: number;
|
|
108
|
+
/** Mean compression ratio change. */
|
|
109
|
+
meanCompressionRatioDelta: number;
|
|
110
|
+
};
|
|
111
|
+
/** Human-readable summary. */
|
|
112
|
+
summary: string[];
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/eval/experiments/index-differentiation/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,uCAAuC;AACvC,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IAEnB,sEAAsE;IACtE,gBAAgB,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;;OAKG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,+EAA+E;AAC/E,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,qGAAqG;IACrG,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC;IAChB,sDAAsD;IACtD,QAAQ,EAAE,KAAK,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,qBAAqB,EAAE,MAAM,CAAC;KAC/B,CAAC,CAAC;CACJ;AAED,qEAAqE;AACrE,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,2EAA2E;IAC3E,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kDAAkD;IAClD,iBAAiB,EAAE,KAAK,CAAC;QACvB,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,8BAA8B;AAC9B,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IAGzB,UAAU,EAAE;QACV,OAAO,EAAE,uBAAuB,EAAE,CAAC;QACnC,2DAA2D;QAC3D,oBAAoB,EAAE,MAAM,CAAC;QAC7B,uEAAuE;QACvE,mBAAmB,EAAE,MAAM,CAAC;KAC7B,CAAC;IAGF,cAAc,EAAE;QACd,OAAO,EAAE,oBAAoB,EAAE,CAAC;QAChC,sDAAsD;QACtD,UAAU,EAAE,MAAM,CAAC;QACnB,kDAAkD;QAClD,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IAGF,SAAS,CAAC,EAAE;QACV,OAAO,EAAE,KAAK,CAAC;YACb,SAAS,EAAE,MAAM,CAAC;YAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;YAC3B,UAAU,EAAE,MAAM,CAAC;YACnB,iBAAiB,EAAE,MAAM,CAAC;YAC1B,oBAAoB,EAAE,MAAM,CAAC;YAC7B,gBAAgB,EAAE,MAAM,CAAC;YACzB,uBAAuB,EAAE,MAAM,CAAC;SACjC,CAAC,CAAC;QACH,uDAAuD;QACvD,oBAAoB,EAAE,MAAM,CAAC;QAC7B,6DAA6D;QAC7D,uBAAuB,EAAE,MAAM,CAAC;QAChC,mDAAmD;QACnD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,iEAAiE;QACjE,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IAGF,UAAU,CAAC,EAAE;QACX,OAAO,EAAE,gBAAgB,EAAE,CAAC;QAC5B,4CAA4C;QAC5C,YAAY,EAAE,MAAM,CAAC;QACrB,qCAAqC;QACrC,yBAAyB,EAAE,MAAM,CAAC;KACnC,CAAC;IAEF,8BAA8B;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for the index entry differentiation experiment.
|
|
3
|
+
*
|
|
4
|
+
* Tests whether semantic index entries within the same cluster are
|
|
5
|
+
* too similar, and whether cluster-aware refinement improves discrimination.
|
|
6
|
+
*/
|
|
7
|
+
export {};
|
|
8
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/eval/experiments/index-differentiation/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jeopardy vs Summary Index Entry Comparison
|
|
3
|
+
*
|
|
4
|
+
* A/B test comparing two index entry generation strategies:
|
|
5
|
+
* A) Summary-style: "This chunk discusses configuring ESLint..."
|
|
6
|
+
* B) Jeopardy-style: "How to fix ESLint module resolution with TS path aliases?"
|
|
7
|
+
*
|
|
8
|
+
* For a sample of chunks:
|
|
9
|
+
* 1. Gather existing summary entries + embeddings
|
|
10
|
+
* 2. Generate Jeopardy entries + embed them
|
|
11
|
+
* 3. Generate natural language search queries (independent ground truth)
|
|
12
|
+
* 4. For each query, find the ground truth chunk among all sample entries
|
|
13
|
+
* 5. Compare recall: which entry style is found more often?
|
|
14
|
+
*
|
|
15
|
+
* Usage:
|
|
16
|
+
* npx tsx src/eval/experiments/index-vs-chunk/jeopardy-experiment.ts [--sample-size N]
|
|
17
|
+
*/
|
|
18
|
+
export {};
|
|
19
|
+
//# sourceMappingURL=jeopardy-experiment.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jeopardy-experiment.d.ts","sourceRoot":"","sources":["../../../../src/eval/experiments/index-vs-chunk/jeopardy-experiment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG"}
|