@ruvector/ruvllm 0.2.4 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +370 -286
- package/bin/cli.js +619 -2
- package/dist/cjs/benchmarks/embedding-benchmark.d.ts +102 -0
- package/dist/cjs/benchmarks/embedding-benchmark.d.ts.map +1 -0
- package/dist/cjs/benchmarks/embedding-benchmark.js +436 -0
- package/dist/cjs/benchmarks/index.d.ts +34 -0
- package/dist/cjs/benchmarks/index.d.ts.map +1 -0
- package/dist/cjs/benchmarks/index.js +132 -0
- package/dist/cjs/benchmarks/model-comparison.d.ts +71 -0
- package/dist/cjs/benchmarks/model-comparison.d.ts.map +1 -0
- package/dist/cjs/benchmarks/model-comparison.js +476 -0
- package/dist/cjs/benchmarks/routing-benchmark.d.ts +70 -0
- package/dist/cjs/benchmarks/routing-benchmark.d.ts.map +1 -0
- package/dist/cjs/benchmarks/routing-benchmark.js +289 -0
- package/dist/cjs/contrastive.d.ts +229 -0
- package/dist/cjs/contrastive.d.ts.map +1 -0
- package/dist/cjs/contrastive.js +589 -0
- package/dist/cjs/engine.d.ts +3 -0
- package/dist/cjs/engine.d.ts.map +1 -1
- package/dist/cjs/engine.js +26 -6
- package/dist/cjs/index.d.ts +4 -0
- package/dist/cjs/index.d.ts.map +1 -1
- package/dist/cjs/index.js +9 -1
- package/dist/cjs/models.d.ts +129 -0
- package/dist/cjs/models.d.ts.map +1 -0
- package/dist/cjs/models.js +323 -0
- package/dist/cjs/native.d.ts +3 -3
- package/dist/cjs/native.d.ts.map +1 -1
- package/dist/cjs/native.js +1 -1
- package/dist/cjs/rlm/controller.d.ts +245 -0
- package/dist/cjs/rlm/controller.d.ts.map +1 -0
- package/dist/cjs/rlm/controller.js +554 -0
- package/dist/cjs/rlm/index.d.ts +62 -0
- package/dist/cjs/rlm/index.d.ts.map +1 -0
- package/dist/cjs/rlm/index.js +97 -0
- package/dist/cjs/rlm/training.d.ts +333 -0
- package/dist/cjs/rlm/training.d.ts.map +1 -0
- package/dist/cjs/rlm/training.js +680 -0
- package/dist/cjs/rlm/types.d.ts +189 -0
- package/dist/cjs/rlm/types.d.ts.map +1 -0
- package/dist/cjs/rlm/types.js +10 -0
- package/dist/esm/benchmarks/embedding-benchmark.d.ts +102 -0
- package/dist/esm/benchmarks/embedding-benchmark.d.ts.map +1 -0
- package/dist/esm/benchmarks/embedding-benchmark.js +427 -0
- package/dist/esm/benchmarks/index.d.ts +34 -0
- package/dist/esm/benchmarks/index.d.ts.map +1 -0
- package/dist/esm/benchmarks/index.js +110 -0
- package/dist/esm/benchmarks/model-comparison.d.ts +71 -0
- package/dist/esm/benchmarks/model-comparison.d.ts.map +1 -0
- package/dist/esm/benchmarks/model-comparison.js +434 -0
- package/dist/esm/benchmarks/routing-benchmark.d.ts +70 -0
- package/dist/esm/benchmarks/routing-benchmark.d.ts.map +1 -0
- package/dist/esm/benchmarks/routing-benchmark.js +283 -0
- package/dist/esm/contrastive.d.ts +229 -0
- package/dist/esm/contrastive.d.ts.map +1 -0
- package/dist/esm/contrastive.js +578 -0
- package/dist/esm/engine.d.ts +3 -0
- package/dist/esm/engine.d.ts.map +1 -1
- package/dist/esm/engine.js +26 -6
- package/dist/esm/index.d.ts +4 -0
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +9 -1
- package/dist/esm/models.d.ts +129 -0
- package/dist/esm/models.d.ts.map +1 -0
- package/dist/esm/models.js +282 -0
- package/dist/esm/native.d.ts +3 -3
- package/dist/esm/native.d.ts.map +1 -1
- package/dist/esm/native.js +1 -1
- package/dist/esm/rlm/controller.d.ts +245 -0
- package/dist/esm/rlm/controller.d.ts.map +1 -0
- package/dist/esm/rlm/controller.js +550 -0
- package/dist/esm/rlm/index.d.ts +62 -0
- package/dist/esm/rlm/index.d.ts.map +1 -0
- package/dist/esm/rlm/index.js +71 -0
- package/dist/esm/rlm/training.d.ts +333 -0
- package/dist/esm/rlm/training.d.ts.map +1 -0
- package/dist/esm/rlm/training.js +672 -0
- package/dist/esm/rlm/types.d.ts +189 -0
- package/dist/esm/rlm/types.d.ts.map +1 -0
- package/dist/esm/rlm/types.js +9 -0
- package/package.json +13 -8
- package/scripts/ensemble-model-compare.js +287 -0
- package/scripts/huggingface/README.md +537 -0
- package/scripts/huggingface/publish.sh +112 -0
- package/scripts/hybrid-model-compare.js +373 -0
- package/scripts/improved-model-compare.js +288 -0
- package/scripts/optimized-model-compare.js +364 -0
- package/scripts/real-model-compare.js +280 -0
- package/scripts/training/agentic-flow-capabilities.json +329 -0
- package/scripts/training/claude-code-synth.js +505 -0
- package/scripts/training/claude-flow-capabilities.json +1322 -0
- package/scripts/training/claude-hard-negatives.js +222 -0
- package/scripts/training/contrastive-finetune.js +517 -0
- package/scripts/training/ecosystem-triplets.jsonl +1467 -0
- package/scripts/training/generate-ecosystem-triplets.js +1324 -0
- package/scripts/training/rlm-dataset.js +671 -0
- package/scripts/training/routing-dataset.js +634 -0
- package/scripts/training/ruvector-capabilities.json +798 -0
- package/scripts/training/validate-ecosystem.js +381 -0
- package/scripts/training/validation-results.json +23 -0
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RLM (Retrieval Language Model) Type Definitions
|
|
3
|
+
*
|
|
4
|
+
* Types for the recursive retrieval-augmented generation system
|
|
5
|
+
* that breaks down complex queries into sub-queries and synthesizes
|
|
6
|
+
* answers from retrieved memory spans.
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Configuration for the RLM controller
|
|
10
|
+
*
|
|
11
|
+
* @example
|
|
12
|
+
* ```typescript
|
|
13
|
+
* const config: RlmConfig = {
|
|
14
|
+
* maxDepth: 3,
|
|
15
|
+
* maxSubQueries: 5,
|
|
16
|
+
* tokenBudget: 4096,
|
|
17
|
+
* enableCache: true,
|
|
18
|
+
* cacheTtl: 300000, // 5 minutes
|
|
19
|
+
* retrievalTopK: 10,
|
|
20
|
+
* minQualityScore: 0.7,
|
|
21
|
+
* enableReflection: true,
|
|
22
|
+
* maxReflectionIterations: 2,
|
|
23
|
+
* };
|
|
24
|
+
* ```
|
|
25
|
+
*/
|
|
26
|
+
export interface RlmConfig {
|
|
27
|
+
/** Maximum recursion depth for sub-queries (default: 3) */
|
|
28
|
+
maxDepth?: number;
|
|
29
|
+
/** Maximum number of sub-queries per level (default: 5) */
|
|
30
|
+
maxSubQueries?: number;
|
|
31
|
+
/** Token budget for generation (default: 4096) */
|
|
32
|
+
tokenBudget?: number;
|
|
33
|
+
/** Enable response caching (default: true) */
|
|
34
|
+
enableCache?: boolean;
|
|
35
|
+
/** Cache TTL in milliseconds (default: 300000 = 5 minutes) */
|
|
36
|
+
cacheTtl?: number;
|
|
37
|
+
/** Number of memory spans to retrieve (default: 10) */
|
|
38
|
+
retrievalTopK?: number;
|
|
39
|
+
/** Minimum quality score to accept answer (default: 0.7) */
|
|
40
|
+
minQualityScore?: number;
|
|
41
|
+
/** Enable self-reflection loop (default: false) */
|
|
42
|
+
enableReflection?: boolean;
|
|
43
|
+
/** Maximum reflection iterations (default: 2) */
|
|
44
|
+
maxReflectionIterations?: number;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Answer produced by the RLM controller
|
|
48
|
+
*
|
|
49
|
+
* @example
|
|
50
|
+
* ```typescript
|
|
51
|
+
* const answer: RlmAnswer = {
|
|
52
|
+
* text: 'Machine learning is a subset of artificial intelligence...',
|
|
53
|
+
* confidence: 0.92,
|
|
54
|
+
* qualityScore: 0.88,
|
|
55
|
+
* sources: [
|
|
56
|
+
* { id: 'mem-1', text: 'ML definition from textbook', similarityScore: 0.95, metadata: {} },
|
|
57
|
+
* ],
|
|
58
|
+
* subQueries: [
|
|
59
|
+
* { query: 'What is artificial intelligence?', answer: 'AI is...', depth: 1 },
|
|
60
|
+
* ],
|
|
61
|
+
* tokenUsage: { prompt: 512, completion: 256, total: 768 },
|
|
62
|
+
* cached: false,
|
|
63
|
+
* };
|
|
64
|
+
* ```
|
|
65
|
+
*/
|
|
66
|
+
export interface RlmAnswer {
|
|
67
|
+
/** The generated answer text */
|
|
68
|
+
text: string;
|
|
69
|
+
/** Overall confidence in the answer (0.0 - 1.0) */
|
|
70
|
+
confidence: number;
|
|
71
|
+
/** Quality score based on source coverage and coherence (0.0 - 1.0) */
|
|
72
|
+
qualityScore: number;
|
|
73
|
+
/** Memory spans used to generate the answer */
|
|
74
|
+
sources: MemorySpan[];
|
|
75
|
+
/** Sub-queries generated and answered (if recursive) */
|
|
76
|
+
subQueries?: SubQuery[];
|
|
77
|
+
/** Token usage statistics */
|
|
78
|
+
tokenUsage: TokenUsage;
|
|
79
|
+
/** Whether this answer was served from cache */
|
|
80
|
+
cached: boolean;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* A span of memory retrieved for context
|
|
84
|
+
*
|
|
85
|
+
* @example
|
|
86
|
+
* ```typescript
|
|
87
|
+
* const span: MemorySpan = {
|
|
88
|
+
* id: 'mem-abc123',
|
|
89
|
+
* text: 'Relevant context from memory...',
|
|
90
|
+
* similarityScore: 0.89,
|
|
91
|
+
* source: 'documentation',
|
|
92
|
+
* metadata: { timestamp: Date.now(), category: 'technical' },
|
|
93
|
+
* };
|
|
94
|
+
* ```
|
|
95
|
+
*/
|
|
96
|
+
export interface MemorySpan {
|
|
97
|
+
/** Unique identifier for the memory span */
|
|
98
|
+
id: string;
|
|
99
|
+
/** The text content of the memory span */
|
|
100
|
+
text: string;
|
|
101
|
+
/** Cosine similarity score to the query (0.0 - 1.0) */
|
|
102
|
+
similarityScore: number;
|
|
103
|
+
/** Optional source identifier (e.g., document name, URL) */
|
|
104
|
+
source?: string;
|
|
105
|
+
/** Additional metadata associated with this span */
|
|
106
|
+
metadata: Record<string, unknown>;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* A sub-query generated during recursive retrieval
|
|
110
|
+
*/
|
|
111
|
+
export interface SubQuery {
|
|
112
|
+
/** The generated sub-query text */
|
|
113
|
+
query: string;
|
|
114
|
+
/** The answer to the sub-query */
|
|
115
|
+
answer: string;
|
|
116
|
+
/** Recursion depth at which this sub-query was generated */
|
|
117
|
+
depth: number;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Token usage statistics for a query
|
|
121
|
+
*/
|
|
122
|
+
export interface TokenUsage {
|
|
123
|
+
/** Tokens used in the prompt (including context) */
|
|
124
|
+
prompt: number;
|
|
125
|
+
/** Tokens generated in the completion */
|
|
126
|
+
completion: number;
|
|
127
|
+
/** Total tokens used (prompt + completion) */
|
|
128
|
+
total: number;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Streaming token event
|
|
132
|
+
*
|
|
133
|
+
* Discriminated union for streaming responses:
|
|
134
|
+
* - `type: 'token'` - A partial token was generated
|
|
135
|
+
* - `type: 'done'` - Generation complete with final answer
|
|
136
|
+
*
|
|
137
|
+
* @example
|
|
138
|
+
* ```typescript
|
|
139
|
+
* for await (const event of controller.queryStream('What is AI?')) {
|
|
140
|
+
* if (event.type === 'token') {
|
|
141
|
+
* process.stdout.write(event.text);
|
|
142
|
+
* } else {
|
|
143
|
+
* console.log('\n\nFinal answer:', event.answer.text);
|
|
144
|
+
* }
|
|
145
|
+
* }
|
|
146
|
+
* ```
|
|
147
|
+
*/
|
|
148
|
+
export type StreamToken = {
|
|
149
|
+
/** Token event type */
|
|
150
|
+
type: 'token';
|
|
151
|
+
/** The partial text token */
|
|
152
|
+
text: string;
|
|
153
|
+
/** Always false for token events */
|
|
154
|
+
done: false;
|
|
155
|
+
} | {
|
|
156
|
+
/** Done event type */
|
|
157
|
+
type: 'done';
|
|
158
|
+
/** The complete answer */
|
|
159
|
+
answer: RlmAnswer;
|
|
160
|
+
/** Always true for done events */
|
|
161
|
+
done: true;
|
|
162
|
+
};
|
|
163
|
+
/**
|
|
164
|
+
* Internal cache entry for RLM answers
|
|
165
|
+
*/
|
|
166
|
+
export interface RlmCacheEntry {
|
|
167
|
+
/** The cached answer */
|
|
168
|
+
answer: RlmAnswer;
|
|
169
|
+
/** Timestamp when the entry was cached */
|
|
170
|
+
timestamp: number;
|
|
171
|
+
/** Query hash for cache key */
|
|
172
|
+
queryHash: string;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Reflection result from self-evaluation loop
|
|
176
|
+
*/
|
|
177
|
+
export interface ReflectionResult {
|
|
178
|
+
/** Whether the answer passed reflection criteria */
|
|
179
|
+
passed: boolean;
|
|
180
|
+
/** Critique of the current answer */
|
|
181
|
+
critique?: string;
|
|
182
|
+
/** Suggested improvements */
|
|
183
|
+
suggestions?: string[];
|
|
184
|
+
/** Updated quality score after reflection */
|
|
185
|
+
updatedScore: number;
|
|
186
|
+
/** Number of reflection iterations performed */
|
|
187
|
+
iterations: number;
|
|
188
|
+
}
|
|
189
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/rlm/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,SAAS;IACxB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,8CAA8C;IAC9C,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,uDAAuD;IACvD,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,4DAA4D;IAC5D,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB,mDAAmD;IACnD,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,iDAAiD;IACjD,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,SAAS;IACxB,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IAEb,mDAAmD;IACnD,UAAU,EAAE,MAAM,CAAC;IAEnB,uEAAuE;IACvE,YAAY,EAAE,MAAM,CAAC;IAErB,+CAA+C;IAC/C,OAAO,EAAE,UAAU,EAAE,CAAC;IAEtB,wDAAwD;IACxD,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;IAExB,6BAA6B;IAC7B,UAAU,EAAE,UAAU,CAAC;IAEvB,gDAAgD;IAChD,MAAM,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,UAAU;IACzB,4CAA4C;IAC5C,EAAE,EAAE,MAAM,CAAC;IAEX,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IAEb,uDAAuD;IACvD,eAAe,EAAE,MAAM,CAAC;IAExB,4DAA4D;IAC5D,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAC;IAEd,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IAEf,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IAEf,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IAEnB,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,MAAM,WAAW,GACnB;IACE,uBAAuB;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,IAAI,EAAE,KAAK,CAAC;CACb,GACD;IACE,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,0BAA0B;IAC1B,MAAM,EAAE,SAAS,CAAC;IAClB,kCAAkC;IAClC,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEN;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,wBAAwB;IACxB,MAAM,EAAE,SAAS,CAAC;IAElB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAElB,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,oDAAoD;IACpD,MAAM,EAAE,OAAO,CAAC;IAEhB,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,6BAA6B;IAC7B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,CAAC;IAErB,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;CACpB"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* RLM (Retrieval Language Model) Type Definitions
|
|
4
|
+
*
|
|
5
|
+
* Types for the recursive retrieval-augmented generation system
|
|
6
|
+
* that breaks down complex queries into sub-queries and synthesizes
|
|
7
|
+
* answers from retrieved memory spans.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHlwZXMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcmxtL3R5cGVzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQTs7Ozs7O0dBTUciLCJzb3VyY2VzQ29udGVudCI6WyIvKipcbiAqIFJMTSAoUmV0cmlldmFsIExhbmd1YWdlIE1vZGVsKSBUeXBlIERlZmluaXRpb25zXG4gKlxuICogVHlwZXMgZm9yIHRoZSByZWN1cnNpdmUgcmV0cmlldmFsLWF1Z21lbnRlZCBnZW5lcmF0aW9uIHN5c3RlbVxuICogdGhhdCBicmVha3MgZG93biBjb21wbGV4IHF1ZXJpZXMgaW50byBzdWItcXVlcmllcyBhbmQgc3ludGhlc2l6ZXNcbiAqIGFuc3dlcnMgZnJvbSByZXRyaWV2ZWQgbWVtb3J5IHNwYW5zLlxuICovXG5cbi8qKlxuICogQ29uZmlndXJhdGlvbiBmb3IgdGhlIFJMTSBjb250cm9sbGVyXG4gKlxuICogQGV4YW1wbGVcbiAqIGBgYHR5cGVzY3JpcHRcbiAqIGNvbnN0IGNvbmZpZzogUmxtQ29uZmlnID0ge1xuICogICBtYXhEZXB0aDogMyxcbiAqICAgbWF4U3ViUXVlcmllczogNSxcbiAqICAgdG9rZW5CdWRnZXQ6IDQwOTYsXG4gKiAgIGVuYWJsZUNhY2hlOiB0cnVlLFxuICogICBjYWNoZVR0bDogMzAwMDAwLCAvLyA1IG1pbnV0ZXNcbiAqICAgcmV0cmlldmFsVG9wSzogMTAsXG4gKiAgIG1pblF1YWxpdHlTY29yZTogMC43LFxuICogICBlbmFibGVSZWZsZWN0aW9uOiB0cnVlLFxuICogICBtYXhSZWZsZWN0aW9uSXRlcmF0aW9uczogMixcbiAqIH07XG4gKiBgYGBcbiAqL1xuZXhwb3J0IGludGVyZmFjZSBSbG1Db25maWcge1xuICAvKiogTWF4aW11bSByZWN1cnNpb24gZGVwdGggZm9yIHN1Yi1xdWVyaWVzIChkZWZhdWx0OiAzKSAqL1xuICBtYXhEZXB0aD86IG51bWJlcjtcblxuICAvKiogTWF4aW11bSBudW1iZXIgb2Ygc3ViLXF1ZXJpZXMgcGVyIGxldmVsIChkZWZhdWx0OiA1KSAqL1xuICBtYXhTdWJRdWVyaWVzPzogbnVtYmVyO1xuXG4gIC8qKiBUb2tlbiBidWRnZXQgZm9yIGdlbmVyYXRpb24gKGRlZmF1bHQ6IDQwOTYpICovXG4gIHRva2VuQnVkZ2V0PzogbnVtYmVyO1xuXG4gIC8qKiBFbmFibGUgcmVzcG9uc2UgY2FjaGluZyAoZGVmYXVsdDogdHJ1ZSkgKi9cbiAgZW5hYmxlQ2FjaGU/OiBib29sZWFuO1xuXG4gIC8qKiBDYWNoZSBUVEwgaW4gbWlsbGlzZWNvbmRzIChkZWZhdWx0OiAzMDAwMDAgPSA1IG1pbnV0ZXMpICovXG4gIGNhY2hlVHRsPzogbnVtYmVyO1xuXG4gIC8qKiBOdW1iZXIgb2YgbWVtb3J5IHNwYW5zIHRvIHJldHJpZXZlIChkZWZhdWx0OiAxMCkgKi9cbiAgcmV0cmlldmFsVG9wSz86IG51bWJlcjtcblxuICAvKiogTWluaW11bSBxdWFsaXR5IHNjb3JlIHRvIGFjY2VwdCBhbnN3ZXIgKGRlZmF1bHQ6IDAuNykgKi9cbiAgbWluUXVhbGl0eVNjb3JlPzogbnVtYmVyO1xuXG4gIC8qKiBFbmFibGUgc2VsZi1yZWZsZWN0aW9uIGxvb3AgKGRlZmF1bHQ6IGZhbHNlKSAqL1xuICBlbmFibGVSZWZsZWN0aW9uPzogYm9vbGVhbjtcblxuICAvKiogTWF4aW11bSByZWZsZWN0aW9uIGl0ZXJhdGlvbnMgKGRlZmF1bHQ6IDIpICovXG4gIG1heFJlZmxlY3Rpb25JdGVyYXRpb25zPzogbnVtYmVyO1xufVxuXG4vKipcbiAqIEFuc3dlciBwcm9kdWNlZCBieSB0aGUgUkxNIGNvbnRyb2xsZXJcbiAqXG4gKiBAZXhhbXBsZVxuICogYGBgdHlwZXNjcmlwdFxuICogY29uc3QgYW5zd2VyOiBSbG1BbnN3ZXIgPSB7XG4gKiAgIHRleHQ6ICdNYWNoaW5lIGxlYXJuaW5nIGlzIGEgc3Vic2V0IG9mIGFydGlmaWNpYWwgaW50ZWxsaWdlbmNlLi4uJyxcbiAqICAgY29uZmlkZW5jZTogMC45MixcbiAqICAgcXVhbGl0eVNjb3JlOiAwLjg4LFxuICogICBzb3VyY2VzOiBbXG4gKiAgICAgeyBpZDogJ21lbS0xJywgdGV4dDogJ01MIGRlZmluaXRpb24gZnJvbSB0ZXh0Ym9vaycsIHNpbWlsYXJpdHlTY29yZTogMC45NSwgbWV0YWRhdGE6IHt9IH0sXG4gKiAgIF0sXG4gKiAgIHN1YlF1ZXJpZXM6IFtcbiAqICAgICB7IHF1ZXJ5OiAnV2hhdCBpcyBhcnRpZmljaWFsIGludGVsbGlnZW5jZT8nLCBhbnN3ZXI6ICdBSSBpcy4uLicsIGRlcHRoOiAxIH0sXG4gKiAgIF0sXG4gKiAgIHRva2VuVXNhZ2U6IHsgcHJvbXB0OiA1MTIsIGNvbXBsZXRpb246IDI1NiwgdG90YWw6IDc2OCB9LFxuICogICBjYWNoZWQ6IGZhbHNlLFxuICogfTtcbiAqIGBgYFxuICovXG5leHBvcnQgaW50ZXJmYWNlIFJsbUFuc3dlciB7XG4gIC8qKiBUaGUgZ2VuZXJhdGVkIGFuc3dlciB0ZXh0ICovXG4gIHRleHQ6IHN0cmluZztcblxuICAvKiogT3ZlcmFsbCBjb25maWRlbmNlIGluIHRoZSBhbnN3ZXIgKDAuMCAtIDEuMCkgKi9cbiAgY29uZmlkZW5jZTogbnVtYmVyO1xuXG4gIC8qKiBRdWFsaXR5IHNjb3JlIGJhc2VkIG9uIHNvdXJjZSBjb3ZlcmFnZSBhbmQgY29oZXJlbmNlICgwLjAgLSAxLjApICovXG4gIHF1YWxpdHlTY29yZTogbnVtYmVyO1xuXG4gIC8qKiBNZW1vcnkgc3BhbnMgdXNlZCB0byBnZW5lcmF0ZSB0aGUgYW5zd2VyICovXG4gIHNvdXJjZXM6IE1lbW9yeVNwYW5bXTtcblxuICAvKiogU3ViLXF1ZXJpZXMgZ2VuZXJhdGVkIGFuZCBhbnN3ZXJlZCAoaWYgcmVjdXJzaXZlKSAqL1xuICBzdWJRdWVyaWVzPzogU3ViUXVlcnlbXTtcblxuICAvKiogVG9rZW4gdXNhZ2Ugc3RhdGlzdGljcyAqL1xuICB0b2tlblVzYWdlOiBUb2tlblVzYWdlO1xuXG4gIC8qKiBXaGV0aGVyIHRoaXMgYW5zd2VyIHdhcyBzZXJ2ZWQgZnJvbSBjYWNoZSAqL1xuICBjYWNoZWQ6IGJvb2xlYW47XG59XG5cbi8qKlxuICogQSBzcGFuIG9mIG1lbW9yeSByZXRyaWV2ZWQgZm9yIGNvbnRleHRcbiAqXG4gKiBAZXhhbXBsZVxuICogYGBgdHlwZXNjcmlwdFxuICogY29uc3Qgc3BhbjogTWVtb3J5U3BhbiA9IHtcbiAqICAgaWQ6ICdtZW0tYWJjMTIzJyxcbiAqICAgdGV4dDogJ1JlbGV2YW50IGNvbnRleHQgZnJvbSBtZW1vcnkuLi4nLFxuICogICBzaW1pbGFyaXR5U2NvcmU6IDAuODksXG4gKiAgIHNvdXJjZTogJ2RvY3VtZW50YXRpb24nLFxuICogICBtZXRhZGF0YTogeyB0aW1lc3RhbXA6IERhdGUubm93KCksIGNhdGVnb3J5OiAndGVjaG5pY2FsJyB9LFxuICogfTtcbiAqIGBgYFxuICovXG5leHBvcnQgaW50ZXJmYWNlIE1lbW9yeVNwYW4ge1xuICAvKiogVW5pcXVlIGlkZW50aWZpZXIgZm9yIHRoZSBtZW1vcnkgc3BhbiAqL1xuICBpZDogc3RyaW5nO1xuXG4gIC8qKiBUaGUgdGV4dCBjb250ZW50IG9mIHRoZSBtZW1vcnkgc3BhbiAqL1xuICB0ZXh0OiBzdHJpbmc7XG5cbiAgLyoqIENvc2luZSBzaW1pbGFyaXR5IHNjb3JlIHRvIHRoZSBxdWVyeSAoMC4wIC0gMS4wKSAqL1xuICBzaW1pbGFyaXR5U2NvcmU6IG51bWJlcjtcblxuICAvKiogT3B0aW9uYWwgc291cmNlIGlkZW50aWZpZXIgKGUuZy4sIGRvY3VtZW50IG5hbWUsIFVSTCkgKi9cbiAgc291cmNlPzogc3RyaW5nO1xuXG4gIC8qKiBBZGRpdGlvbmFsIG1ldGFkYXRhIGFzc29jaWF0ZWQgd2l0aCB0aGlzIHNwYW4gKi9cbiAgbWV0YWRhdGE6IFJlY29yZDxzdHJpbmcsIHVua25vd24+O1xufVxuXG4vKipcbiAqIEEgc3ViLXF1ZXJ5IGdlbmVyYXRlZCBkdXJpbmcgcmVjdXJzaXZlIHJldHJpZXZhbFxuICovXG5leHBvcnQgaW50ZXJmYWNlIFN1YlF1ZXJ5IHtcbiAgLyoqIFRoZSBnZW5lcmF0ZWQgc3ViLXF1ZXJ5IHRleHQgKi9cbiAgcXVlcnk6IHN0cmluZztcblxuICAvKiogVGhlIGFuc3dlciB0byB0aGUgc3ViLXF1ZXJ5ICovXG4gIGFuc3dlcjogc3RyaW5nO1xuXG4gIC8qKiBSZWN1cnNpb24gZGVwdGggYXQgd2hpY2ggdGhpcyBzdWItcXVlcnkgd2FzIGdlbmVyYXRlZCAqL1xuICBkZXB0aDogbnVtYmVyO1xufVxuXG4vKipcbiAqIFRva2VuIHVzYWdlIHN0YXRpc3RpY3MgZm9yIGEgcXVlcnlcbiAqL1xuZXhwb3J0IGludGVyZmFjZSBUb2tlblVzYWdlIHtcbiAgLyoqIFRva2VucyB1c2VkIGluIHRoZSBwcm9tcHQgKGluY2x1ZGluZyBjb250ZXh0KSAqL1xuICBwcm9tcHQ6IG51bWJlcjtcblxuICAvKiogVG9rZW5zIGdlbmVyYXRlZCBpbiB0aGUgY29tcGxldGlvbiAqL1xuICBjb21wbGV0aW9uOiBudW1iZXI7XG5cbiAgLyoqIFRvdGFsIHRva2VucyB1c2VkIChwcm9tcHQgKyBjb21wbGV0aW9uKSAqL1xuICB0b3RhbDogbnVtYmVyO1xufVxuXG4vKipcbiAqIFN0cmVhbWluZyB0b2tlbiBldmVudFxuICpcbiAqIERpc2NyaW1pbmF0ZWQgdW5pb24gZm9yIHN0cmVhbWluZyByZXNwb25zZXM6XG4gKiAtIGB0eXBlOiAndG9rZW4nYCAtIEEgcGFydGlhbCB0b2tlbiB3YXMgZ2VuZXJhdGVkXG4gKiAtIGB0eXBlOiAnZG9uZSdgIC0gR2VuZXJhdGlvbiBjb21wbGV0ZSB3aXRoIGZpbmFsIGFuc3dlclxuICpcbiAqIEBleGFtcGxlXG4gKiBgYGB0eXBlc2NyaXB0XG4gKiBmb3IgYXdhaXQgKGNvbnN0IGV2ZW50IG9mIGNvbnRyb2xsZXIucXVlcnlTdHJlYW0oJ1doYXQgaXMgQUk/JykpIHtcbiAqICAgaWYgKGV2ZW50LnR5cGUgPT09ICd0b2tlbicpIHtcbiAqICAgICBwcm9jZXNzLnN0ZG91dC53cml0ZShldmVudC50ZXh0KTtcbiAqICAgfSBlbHNlIHtcbiAqICAgICBjb25zb2xlLmxvZygnXFxuXFxuRmluYWwgYW5zd2VyOicsIGV2ZW50LmFuc3dlci50ZXh0KTtcbiAqICAgfVxuICogfVxuICogYGBgXG4gKi9cbmV4cG9ydCB0eXBlIFN0cmVhbVRva2VuID1cbiAgfCB7XG4gICAgICAvKiogVG9rZW4gZXZlbnQgdHlwZSAqL1xuICAgICAgdHlwZTogJ3Rva2VuJztcbiAgICAgIC8qKiBUaGUgcGFydGlhbCB0ZXh0IHRva2VuICovXG4gICAgICB0ZXh0OiBzdHJpbmc7XG4gICAgICAvKiogQWx3YXlzIGZhbHNlIGZvciB0b2tlbiBldmVudHMgKi9cbiAgICAgIGRvbmU6IGZhbHNlO1xuICAgIH1cbiAgfCB7XG4gICAgICAvKiogRG9uZSBldmVudCB0eXBlICovXG4gICAgICB0eXBlOiAnZG9uZSc7XG4gICAgICAvKiogVGhlIGNvbXBsZXRlIGFuc3dlciAqL1xuICAgICAgYW5zd2VyOiBSbG1BbnN3ZXI7XG4gICAgICAvKiogQWx3YXlzIHRydWUgZm9yIGRvbmUgZXZlbnRzICovXG4gICAgICBkb25lOiB0cnVlO1xuICAgIH07XG5cbi8qKlxuICogSW50ZXJuYWwgY2FjaGUgZW50cnkgZm9yIFJMTSBhbnN3ZXJzXG4gKi9cbmV4cG9ydCBpbnRlcmZhY2UgUmxtQ2FjaGVFbnRyeSB7XG4gIC8qKiBUaGUgY2FjaGVkIGFuc3dlciAqL1xuICBhbnN3ZXI6IFJsbUFuc3dlcjtcblxuICAvKiogVGltZXN0YW1wIHdoZW4gdGhlIGVudHJ5IHdhcyBjYWNoZWQgKi9cbiAgdGltZXN0YW1wOiBudW1iZXI7XG5cbiAgLyoqIFF1ZXJ5IGhhc2ggZm9yIGNhY2hlIGtleSAqL1xuICBxdWVyeUhhc2g6IHN0cmluZztcbn1cblxuLyoqXG4gKiBSZWZsZWN0aW9uIHJlc3VsdCBmcm9tIHNlbGYtZXZhbHVhdGlvbiBsb29wXG4gKi9cbmV4cG9ydCBpbnRlcmZhY2UgUmVmbGVjdGlvblJlc3VsdCB7XG4gIC8qKiBXaGV0aGVyIHRoZSBhbnN3ZXIgcGFzc2VkIHJlZmxlY3Rpb24gY3JpdGVyaWEgKi9cbiAgcGFzc2VkOiBib29sZWFuO1xuXG4gIC8qKiBDcml0aXF1ZSBvZiB0aGUgY3VycmVudCBhbnN3ZXIgKi9cbiAgY3JpdGlxdWU/OiBzdHJpbmc7XG5cbiAgLyoqIFN1Z2dlc3RlZCBpbXByb3ZlbWVudHMgKi9cbiAgc3VnZ2VzdGlvbnM/OiBzdHJpbmdbXTtcblxuICAvKiogVXBkYXRlZCBxdWFsaXR5IHNjb3JlIGFmdGVyIHJlZmxlY3Rpb24gKi9cbiAgdXBkYXRlZFNjb3JlOiBudW1iZXI7XG5cbiAgLyoqIE51bWJlciBvZiByZWZsZWN0aW9uIGl0ZXJhdGlvbnMgcGVyZm9ybWVkICovXG4gIGl0ZXJhdGlvbnM6IG51bWJlcjtcbn1cbiJdfQ==
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding Quality Benchmark for RuvLTRA Models
|
|
3
|
+
*
|
|
4
|
+
* Tests embedding quality for Claude Code use cases:
|
|
5
|
+
* - Code similarity detection
|
|
6
|
+
* - Task clustering
|
|
7
|
+
* - Semantic search accuracy
|
|
8
|
+
*/
|
|
9
|
+
export interface EmbeddingPair {
|
|
10
|
+
id: string;
|
|
11
|
+
text1: string;
|
|
12
|
+
text2: string;
|
|
13
|
+
similarity: 'high' | 'medium' | 'low' | 'none';
|
|
14
|
+
category: string;
|
|
15
|
+
}
|
|
16
|
+
export interface EmbeddingResult {
|
|
17
|
+
pairId: string;
|
|
18
|
+
expectedSimilarity: string;
|
|
19
|
+
computedScore: number;
|
|
20
|
+
correct: boolean;
|
|
21
|
+
latencyMs: number;
|
|
22
|
+
}
|
|
23
|
+
export interface ClusterTestCase {
|
|
24
|
+
id: string;
|
|
25
|
+
items: string[];
|
|
26
|
+
expectedCluster: string;
|
|
27
|
+
}
|
|
28
|
+
export interface EmbeddingBenchmarkResults {
|
|
29
|
+
similarityAccuracy: number;
|
|
30
|
+
similarityByCategory: Record<string, number>;
|
|
31
|
+
avgSimilarityLatencyMs: number;
|
|
32
|
+
clusterPurity: number;
|
|
33
|
+
silhouetteScore: number;
|
|
34
|
+
searchMRR: number;
|
|
35
|
+
searchNDCG: number;
|
|
36
|
+
similarityResults: EmbeddingResult[];
|
|
37
|
+
totalPairs: number;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Ground truth similarity pairs for testing
|
|
41
|
+
* Tests whether embeddings correctly capture semantic similarity
|
|
42
|
+
*/
|
|
43
|
+
export declare const SIMILARITY_TEST_PAIRS: EmbeddingPair[];
|
|
44
|
+
/**
|
|
45
|
+
* Search relevance test cases
|
|
46
|
+
* Query + documents with relevance scores
|
|
47
|
+
*/
|
|
48
|
+
export interface SearchTestCase {
|
|
49
|
+
id: string;
|
|
50
|
+
query: string;
|
|
51
|
+
documents: {
|
|
52
|
+
text: string;
|
|
53
|
+
relevance: number;
|
|
54
|
+
}[];
|
|
55
|
+
}
|
|
56
|
+
export declare const SEARCH_TEST_CASES: SearchTestCase[];
|
|
57
|
+
/**
|
|
58
|
+
* Cluster test cases - items that should cluster together
|
|
59
|
+
*/
|
|
60
|
+
export declare const CLUSTER_TEST_CASES: ClusterTestCase[];
|
|
61
|
+
/**
|
|
62
|
+
* Check if computed similarity matches expected category
|
|
63
|
+
*/
|
|
64
|
+
export declare function isCorrectSimilarity(expected: 'high' | 'medium' | 'low' | 'none', computed: number): boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Calculate Mean Reciprocal Rank for search results
|
|
67
|
+
*/
|
|
68
|
+
export declare function calculateMRR(rankings: {
|
|
69
|
+
relevant: boolean;
|
|
70
|
+
}[][]): number;
|
|
71
|
+
/**
|
|
72
|
+
* Calculate NDCG for search results
|
|
73
|
+
*/
|
|
74
|
+
export declare function calculateNDCG(results: {
|
|
75
|
+
relevance: number;
|
|
76
|
+
}[], idealOrder: {
|
|
77
|
+
relevance: number;
|
|
78
|
+
}[]): number;
|
|
79
|
+
/**
|
|
80
|
+
* Calculate silhouette score for clustering
|
|
81
|
+
*/
|
|
82
|
+
export declare function calculateSilhouette(embeddings: number[][], labels: number[]): number;
|
|
83
|
+
/**
|
|
84
|
+
* Run the embedding benchmark
|
|
85
|
+
*/
|
|
86
|
+
export declare function runEmbeddingBenchmark(embedder: (text: string) => number[], similarityFn: (a: number[], b: number[]) => number): EmbeddingBenchmarkResults;
|
|
87
|
+
/**
|
|
88
|
+
* Format embedding benchmark results for display
|
|
89
|
+
*/
|
|
90
|
+
export declare function formatEmbeddingResults(results: EmbeddingBenchmarkResults): string;
|
|
91
|
+
declare const _default: {
|
|
92
|
+
SIMILARITY_TEST_PAIRS: EmbeddingPair[];
|
|
93
|
+
SEARCH_TEST_CASES: SearchTestCase[];
|
|
94
|
+
CLUSTER_TEST_CASES: ClusterTestCase[];
|
|
95
|
+
runEmbeddingBenchmark: typeof runEmbeddingBenchmark;
|
|
96
|
+
formatEmbeddingResults: typeof formatEmbeddingResults;
|
|
97
|
+
isCorrectSimilarity: typeof isCorrectSimilarity;
|
|
98
|
+
calculateMRR: typeof calculateMRR;
|
|
99
|
+
calculateNDCG: typeof calculateNDCG;
|
|
100
|
+
};
|
|
101
|
+
export default _default;
|
|
102
|
+
//# sourceMappingURL=embedding-benchmark.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-benchmark.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/embedding-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;IAC/C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IAExC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,sBAAsB,EAAE,MAAM,CAAC;IAG/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IAGxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IAGnB,iBAAiB,EAAE,eAAe,EAAE,CAAC;IACrC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,EAAE,aAAa,EA8ChD,CAAC;AAEF;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAClD;AAED,eAAO,MAAM,iBAAiB,EAAE,cAAc,EAwD7C,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,kBAAkB,EAAE,eAAe,EAwD/C,CAAC;AAYF;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,EAC5C,QAAQ,EAAE,MAAM,GACf,OAAO,CAGT;AAED;;GAEG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE;IAAE,QAAQ,EAAE,OAAO,CAAA;CAAE,EAAE,EAAE,GAClC,MAAM,CASR;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,EAAE,EAChC,UAAU,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,EAAE,GAClC,MAAM,CAUR;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,MAAM,EAAE,MAAM,EAAE,GACf,MAAM,CA8CR;AAUD;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,EACpC,YAAY,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,MAAM,GACjD,yBAAyB,CA0G3B;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,yBAAyB,GAAG,MAAM,CAyDjF;;;;;;;;;;;AAED,wBASE"}
|