@ruvector/ruvllm 0.2.4 → 2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +370 -286
  2. package/bin/cli.js +619 -2
  3. package/dist/cjs/benchmarks/embedding-benchmark.d.ts +102 -0
  4. package/dist/cjs/benchmarks/embedding-benchmark.d.ts.map +1 -0
  5. package/dist/cjs/benchmarks/embedding-benchmark.js +436 -0
  6. package/dist/cjs/benchmarks/index.d.ts +34 -0
  7. package/dist/cjs/benchmarks/index.d.ts.map +1 -0
  8. package/dist/cjs/benchmarks/index.js +132 -0
  9. package/dist/cjs/benchmarks/model-comparison.d.ts +71 -0
  10. package/dist/cjs/benchmarks/model-comparison.d.ts.map +1 -0
  11. package/dist/cjs/benchmarks/model-comparison.js +476 -0
  12. package/dist/cjs/benchmarks/routing-benchmark.d.ts +70 -0
  13. package/dist/cjs/benchmarks/routing-benchmark.d.ts.map +1 -0
  14. package/dist/cjs/benchmarks/routing-benchmark.js +289 -0
  15. package/dist/cjs/contrastive.d.ts +229 -0
  16. package/dist/cjs/contrastive.d.ts.map +1 -0
  17. package/dist/cjs/contrastive.js +589 -0
  18. package/dist/cjs/engine.d.ts +3 -0
  19. package/dist/cjs/engine.d.ts.map +1 -1
  20. package/dist/cjs/engine.js +26 -6
  21. package/dist/cjs/index.d.ts +4 -0
  22. package/dist/cjs/index.d.ts.map +1 -1
  23. package/dist/cjs/index.js +9 -1
  24. package/dist/cjs/models.d.ts +129 -0
  25. package/dist/cjs/models.d.ts.map +1 -0
  26. package/dist/cjs/models.js +323 -0
  27. package/dist/cjs/native.d.ts +3 -3
  28. package/dist/cjs/native.d.ts.map +1 -1
  29. package/dist/cjs/native.js +1 -1
  30. package/dist/cjs/rlm/controller.d.ts +245 -0
  31. package/dist/cjs/rlm/controller.d.ts.map +1 -0
  32. package/dist/cjs/rlm/controller.js +554 -0
  33. package/dist/cjs/rlm/index.d.ts +62 -0
  34. package/dist/cjs/rlm/index.d.ts.map +1 -0
  35. package/dist/cjs/rlm/index.js +97 -0
  36. package/dist/cjs/rlm/training.d.ts +333 -0
  37. package/dist/cjs/rlm/training.d.ts.map +1 -0
  38. package/dist/cjs/rlm/training.js +680 -0
  39. package/dist/cjs/rlm/types.d.ts +189 -0
  40. package/dist/cjs/rlm/types.d.ts.map +1 -0
  41. package/dist/cjs/rlm/types.js +10 -0
  42. package/dist/esm/benchmarks/embedding-benchmark.d.ts +102 -0
  43. package/dist/esm/benchmarks/embedding-benchmark.d.ts.map +1 -0
  44. package/dist/esm/benchmarks/embedding-benchmark.js +427 -0
  45. package/dist/esm/benchmarks/index.d.ts +34 -0
  46. package/dist/esm/benchmarks/index.d.ts.map +1 -0
  47. package/dist/esm/benchmarks/index.js +110 -0
  48. package/dist/esm/benchmarks/model-comparison.d.ts +71 -0
  49. package/dist/esm/benchmarks/model-comparison.d.ts.map +1 -0
  50. package/dist/esm/benchmarks/model-comparison.js +434 -0
  51. package/dist/esm/benchmarks/routing-benchmark.d.ts +70 -0
  52. package/dist/esm/benchmarks/routing-benchmark.d.ts.map +1 -0
  53. package/dist/esm/benchmarks/routing-benchmark.js +283 -0
  54. package/dist/esm/contrastive.d.ts +229 -0
  55. package/dist/esm/contrastive.d.ts.map +1 -0
  56. package/dist/esm/contrastive.js +578 -0
  57. package/dist/esm/engine.d.ts +3 -0
  58. package/dist/esm/engine.d.ts.map +1 -1
  59. package/dist/esm/engine.js +26 -6
  60. package/dist/esm/index.d.ts +4 -0
  61. package/dist/esm/index.d.ts.map +1 -1
  62. package/dist/esm/index.js +9 -1
  63. package/dist/esm/models.d.ts +129 -0
  64. package/dist/esm/models.d.ts.map +1 -0
  65. package/dist/esm/models.js +282 -0
  66. package/dist/esm/native.d.ts +3 -3
  67. package/dist/esm/native.d.ts.map +1 -1
  68. package/dist/esm/native.js +1 -1
  69. package/dist/esm/rlm/controller.d.ts +245 -0
  70. package/dist/esm/rlm/controller.d.ts.map +1 -0
  71. package/dist/esm/rlm/controller.js +550 -0
  72. package/dist/esm/rlm/index.d.ts +62 -0
  73. package/dist/esm/rlm/index.d.ts.map +1 -0
  74. package/dist/esm/rlm/index.js +71 -0
  75. package/dist/esm/rlm/training.d.ts +333 -0
  76. package/dist/esm/rlm/training.d.ts.map +1 -0
  77. package/dist/esm/rlm/training.js +672 -0
  78. package/dist/esm/rlm/types.d.ts +189 -0
  79. package/dist/esm/rlm/types.d.ts.map +1 -0
  80. package/dist/esm/rlm/types.js +9 -0
  81. package/package.json +13 -8
  82. package/scripts/ensemble-model-compare.js +287 -0
  83. package/scripts/huggingface/README.md +537 -0
  84. package/scripts/huggingface/publish.sh +112 -0
  85. package/scripts/hybrid-model-compare.js +373 -0
  86. package/scripts/improved-model-compare.js +288 -0
  87. package/scripts/optimized-model-compare.js +364 -0
  88. package/scripts/real-model-compare.js +280 -0
  89. package/scripts/training/agentic-flow-capabilities.json +329 -0
  90. package/scripts/training/claude-code-synth.js +505 -0
  91. package/scripts/training/claude-flow-capabilities.json +1322 -0
  92. package/scripts/training/claude-hard-negatives.js +222 -0
  93. package/scripts/training/contrastive-finetune.js +517 -0
  94. package/scripts/training/ecosystem-triplets.jsonl +1467 -0
  95. package/scripts/training/generate-ecosystem-triplets.js +1324 -0
  96. package/scripts/training/rlm-dataset.js +671 -0
  97. package/scripts/training/routing-dataset.js +634 -0
  98. package/scripts/training/ruvector-capabilities.json +798 -0
  99. package/scripts/training/validate-ecosystem.js +381 -0
  100. package/scripts/training/validation-results.json +23 -0
@@ -0,0 +1,189 @@
1
+ /**
2
+ * RLM (Retrieval Language Model) Type Definitions
3
+ *
4
+ * Types for the recursive retrieval-augmented generation system
5
+ * that breaks down complex queries into sub-queries and synthesizes
6
+ * answers from retrieved memory spans.
7
+ */
8
+ /**
9
+ * Configuration for the RLM controller
10
+ *
11
+ * @example
12
+ * ```typescript
13
+ * const config: RlmConfig = {
14
+ * maxDepth: 3,
15
+ * maxSubQueries: 5,
16
+ * tokenBudget: 4096,
17
+ * enableCache: true,
18
+ * cacheTtl: 300000, // 5 minutes
19
+ * retrievalTopK: 10,
20
+ * minQualityScore: 0.7,
21
+ * enableReflection: true,
22
+ * maxReflectionIterations: 2,
23
+ * };
24
+ * ```
25
+ */
26
+ export interface RlmConfig {
27
+ /** Maximum recursion depth for sub-queries (default: 3) */
28
+ maxDepth?: number;
29
+ /** Maximum number of sub-queries per level (default: 5) */
30
+ maxSubQueries?: number;
31
+ /** Token budget for generation (default: 4096) */
32
+ tokenBudget?: number;
33
+ /** Enable response caching (default: true) */
34
+ enableCache?: boolean;
35
+ /** Cache TTL in milliseconds (default: 300000 = 5 minutes) */
36
+ cacheTtl?: number;
37
+ /** Number of memory spans to retrieve (default: 10) */
38
+ retrievalTopK?: number;
39
+ /** Minimum quality score to accept answer (default: 0.7) */
40
+ minQualityScore?: number;
41
+ /** Enable self-reflection loop (default: false) */
42
+ enableReflection?: boolean;
43
+ /** Maximum reflection iterations (default: 2) */
44
+ maxReflectionIterations?: number;
45
+ }
46
+ /**
47
+ * Answer produced by the RLM controller
48
+ *
49
+ * @example
50
+ * ```typescript
51
+ * const answer: RlmAnswer = {
52
+ * text: 'Machine learning is a subset of artificial intelligence...',
53
+ * confidence: 0.92,
54
+ * qualityScore: 0.88,
55
+ * sources: [
56
+ * { id: 'mem-1', text: 'ML definition from textbook', similarityScore: 0.95, metadata: {} },
57
+ * ],
58
+ * subQueries: [
59
+ * { query: 'What is artificial intelligence?', answer: 'AI is...', depth: 1 },
60
+ * ],
61
+ * tokenUsage: { prompt: 512, completion: 256, total: 768 },
62
+ * cached: false,
63
+ * };
64
+ * ```
65
+ */
66
+ export interface RlmAnswer {
67
+ /** The generated answer text */
68
+ text: string;
69
+ /** Overall confidence in the answer (0.0 - 1.0) */
70
+ confidence: number;
71
+ /** Quality score based on source coverage and coherence (0.0 - 1.0) */
72
+ qualityScore: number;
73
+ /** Memory spans used to generate the answer */
74
+ sources: MemorySpan[];
75
+ /** Sub-queries generated and answered (if recursive) */
76
+ subQueries?: SubQuery[];
77
+ /** Token usage statistics */
78
+ tokenUsage: TokenUsage;
79
+ /** Whether this answer was served from cache */
80
+ cached: boolean;
81
+ }
82
+ /**
83
+ * A span of memory retrieved for context
84
+ *
85
+ * @example
86
+ * ```typescript
87
+ * const span: MemorySpan = {
88
+ * id: 'mem-abc123',
89
+ * text: 'Relevant context from memory...',
90
+ * similarityScore: 0.89,
91
+ * source: 'documentation',
92
+ * metadata: { timestamp: Date.now(), category: 'technical' },
93
+ * };
94
+ * ```
95
+ */
96
+ export interface MemorySpan {
97
+ /** Unique identifier for the memory span */
98
+ id: string;
99
+ /** The text content of the memory span */
100
+ text: string;
101
+ /** Cosine similarity score to the query (0.0 - 1.0) */
102
+ similarityScore: number;
103
+ /** Optional source identifier (e.g., document name, URL) */
104
+ source?: string;
105
+ /** Additional metadata associated with this span */
106
+ metadata: Record<string, unknown>;
107
+ }
108
+ /**
109
+ * A sub-query generated during recursive retrieval
110
+ */
111
+ export interface SubQuery {
112
+ /** The generated sub-query text */
113
+ query: string;
114
+ /** The answer to the sub-query */
115
+ answer: string;
116
+ /** Recursion depth at which this sub-query was generated */
117
+ depth: number;
118
+ }
119
+ /**
120
+ * Token usage statistics for a query
121
+ */
122
+ export interface TokenUsage {
123
+ /** Tokens used in the prompt (including context) */
124
+ prompt: number;
125
+ /** Tokens generated in the completion */
126
+ completion: number;
127
+ /** Total tokens used (prompt + completion) */
128
+ total: number;
129
+ }
130
+ /**
131
+ * Streaming token event
132
+ *
133
+ * Discriminated union for streaming responses:
134
+ * - `type: 'token'` - A partial token was generated
135
+ * - `type: 'done'` - Generation complete with final answer
136
+ *
137
+ * @example
138
+ * ```typescript
139
+ * for await (const event of controller.queryStream('What is AI?')) {
140
+ * if (event.type === 'token') {
141
+ * process.stdout.write(event.text);
142
+ * } else {
143
+ * console.log('\n\nFinal answer:', event.answer.text);
144
+ * }
145
+ * }
146
+ * ```
147
+ */
148
+ export type StreamToken = {
149
+ /** Token event type */
150
+ type: 'token';
151
+ /** The partial text token */
152
+ text: string;
153
+ /** Always false for token events */
154
+ done: false;
155
+ } | {
156
+ /** Done event type */
157
+ type: 'done';
158
+ /** The complete answer */
159
+ answer: RlmAnswer;
160
+ /** Always true for done events */
161
+ done: true;
162
+ };
163
+ /**
164
+ * Internal cache entry for RLM answers
165
+ */
166
+ export interface RlmCacheEntry {
167
+ /** The cached answer */
168
+ answer: RlmAnswer;
169
+ /** Timestamp when the entry was cached */
170
+ timestamp: number;
171
+ /** Query hash for cache key */
172
+ queryHash: string;
173
+ }
174
+ /**
175
+ * Reflection result from self-evaluation loop
176
+ */
177
+ export interface ReflectionResult {
178
+ /** Whether the answer passed reflection criteria */
179
+ passed: boolean;
180
+ /** Critique of the current answer */
181
+ critique?: string;
182
+ /** Suggested improvements */
183
+ suggestions?: string[];
184
+ /** Updated quality score after reflection */
185
+ updatedScore: number;
186
+ /** Number of reflection iterations performed */
187
+ iterations: number;
188
+ }
189
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/rlm/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,SAAS;IACxB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,8CAA8C;IAC9C,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,uDAAuD;IACvD,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,4DAA4D;IAC5D,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB,mDAAmD;IACnD,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,iDAAiD;IACjD,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,SAAS;IACxB,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IAEb,mDAAmD;IACnD,UAAU,EAAE,MAAM,CAAC;IAEnB,uEAAuE;IACvE,YAAY,EAAE,MAAM,CAAC;IAErB,+CAA+C;IAC/C,OAAO,EAAE,UAAU,EAAE,CAAC;IAEtB,wDAAwD;IACxD,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;IAExB,6BAA6B;IAC7B,UAAU,EAAE,UAAU,CAAC;IAEvB,gDAAgD;IAChD,MAAM,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,UAAU;IACzB,4CAA4C;IAC5C,EAAE,EAAE,MAAM,CAAC;IAEX,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IAEb,uDAAuD;IACvD,eAAe,EAAE,MAAM,CAAC;IAExB,4DAA4D;IAC5D,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAC;IAEd,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IAEf,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IAEf,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IAEnB,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,MAAM,WAAW,GACnB;IACE,uBAAuB;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,IAAI,EAAE,KAAK,CAAC;CACb,GACD;IACE,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,0BAA0B;IAC1B,MAAM,EAAE,SAAS,CAAC;IAClB,kCAAkC;IAClC,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEN;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,wBAAwB;IACxB,MAAM,EAAE,SAAS,CAAC;IAElB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAElB,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,oDAAoD;IACpD,MAAM,EAAE,OAAO,CAAC;IAEhB,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,6BAA6B;IAC7B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,CAAC;IAErB,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;CACpB"}
@@ -0,0 +1,10 @@
1
+ "use strict";
2
+ /**
3
+ * RLM (Retrieval Language Model) Type Definitions
4
+ *
5
+ * Types for the recursive retrieval-augmented generation system
6
+ * that breaks down complex queries into sub-queries and synthesizes
7
+ * answers from retrieved memory spans.
8
+ */
9
+ Object.defineProperty(exports, "__esModule", { value: true });
10
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHlwZXMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvcmxtL3R5cGVzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQTs7Ozs7O0dBTUciLCJzb3VyY2VzQ29udGVudCI6WyIvKipcbiAqIFJMTSAoUmV0cmlldmFsIExhbmd1YWdlIE1vZGVsKSBUeXBlIERlZmluaXRpb25zXG4gKlxuICogVHlwZXMgZm9yIHRoZSByZWN1cnNpdmUgcmV0cmlldmFsLWF1Z21lbnRlZCBnZW5lcmF0aW9uIHN5c3RlbVxuICogdGhhdCBicmVha3MgZG93biBjb21wbGV4IHF1ZXJpZXMgaW50byBzdWItcXVlcmllcyBhbmQgc3ludGhlc2l6ZXNcbiAqIGFuc3dlcnMgZnJvbSByZXRyaWV2ZWQgbWVtb3J5IHNwYW5zLlxuICovXG5cbi8qKlxuICogQ29uZmlndXJhdGlvbiBmb3IgdGhlIFJMTSBjb250cm9sbGVyXG4gKlxuICogQGV4YW1wbGVcbiAqIGBgYHR5cGVzY3JpcHRcbiAqIGNvbnN0IGNvbmZpZzogUmxtQ29uZmlnID0ge1xuICogICBtYXhEZXB0aDogMyxcbiAqICAgbWF4U3ViUXVlcmllczogNSxcbiAqICAgdG9rZW5CdWRnZXQ6IDQwOTYsXG4gKiAgIGVuYWJsZUNhY2hlOiB0cnVlLFxuICogICBjYWNoZVR0bDogMzAwMDAwLCAvLyA1IG1pbnV0ZXNcbiAqICAgcmV0cmlldmFsVG9wSzogMTAsXG4gKiAgIG1pblF1YWxpdHlTY29yZTogMC43LFxuICogICBlbmFibGVSZWZsZWN0aW9uOiB0cnVlLFxuICogICBtYXhSZWZsZWN0aW9uSXRlcmF0aW9uczogMixcbiAqIH07XG4gKiBgYGBcbiAqL1xuZXhwb3J0IGludGVyZmFjZSBSbG1Db25maWcge1xuICAvKiogTWF4aW11bSByZWN1cnNpb24gZGVwdGggZm9yIHN1Yi1xdWVyaWVzIChkZWZhdWx0OiAzKSAqL1xuICBtYXhEZXB0aD86IG51bWJlcjtcblxuICAvKiogTWF4aW11bSBudW1iZXIgb2Ygc3ViLXF1ZXJpZXMgcGVyIGxldmVsIChkZWZhdWx0OiA1KSAqL1xuICBtYXhTdWJRdWVyaWVzPzogbnVtYmVyO1xuXG4gIC8qKiBUb2tlbiBidWRnZXQgZm9yIGdlbmVyYXRpb24gKGRlZmF1bHQ6IDQwOTYpICovXG4gIHRva2VuQnVkZ2V0PzogbnVtYmVyO1xuXG4gIC8qKiBFbmFibGUgcmVzcG9uc2UgY2FjaGluZyAoZGVmYXVsdDogdHJ1ZSkgKi9cbiAgZW5hYmxlQ2FjaGU/OiBib29sZWFuO1xuXG4gIC8qKiBDYWNoZSBUVEwgaW4gbWlsbGlzZWNvbmRzIChkZWZhdWx0OiAzMDAwMDAgPSA1IG1pbnV0ZXMpICovXG4gIGNhY2hlVHRsPzogbnVtYmVyO1xuXG4gIC8qKiBOdW1iZXIgb2YgbWVtb3J5IHNwYW5zIHRvIHJldHJpZXZlIChkZWZhdWx0OiAxMCkgKi9cbiAgcmV0cmlldmFsVG9wSz86IG51bWJlcjtcblxuICAvKiogTWluaW11bSBxdWFsaXR5IHNjb3JlIHRvIGFjY2VwdCBhbnN3ZXIgKGRlZmF1bHQ6IDAuNykgKi9cbiAgbWluUXVhbGl0eVNjb3JlPzogbnVtYmVyO1xuXG4gIC8qKiBFbmFibGUgc2VsZi1yZWZsZWN0aW9uIGxvb3AgKGRlZmF1bHQ6IGZhbHNlKSAqL1xuICBlbmFibGVSZWZsZWN0aW9uPzogYm9vbGVhbjtcblxuICAvKiogTWF4aW11bSByZWZsZWN0aW9uIGl0ZXJhdGlvbnMgKGRlZmF1bHQ6IDIpICovXG4gIG1heFJlZmxlY3Rpb25JdGVyYXRpb25zPzogbnVtYmVyO1xufVxuXG4vKipcbiAqIEFuc3dlciBwcm9kdWNlZCBieSB0aGUgUkxNIGNvbnRyb2xsZXJcbiAqXG4gKiBAZXhhbXBsZVxuICogYGBgdHlwZXNjcmlwdFxuICogY29uc3QgYW5zd2VyOiBSbG1BbnN3ZXIgPSB7XG4gKiAgIHRleHQ6ICdNYWNoaW5lIGxlYXJuaW5nIGlzIGEgc3Vic2V0IG9mIGFydGlmaWNpYWwgaW50ZWxsaWdlbmNlLi4uJyxcbiAqICAgY29uZmlkZW5jZTogMC45MixcbiAqICAgcXVhbGl0eVNjb3JlOiAwLjg4LFxuICogICBzb3VyY2VzOiBbXG4gKiAgICAgeyBpZDogJ21lbS0xJywgdGV4dDogJ01MIGRlZmluaXRpb24gZnJvbSB0ZXh0Ym9vaycsIHNpbWlsYXJpdHlTY29yZTogMC45NSwgbWV0YWRhdGE6IHt9IH0sXG4gKiAgIF0sXG4gKiAgIHN1YlF1ZXJpZXM6IFtcbiAqICAgICB7IHF1ZXJ5OiAnV2hhdCBpcyBhcnRpZmljaWFsIGludGVsbGlnZW5jZT8nLCBhbnN3ZXI6ICdBSSBpcy4uLicsIGRlcHRoOiAxIH0sXG4gKiAgIF0sXG4gKiAgIHRva2VuVXNhZ2U6IHsgcHJvbXB0OiA1MTIsIGNvbXBsZXRpb246IDI1NiwgdG90YWw6IDc2OCB9LFxuICogICBjYWNoZWQ6IGZhbHNlLFxuICogfTtcbiAqIGBgYFxuICovXG5leHBvcnQgaW50ZXJmYWNlIFJsbUFuc3dlciB7XG4gIC8qKiBUaGUgZ2VuZXJhdGVkIGFuc3dlciB0ZXh0ICovXG4gIHRleHQ6IHN0cmluZztcblxuICAvKiogT3ZlcmFsbCBjb25maWRlbmNlIGluIHRoZSBhbnN3ZXIgKDAuMCAtIDEuMCkgKi9cbiAgY29uZmlkZW5jZTogbnVtYmVyO1xuXG4gIC8qKiBRdWFsaXR5IHNjb3JlIGJhc2VkIG9uIHNvdXJjZSBjb3ZlcmFnZSBhbmQgY29oZXJlbmNlICgwLjAgLSAxLjApICovXG4gIHF1YWxpdHlTY29yZTogbnVtYmVyO1xuXG4gIC8qKiBNZW1vcnkgc3BhbnMgdXNlZCB0byBnZW5lcmF0ZSB0aGUgYW5zd2VyICovXG4gIHNvdXJjZXM6IE1lbW9yeVNwYW5bXTtcblxuICAvKiogU3ViLXF1ZXJpZXMgZ2VuZXJhdGVkIGFuZCBhbnN3ZXJlZCAoaWYgcmVjdXJzaXZlKSAqL1xuICBzdWJRdWVyaWVzPzogU3ViUXVlcnlbXTtcblxuICAvKiogVG9rZW4gdXNhZ2Ugc3RhdGlzdGljcyAqL1xuICB0b2tlblVzYWdlOiBUb2tlblVzYWdlO1xuXG4gIC8qKiBXaGV0aGVyIHRoaXMgYW5zd2VyIHdhcyBzZXJ2ZWQgZnJvbSBjYWNoZSAqL1xuICBjYWNoZWQ6IGJvb2xlYW47XG59XG5cbi8qKlxuICogQSBzcGFuIG9mIG1lbW9yeSByZXRyaWV2ZWQgZm9yIGNvbnRleHRcbiAqXG4gKiBAZXhhbXBsZVxuICogYGBgdHlwZXNjcmlwdFxuICogY29uc3Qgc3BhbjogTWVtb3J5U3BhbiA9IHtcbiAqICAgaWQ6ICdtZW0tYWJjMTIzJyxcbiAqICAgdGV4dDogJ1JlbGV2YW50IGNvbnRleHQgZnJvbSBtZW1vcnkuLi4nLFxuICogICBzaW1pbGFyaXR5U2NvcmU6IDAuODksXG4gKiAgIHNvdXJjZTogJ2RvY3VtZW50YXRpb24nLFxuICogICBtZXRhZGF0YTogeyB0aW1lc3RhbXA6IERhdGUubm93KCksIGNhdGVnb3J5OiAndGVjaG5pY2FsJyB9LFxuICogfTtcbiAqIGBgYFxuICovXG5leHBvcnQgaW50ZXJmYWNlIE1lbW9yeVNwYW4ge1xuICAvKiogVW5pcXVlIGlkZW50aWZpZXIgZm9yIHRoZSBtZW1vcnkgc3BhbiAqL1xuICBpZDogc3RyaW5nO1xuXG4gIC8qKiBUaGUgdGV4dCBjb250ZW50IG9mIHRoZSBtZW1vcnkgc3BhbiAqL1xuICB0ZXh0OiBzdHJpbmc7XG5cbiAgLyoqIENvc2luZSBzaW1pbGFyaXR5IHNjb3JlIHRvIHRoZSBxdWVyeSAoMC4wIC0gMS4wKSAqL1xuICBzaW1pbGFyaXR5U2NvcmU6IG51bWJlcjtcblxuICAvKiogT3B0aW9uYWwgc291cmNlIGlkZW50aWZpZXIgKGUuZy4sIGRvY3VtZW50IG5hbWUsIFVSTCkgKi9cbiAgc291cmNlPzogc3RyaW5nO1xuXG4gIC8qKiBBZGRpdGlvbmFsIG1ldGFkYXRhIGFzc29jaWF0ZWQgd2l0aCB0aGlzIHNwYW4gKi9cbiAgbWV0YWRhdGE6IFJlY29yZDxzdHJpbmcsIHVua25vd24+O1xufVxuXG4vKipcbiAqIEEgc3ViLXF1ZXJ5IGdlbmVyYXRlZCBkdXJpbmcgcmVjdXJzaXZlIHJldHJpZXZhbFxuICovXG5leHBvcnQgaW50ZXJmYWNlIFN1YlF1ZXJ5IHtcbiAgLyoqIFRoZSBnZW5lcmF0ZWQgc3ViLXF1ZXJ5IHRleHQgKi9cbiAgcXVlcnk6IHN0cmluZztcblxuICAvKiogVGhlIGFuc3dlciB0byB0aGUgc3ViLXF1ZXJ5ICovXG4gIGFuc3dlcjogc3RyaW5nO1xuXG4gIC8qKiBSZWN1cnNpb24gZGVwdGggYXQgd2hpY2ggdGhpcyBzdWItcXVlcnkgd2FzIGdlbmVyYXRlZCAqL1xuICBkZXB0aDogbnVtYmVyO1xufVxuXG4vKipcbiAqIFRva2VuIHVzYWdlIHN0YXRpc3RpY3MgZm9yIGEgcXVlcnlcbiAqL1xuZXhwb3J0IGludGVyZmFjZSBUb2tlblVzYWdlIHtcbiAgLyoqIFRva2VucyB1c2VkIGluIHRoZSBwcm9tcHQgKGluY2x1ZGluZyBjb250ZXh0KSAqL1xuICBwcm9tcHQ6IG51bWJlcjtcblxuICAvKiogVG9rZW5zIGdlbmVyYXRlZCBpbiB0aGUgY29tcGxldGlvbiAqL1xuICBjb21wbGV0aW9uOiBudW1iZXI7XG5cbiAgLyoqIFRvdGFsIHRva2VucyB1c2VkIChwcm9tcHQgKyBjb21wbGV0aW9uKSAqL1xuICB0b3RhbDogbnVtYmVyO1xufVxuXG4vKipcbiAqIFN0cmVhbWluZyB0b2tlbiBldmVudFxuICpcbiAqIERpc2NyaW1pbmF0ZWQgdW5pb24gZm9yIHN0cmVhbWluZyByZXNwb25zZXM6XG4gKiAtIGB0eXBlOiAndG9rZW4nYCAtIEEgcGFydGlhbCB0b2tlbiB3YXMgZ2VuZXJhdGVkXG4gKiAtIGB0eXBlOiAnZG9uZSdgIC0gR2VuZXJhdGlvbiBjb21wbGV0ZSB3aXRoIGZpbmFsIGFuc3dlclxuICpcbiAqIEBleGFtcGxlXG4gKiBgYGB0eXBlc2NyaXB0XG4gKiBmb3IgYXdhaXQgKGNvbnN0IGV2ZW50IG9mIGNvbnRyb2xsZXIucXVlcnlTdHJlYW0oJ1doYXQgaXMgQUk/JykpIHtcbiAqICAgaWYgKGV2ZW50LnR5cGUgPT09ICd0b2tlbicpIHtcbiAqICAgICBwcm9jZXNzLnN0ZG91dC53cml0ZShldmVudC50ZXh0KTtcbiAqICAgfSBlbHNlIHtcbiAqICAgICBjb25zb2xlLmxvZygnXFxuXFxuRmluYWwgYW5zd2VyOicsIGV2ZW50LmFuc3dlci50ZXh0KTtcbiAqICAgfVxuICogfVxuICogYGBgXG4gKi9cbmV4cG9ydCB0eXBlIFN0cmVhbVRva2VuID1cbiAgfCB7XG4gICAgICAvKiogVG9rZW4gZXZlbnQgdHlwZSAqL1xuICAgICAgdHlwZTogJ3Rva2VuJztcbiAgICAgIC8qKiBUaGUgcGFydGlhbCB0ZXh0IHRva2VuICovXG4gICAgICB0ZXh0OiBzdHJpbmc7XG4gICAgICAvKiogQWx3YXlzIGZhbHNlIGZvciB0b2tlbiBldmVudHMgKi9cbiAgICAgIGRvbmU6IGZhbHNlO1xuICAgIH1cbiAgfCB7XG4gICAgICAvKiogRG9uZSBldmVudCB0eXBlICovXG4gICAgICB0eXBlOiAnZG9uZSc7XG4gICAgICAvKiogVGhlIGNvbXBsZXRlIGFuc3dlciAqL1xuICAgICAgYW5zd2VyOiBSbG1BbnN3ZXI7XG4gICAgICAvKiogQWx3YXlzIHRydWUgZm9yIGRvbmUgZXZlbnRzICovXG4gICAgICBkb25lOiB0cnVlO1xuICAgIH07XG5cbi8qKlxuICogSW50ZXJuYWwgY2FjaGUgZW50cnkgZm9yIFJMTSBhbnN3ZXJzXG4gKi9cbmV4cG9ydCBpbnRlcmZhY2UgUmxtQ2FjaGVFbnRyeSB7XG4gIC8qKiBUaGUgY2FjaGVkIGFuc3dlciAqL1xuICBhbnN3ZXI6IFJsbUFuc3dlcjtcblxuICAvKiogVGltZXN0YW1wIHdoZW4gdGhlIGVudHJ5IHdhcyBjYWNoZWQgKi9cbiAgdGltZXN0YW1wOiBudW1iZXI7XG5cbiAgLyoqIFF1ZXJ5IGhhc2ggZm9yIGNhY2hlIGtleSAqL1xuICBxdWVyeUhhc2g6IHN0cmluZztcbn1cblxuLyoqXG4gKiBSZWZsZWN0aW9uIHJlc3VsdCBmcm9tIHNlbGYtZXZhbHVhdGlvbiBsb29wXG4gKi9cbmV4cG9ydCBpbnRlcmZhY2UgUmVmbGVjdGlvblJlc3VsdCB7XG4gIC8qKiBXaGV0aGVyIHRoZSBhbnN3ZXIgcGFzc2VkIHJlZmxlY3Rpb24gY3JpdGVyaWEgKi9cbiAgcGFzc2VkOiBib29sZWFuO1xuXG4gIC8qKiBDcml0aXF1ZSBvZiB0aGUgY3VycmVudCBhbnN3ZXIgKi9cbiAgY3JpdGlxdWU/OiBzdHJpbmc7XG5cbiAgLyoqIFN1Z2dlc3RlZCBpbXByb3ZlbWVudHMgKi9cbiAgc3VnZ2VzdGlvbnM/OiBzdHJpbmdbXTtcblxuICAvKiogVXBkYXRlZCBxdWFsaXR5IHNjb3JlIGFmdGVyIHJlZmxlY3Rpb24gKi9cbiAgdXBkYXRlZFNjb3JlOiBudW1iZXI7XG5cbiAgLyoqIE51bWJlciBvZiByZWZsZWN0aW9uIGl0ZXJhdGlvbnMgcGVyZm9ybWVkICovXG4gIGl0ZXJhdGlvbnM6IG51bWJlcjtcbn1cbiJdfQ==
@@ -0,0 +1,102 @@
1
+ /**
2
+ * Embedding Quality Benchmark for RuvLTRA Models
3
+ *
4
+ * Tests embedding quality for Claude Code use cases:
5
+ * - Code similarity detection
6
+ * - Task clustering
7
+ * - Semantic search accuracy
8
+ */
9
+ export interface EmbeddingPair {
10
+ id: string;
11
+ text1: string;
12
+ text2: string;
13
+ similarity: 'high' | 'medium' | 'low' | 'none';
14
+ category: string;
15
+ }
16
+ export interface EmbeddingResult {
17
+ pairId: string;
18
+ expectedSimilarity: string;
19
+ computedScore: number;
20
+ correct: boolean;
21
+ latencyMs: number;
22
+ }
23
+ export interface ClusterTestCase {
24
+ id: string;
25
+ items: string[];
26
+ expectedCluster: string;
27
+ }
28
+ export interface EmbeddingBenchmarkResults {
29
+ similarityAccuracy: number;
30
+ similarityByCategory: Record<string, number>;
31
+ avgSimilarityLatencyMs: number;
32
+ clusterPurity: number;
33
+ silhouetteScore: number;
34
+ searchMRR: number;
35
+ searchNDCG: number;
36
+ similarityResults: EmbeddingResult[];
37
+ totalPairs: number;
38
+ }
39
+ /**
40
+ * Ground truth similarity pairs for testing
41
+ * Tests whether embeddings correctly capture semantic similarity
42
+ */
43
+ export declare const SIMILARITY_TEST_PAIRS: EmbeddingPair[];
44
+ /**
45
+ * Search relevance test cases
46
+ * Query + documents with relevance scores
47
+ */
48
+ export interface SearchTestCase {
49
+ id: string;
50
+ query: string;
51
+ documents: {
52
+ text: string;
53
+ relevance: number;
54
+ }[];
55
+ }
56
+ export declare const SEARCH_TEST_CASES: SearchTestCase[];
57
+ /**
58
+ * Cluster test cases - items that should cluster together
59
+ */
60
+ export declare const CLUSTER_TEST_CASES: ClusterTestCase[];
61
+ /**
62
+ * Check if computed similarity matches expected category
63
+ */
64
+ export declare function isCorrectSimilarity(expected: 'high' | 'medium' | 'low' | 'none', computed: number): boolean;
65
+ /**
66
+ * Calculate Mean Reciprocal Rank for search results
67
+ */
68
+ export declare function calculateMRR(rankings: {
69
+ relevant: boolean;
70
+ }[][]): number;
71
+ /**
72
+ * Calculate NDCG for search results
73
+ */
74
+ export declare function calculateNDCG(results: {
75
+ relevance: number;
76
+ }[], idealOrder: {
77
+ relevance: number;
78
+ }[]): number;
79
+ /**
80
+ * Calculate silhouette score for clustering
81
+ */
82
+ export declare function calculateSilhouette(embeddings: number[][], labels: number[]): number;
83
+ /**
84
+ * Run the embedding benchmark
85
+ */
86
+ export declare function runEmbeddingBenchmark(embedder: (text: string) => number[], similarityFn: (a: number[], b: number[]) => number): EmbeddingBenchmarkResults;
87
+ /**
88
+ * Format embedding benchmark results for display
89
+ */
90
+ export declare function formatEmbeddingResults(results: EmbeddingBenchmarkResults): string;
91
+ declare const _default: {
92
+ SIMILARITY_TEST_PAIRS: EmbeddingPair[];
93
+ SEARCH_TEST_CASES: SearchTestCase[];
94
+ CLUSTER_TEST_CASES: ClusterTestCase[];
95
+ runEmbeddingBenchmark: typeof runEmbeddingBenchmark;
96
+ formatEmbeddingResults: typeof formatEmbeddingResults;
97
+ isCorrectSimilarity: typeof isCorrectSimilarity;
98
+ calculateMRR: typeof calculateMRR;
99
+ calculateNDCG: typeof calculateNDCG;
100
+ };
101
+ export default _default;
102
+ //# sourceMappingURL=embedding-benchmark.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embedding-benchmark.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/embedding-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;IAC/C,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IAExC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,sBAAsB,EAAE,MAAM,CAAC;IAG/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IAGxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IAGnB,iBAAiB,EAAE,eAAe,EAAE,CAAC;IACrC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,EAAE,aAAa,EA8ChD,CAAC;AAEF;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAClD;AAED,eAAO,MAAM,iBAAiB,EAAE,cAAc,EAwD7C,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,kBAAkB,EAAE,eAAe,EAwD/C,CAAC;AAYF;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,EAC5C,QAAQ,EAAE,MAAM,GACf,OAAO,CAGT;AAED;;GAEG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE;IAAE,QAAQ,EAAE,OAAO,CAAA;CAAE,EAAE,EAAE,GAClC,MAAM,CASR;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,EAAE,EAChC,UAAU,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,EAAE,GAClC,MAAM,CAUR;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,MAAM,EAAE,MAAM,EAAE,GACf,MAAM,CA8CR;AAUD;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,EACpC,YAAY,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,MAAM,GACjD,yBAAyB,CA0G3B;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,yBAAyB,GAAG,MAAM,CAyDjF;;;;;;;;;;;AAED,wBASE"}