@ruvector/ruvllm 0.2.4 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +413 -293
- package/bin/cli.js +619 -2
- package/dist/cjs/benchmarks/embedding-benchmark.d.ts +102 -0
- package/dist/cjs/benchmarks/embedding-benchmark.d.ts.map +1 -0
- package/dist/cjs/benchmarks/embedding-benchmark.js +436 -0
- package/dist/cjs/benchmarks/index.d.ts +34 -0
- package/dist/cjs/benchmarks/index.d.ts.map +1 -0
- package/dist/cjs/benchmarks/index.js +132 -0
- package/dist/cjs/benchmarks/model-comparison.d.ts +71 -0
- package/dist/cjs/benchmarks/model-comparison.d.ts.map +1 -0
- package/dist/cjs/benchmarks/model-comparison.js +476 -0
- package/dist/cjs/benchmarks/routing-benchmark.d.ts +70 -0
- package/dist/cjs/benchmarks/routing-benchmark.d.ts.map +1 -0
- package/dist/cjs/benchmarks/routing-benchmark.js +289 -0
- package/dist/cjs/contrastive.d.ts +229 -0
- package/dist/cjs/contrastive.d.ts.map +1 -0
- package/dist/cjs/contrastive.js +589 -0
- package/dist/cjs/engine.d.ts +3 -0
- package/dist/cjs/engine.d.ts.map +1 -1
- package/dist/cjs/engine.js +26 -6
- package/dist/cjs/index.d.ts +4 -0
- package/dist/cjs/index.d.ts.map +1 -1
- package/dist/cjs/index.js +9 -1
- package/dist/cjs/models.d.ts +129 -0
- package/dist/cjs/models.d.ts.map +1 -0
- package/dist/cjs/models.js +323 -0
- package/dist/cjs/native.d.ts +3 -3
- package/dist/cjs/native.d.ts.map +1 -1
- package/dist/cjs/native.js +1 -1
- package/dist/cjs/rlm/controller.d.ts +245 -0
- package/dist/cjs/rlm/controller.d.ts.map +1 -0
- package/dist/cjs/rlm/controller.js +554 -0
- package/dist/cjs/rlm/index.d.ts +62 -0
- package/dist/cjs/rlm/index.d.ts.map +1 -0
- package/dist/cjs/rlm/index.js +97 -0
- package/dist/cjs/rlm/training.d.ts +333 -0
- package/dist/cjs/rlm/training.d.ts.map +1 -0
- package/dist/cjs/rlm/training.js +680 -0
- package/dist/cjs/rlm/types.d.ts +189 -0
- package/dist/cjs/rlm/types.d.ts.map +1 -0
- package/dist/cjs/rlm/types.js +10 -0
- package/dist/esm/benchmarks/embedding-benchmark.d.ts +102 -0
- package/dist/esm/benchmarks/embedding-benchmark.d.ts.map +1 -0
- package/dist/esm/benchmarks/embedding-benchmark.js +427 -0
- package/dist/esm/benchmarks/index.d.ts +34 -0
- package/dist/esm/benchmarks/index.d.ts.map +1 -0
- package/dist/esm/benchmarks/index.js +110 -0
- package/dist/esm/benchmarks/model-comparison.d.ts +71 -0
- package/dist/esm/benchmarks/model-comparison.d.ts.map +1 -0
- package/dist/esm/benchmarks/model-comparison.js +434 -0
- package/dist/esm/benchmarks/routing-benchmark.d.ts +70 -0
- package/dist/esm/benchmarks/routing-benchmark.d.ts.map +1 -0
- package/dist/esm/benchmarks/routing-benchmark.js +283 -0
- package/dist/esm/contrastive.d.ts +229 -0
- package/dist/esm/contrastive.d.ts.map +1 -0
- package/dist/esm/contrastive.js +578 -0
- package/dist/esm/engine.d.ts +3 -0
- package/dist/esm/engine.d.ts.map +1 -1
- package/dist/esm/engine.js +26 -6
- package/dist/esm/index.d.ts +4 -0
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +9 -1
- package/dist/esm/models.d.ts +129 -0
- package/dist/esm/models.d.ts.map +1 -0
- package/dist/esm/models.js +282 -0
- package/dist/esm/native.d.ts +3 -3
- package/dist/esm/native.d.ts.map +1 -1
- package/dist/esm/native.js +1 -1
- package/dist/esm/rlm/controller.d.ts +245 -0
- package/dist/esm/rlm/controller.d.ts.map +1 -0
- package/dist/esm/rlm/controller.js +550 -0
- package/dist/esm/rlm/index.d.ts +62 -0
- package/dist/esm/rlm/index.d.ts.map +1 -0
- package/dist/esm/rlm/index.js +71 -0
- package/dist/esm/rlm/training.d.ts +333 -0
- package/dist/esm/rlm/training.d.ts.map +1 -0
- package/dist/esm/rlm/training.js +672 -0
- package/dist/esm/rlm/types.d.ts +189 -0
- package/dist/esm/rlm/types.d.ts.map +1 -0
- package/dist/esm/rlm/types.js +9 -0
- package/package.json +13 -8
- package/scripts/ensemble-model-compare.js +287 -0
- package/scripts/huggingface/README.md +537 -0
- package/scripts/huggingface/publish.sh +112 -0
- package/scripts/hybrid-model-compare.js +373 -0
- package/scripts/improved-model-compare.js +288 -0
- package/scripts/optimized-model-compare.js +364 -0
- package/scripts/real-model-compare.js +280 -0
- package/scripts/training/agentic-flow-capabilities.json +329 -0
- package/scripts/training/claude-code-synth.js +505 -0
- package/scripts/training/claude-flow-capabilities.json +1322 -0
- package/scripts/training/claude-hard-negatives.js +222 -0
- package/scripts/training/contrastive-finetune.js +517 -0
- package/scripts/training/ecosystem-triplets.jsonl +1467 -0
- package/scripts/training/generate-ecosystem-triplets.js +1324 -0
- package/scripts/training/rlm-dataset.js +671 -0
- package/scripts/training/routing-dataset.js +634 -0
- package/scripts/training/ruvector-capabilities.json +798 -0
- package/scripts/training/validate-ecosystem.js +381 -0
- package/scripts/training/validation-results.json +23 -0
|
@@ -0,0 +1,671 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* RLM Training Dataset Generator
|
|
5
|
+
*
|
|
6
|
+
* Generates synthetic RLM training examples for task decomposition,
|
|
7
|
+
* answer synthesis, and agent routing. Integrates with ReasoningBank
|
|
8
|
+
* patterns for realistic training data.
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* node rlm-dataset.js [--output <file>] [--count <n>] [--quality <min>]
|
|
12
|
+
*
|
|
13
|
+
* Example:
|
|
14
|
+
* node rlm-dataset.js --output rlm-training.json --count 1000 --quality 0.7
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const fs = require('fs');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const crypto = require('crypto');
|
|
20
|
+
|
|
21
|
+
// Import routing dataset for agent definitions
|
|
22
|
+
const { AGENT_TRAINING_DATA, getDatasetStats } = require('./routing-dataset');
|
|
23
|
+
|
|
24
|
+
// =============================================================================
|
|
25
|
+
// Configuration
|
|
26
|
+
// =============================================================================
|
|
27
|
+
|
|
28
|
+
const DEFAULT_CONFIG = {
|
|
29
|
+
outputFile: 'rlm-training-dataset.json',
|
|
30
|
+
exampleCount: 500,
|
|
31
|
+
minQuality: 0.6,
|
|
32
|
+
hardNegativeRatio: 0.3,
|
|
33
|
+
maxDecompositionDepth: 5,
|
|
34
|
+
embeddingDim: 768,
|
|
35
|
+
seed: 42,
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
// =============================================================================
|
|
39
|
+
// Decomposition Strategy Definitions
|
|
40
|
+
// =============================================================================
|
|
41
|
+
|
|
42
|
+
const DECOMPOSITION_STRATEGIES = {
|
|
43
|
+
sequential: {
|
|
44
|
+
name: 'sequential',
|
|
45
|
+
description: 'Steps executed in order, each depending on the previous',
|
|
46
|
+
complexityWeight: 1.5,
|
|
47
|
+
},
|
|
48
|
+
parallel: {
|
|
49
|
+
name: 'parallel',
|
|
50
|
+
description: 'Independent sub-queries that can run concurrently',
|
|
51
|
+
complexityWeight: 1.5,
|
|
52
|
+
},
|
|
53
|
+
hierarchical: {
|
|
54
|
+
name: 'hierarchical',
|
|
55
|
+
description: 'Tree structure with parent-child dependencies',
|
|
56
|
+
complexityWeight: 2.0,
|
|
57
|
+
},
|
|
58
|
+
'dag-based': {
|
|
59
|
+
name: 'dag-based',
|
|
60
|
+
description: 'Complex DAG with arbitrary dependencies',
|
|
61
|
+
complexityWeight: 3.0,
|
|
62
|
+
},
|
|
63
|
+
iterative: {
|
|
64
|
+
name: 'iterative',
|
|
65
|
+
description: 'Query -> result -> refined query cycles',
|
|
66
|
+
complexityWeight: 2.5,
|
|
67
|
+
},
|
|
68
|
+
none: {
|
|
69
|
+
name: 'none',
|
|
70
|
+
description: 'Simple query needing no decomposition',
|
|
71
|
+
complexityWeight: 1.0,
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
// =============================================================================
|
|
76
|
+
// Complex Query Templates
|
|
77
|
+
// =============================================================================
|
|
78
|
+
|
|
79
|
+
const COMPLEX_QUERY_TEMPLATES = [
|
|
80
|
+
// Multi-step development tasks
|
|
81
|
+
{
|
|
82
|
+
template: 'Build a {feature} for the {system} that includes {requirement1} and {requirement2}',
|
|
83
|
+
decomposition: ['research', 'design', 'implement', 'test', 'document'],
|
|
84
|
+
strategy: 'sequential',
|
|
85
|
+
agents: ['researcher', 'architect', 'coder', 'tester', 'documenter'],
|
|
86
|
+
quality: 0.9,
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
template: 'Refactor the {component} to use {pattern} while maintaining backward compatibility',
|
|
90
|
+
decomposition: ['analyze', 'design_refactor', 'implement_changes', 'write_tests', 'review'],
|
|
91
|
+
strategy: 'sequential',
|
|
92
|
+
agents: ['researcher', 'architect', 'coder', 'tester', 'reviewer'],
|
|
93
|
+
quality: 0.85,
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
template: 'Add {feature} support to the application with proper error handling and logging',
|
|
97
|
+
decomposition: ['requirements', 'implementation', 'error_handling', 'logging', 'testing'],
|
|
98
|
+
strategy: 'parallel',
|
|
99
|
+
agents: ['researcher', 'coder', 'coder', 'coder', 'tester'],
|
|
100
|
+
quality: 0.88,
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
template: 'Investigate and fix the performance issue in {component} causing {symptom}',
|
|
104
|
+
decomposition: ['investigate', 'identify_cause', 'design_fix', 'implement_fix', 'verify'],
|
|
105
|
+
strategy: 'sequential',
|
|
106
|
+
agents: ['researcher', 'debugger', 'architect', 'coder', 'tester'],
|
|
107
|
+
quality: 0.82,
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
template: 'Create a comprehensive API for {domain} with authentication, rate limiting, and documentation',
|
|
111
|
+
decomposition: ['design_api', 'implement_endpoints', 'add_auth', 'add_rate_limiting', 'write_docs'],
|
|
112
|
+
strategy: 'hierarchical',
|
|
113
|
+
agents: ['architect', 'coder', 'security-architect', 'coder', 'api-docs'],
|
|
114
|
+
quality: 0.9,
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
template: 'Migrate the {system} from {old_tech} to {new_tech} with minimal downtime',
|
|
118
|
+
decomposition: ['plan_migration', 'prepare_infrastructure', 'migrate_data', 'switch_traffic', 'cleanup'],
|
|
119
|
+
strategy: 'sequential',
|
|
120
|
+
agents: ['planner', 'devops', 'coder', 'devops', 'devops'],
|
|
121
|
+
quality: 0.87,
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
template: 'Implement a {algorithm} for {use_case} optimized for {constraint}',
|
|
125
|
+
decomposition: ['research_approaches', 'design_algorithm', 'implement', 'optimize', 'benchmark'],
|
|
126
|
+
strategy: 'sequential',
|
|
127
|
+
agents: ['researcher', 'architect', 'coder', 'optimizer', 'tester'],
|
|
128
|
+
quality: 0.85,
|
|
129
|
+
},
|
|
130
|
+
{
|
|
131
|
+
template: 'Set up CI/CD pipeline for {project} with automated testing, security scanning, and deployment',
|
|
132
|
+
decomposition: ['design_pipeline', 'setup_tests', 'add_security', 'configure_deployment', 'document'],
|
|
133
|
+
strategy: 'dag-based',
|
|
134
|
+
agents: ['devops', 'tester', 'security-architect', 'devops', 'documenter'],
|
|
135
|
+
quality: 0.88,
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
template: 'Review and improve the security of {component} following OWASP guidelines',
|
|
139
|
+
decomposition: ['audit_current', 'identify_vulnerabilities', 'plan_fixes', 'implement_fixes', 'verify'],
|
|
140
|
+
strategy: 'sequential',
|
|
141
|
+
agents: ['security-architect', 'security-architect', 'architect', 'coder', 'security-architect'],
|
|
142
|
+
quality: 0.9,
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
template: 'Design and implement a {data_structure} optimized for {operation} operations',
|
|
146
|
+
decomposition: ['research_options', 'design_structure', 'implement', 'write_tests', 'benchmark'],
|
|
147
|
+
strategy: 'sequential',
|
|
148
|
+
agents: ['researcher', 'architect', 'coder', 'tester', 'optimizer'],
|
|
149
|
+
quality: 0.85,
|
|
150
|
+
},
|
|
151
|
+
];
|
|
152
|
+
|
|
153
|
+
// Fill-in values for templates
|
|
154
|
+
const TEMPLATE_VALUES = {
|
|
155
|
+
feature: ['dark mode', 'real-time sync', 'offline support', 'multi-language', 'analytics dashboard', 'notification system', 'file upload', 'export functionality'],
|
|
156
|
+
system: ['web application', 'mobile app', 'backend service', 'CLI tool', 'browser extension', 'desktop app'],
|
|
157
|
+
requirement1: ['user authentication', 'data validation', 'error handling', 'caching', 'pagination', 'search'],
|
|
158
|
+
requirement2: ['logging', 'metrics', 'accessibility', 'responsive design', 'keyboard shortcuts', 'undo/redo'],
|
|
159
|
+
component: ['authentication module', 'payment service', 'user profile', 'notification system', 'search engine', 'data pipeline'],
|
|
160
|
+
pattern: ['async/await', 'event sourcing', 'CQRS', 'microservices', 'domain-driven design', 'hexagonal architecture'],
|
|
161
|
+
symptom: ['slow response times', 'high memory usage', 'database connection leaks', 'timeout errors', 'intermittent failures'],
|
|
162
|
+
domain: ['user management', 'inventory', 'payments', 'analytics', 'content management', 'scheduling'],
|
|
163
|
+
old_tech: ['MongoDB', 'Express.js', 'JavaScript', 'REST API', 'monolith', 'SQL Server'],
|
|
164
|
+
new_tech: ['PostgreSQL', 'NestJS', 'TypeScript', 'GraphQL', 'microservices', 'MongoDB'],
|
|
165
|
+
algorithm: ['search algorithm', 'sorting algorithm', 'caching strategy', 'load balancing', 'rate limiting', 'data compression'],
|
|
166
|
+
use_case: ['real-time search', 'batch processing', 'stream processing', 'recommendation engine', 'fraud detection'],
|
|
167
|
+
constraint: ['memory efficiency', 'low latency', 'high throughput', 'horizontal scaling', 'minimal dependencies'],
|
|
168
|
+
project: ['Node.js application', 'React frontend', 'Python service', 'Rust library', 'Go microservice'],
|
|
169
|
+
data_structure: ['priority queue', 'LRU cache', 'trie', 'bloom filter', 'skip list', 'B-tree'],
|
|
170
|
+
operation: ['insert', 'search', 'delete', 'range query', 'bulk update'],
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
// =============================================================================
|
|
174
|
+
// Hard Negative Pairs (Confusable Agent Combinations)
|
|
175
|
+
// =============================================================================
|
|
176
|
+
|
|
177
|
+
const HARD_NEGATIVE_PAIRS = [
|
|
178
|
+
['coder', 'debugger'],
|
|
179
|
+
['coder', 'refactorer'],
|
|
180
|
+
['researcher', 'reviewer'],
|
|
181
|
+
['tester', 'reviewer'],
|
|
182
|
+
['architect', 'planner'],
|
|
183
|
+
['documenter', 'api-docs'],
|
|
184
|
+
['optimizer', 'debugger'],
|
|
185
|
+
['devops', 'architect'],
|
|
186
|
+
['security-architect', 'reviewer'],
|
|
187
|
+
];
|
|
188
|
+
|
|
189
|
+
// =============================================================================
|
|
190
|
+
// Utility Functions
|
|
191
|
+
// =============================================================================
|
|
192
|
+
|
|
193
|
+
function generateUUID() {
|
|
194
|
+
if (crypto.randomUUID) {
|
|
195
|
+
return crypto.randomUUID();
|
|
196
|
+
}
|
|
197
|
+
// Fallback for older Node versions
|
|
198
|
+
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
|
|
199
|
+
const r = Math.random() * 16 | 0;
|
|
200
|
+
const v = c === 'x' ? r : (r & 0x3 | 0x8);
|
|
201
|
+
return v.toString(16);
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function seededRandom(seed) {
|
|
206
|
+
let state = seed;
|
|
207
|
+
return function() {
|
|
208
|
+
state = (state * 1103515245 + 12345) & 0x7fffffff;
|
|
209
|
+
return state / 0x7fffffff;
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function shuffle(array, rng) {
|
|
214
|
+
const shuffled = [...array];
|
|
215
|
+
for (let i = shuffled.length - 1; i > 0; i--) {
|
|
216
|
+
const j = Math.floor(rng() * (i + 1));
|
|
217
|
+
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
|
218
|
+
}
|
|
219
|
+
return shuffled;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function pickRandom(array, rng) {
|
|
223
|
+
return array[Math.floor(rng() * array.length)];
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function generateEmbedding(dim, rng) {
|
|
227
|
+
const embedding = [];
|
|
228
|
+
for (let i = 0; i < dim; i++) {
|
|
229
|
+
embedding.push(rng() * 2 - 1);
|
|
230
|
+
}
|
|
231
|
+
// Normalize
|
|
232
|
+
const norm = Math.sqrt(embedding.reduce((sum, v) => sum + v * v, 0));
|
|
233
|
+
return embedding.map(v => v / norm);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function isHardNegative(agent1, agent2) {
|
|
237
|
+
return HARD_NEGATIVE_PAIRS.some(
|
|
238
|
+
([a, b]) => (agent1 === a && agent2 === b) || (agent1 === b && agent2 === a)
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// =============================================================================
|
|
243
|
+
// Dataset Generation Functions
|
|
244
|
+
// =============================================================================
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Fill a template with random values
|
|
248
|
+
*/
|
|
249
|
+
function fillTemplate(template, rng) {
|
|
250
|
+
let filled = template;
|
|
251
|
+
for (const [key, values] of Object.entries(TEMPLATE_VALUES)) {
|
|
252
|
+
const regex = new RegExp(`\\{${key}\\}`, 'g');
|
|
253
|
+
filled = filled.replace(regex, () => pickRandom(values, rng));
|
|
254
|
+
}
|
|
255
|
+
return filled;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Generate a sub-query from a decomposition step
|
|
260
|
+
*/
|
|
261
|
+
function generateSubQuery(id, step, agent, rng, dependencies = []) {
|
|
262
|
+
const stepDescriptions = {
|
|
263
|
+
research: 'Research best practices and options for',
|
|
264
|
+
design: 'Design the architecture for',
|
|
265
|
+
implement: 'Implement the functionality for',
|
|
266
|
+
test: 'Write comprehensive tests for',
|
|
267
|
+
document: 'Document the implementation of',
|
|
268
|
+
analyze: 'Analyze the current state of',
|
|
269
|
+
design_refactor: 'Design the refactoring approach for',
|
|
270
|
+
implement_changes: 'Implement the refactored code for',
|
|
271
|
+
write_tests: 'Write regression tests for',
|
|
272
|
+
review: 'Review the code quality of',
|
|
273
|
+
requirements: 'Gather and analyze requirements for',
|
|
274
|
+
implementation: 'Build the core implementation of',
|
|
275
|
+
error_handling: 'Add error handling to',
|
|
276
|
+
logging: 'Implement logging for',
|
|
277
|
+
testing: 'Create test suite for',
|
|
278
|
+
investigate: 'Investigate the root cause of',
|
|
279
|
+
identify_cause: 'Identify the specific cause of',
|
|
280
|
+
design_fix: 'Design a fix for',
|
|
281
|
+
implement_fix: 'Implement the fix for',
|
|
282
|
+
verify: 'Verify the fix works for',
|
|
283
|
+
design_api: 'Design the API structure for',
|
|
284
|
+
implement_endpoints: 'Implement API endpoints for',
|
|
285
|
+
add_auth: 'Add authentication to',
|
|
286
|
+
add_rate_limiting: 'Implement rate limiting for',
|
|
287
|
+
write_docs: 'Write API documentation for',
|
|
288
|
+
plan_migration: 'Create migration plan for',
|
|
289
|
+
prepare_infrastructure: 'Prepare infrastructure for',
|
|
290
|
+
migrate_data: 'Migrate data for',
|
|
291
|
+
switch_traffic: 'Switch traffic for',
|
|
292
|
+
cleanup: 'Clean up old resources for',
|
|
293
|
+
research_approaches: 'Research possible approaches for',
|
|
294
|
+
design_algorithm: 'Design the algorithm for',
|
|
295
|
+
optimize: 'Optimize performance of',
|
|
296
|
+
benchmark: 'Benchmark performance of',
|
|
297
|
+
design_pipeline: 'Design CI/CD pipeline for',
|
|
298
|
+
setup_tests: 'Set up automated testing for',
|
|
299
|
+
add_security: 'Add security scanning to',
|
|
300
|
+
configure_deployment: 'Configure deployment for',
|
|
301
|
+
audit_current: 'Audit current security of',
|
|
302
|
+
identify_vulnerabilities: 'Identify vulnerabilities in',
|
|
303
|
+
plan_fixes: 'Plan security fixes for',
|
|
304
|
+
implement_fixes: 'Implement security fixes for',
|
|
305
|
+
research_options: 'Research implementation options for',
|
|
306
|
+
design_structure: 'Design data structure for',
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
const description = stepDescriptions[step] || `Process ${step} for`;
|
|
310
|
+
|
|
311
|
+
return {
|
|
312
|
+
id,
|
|
313
|
+
query: `${description} the component`,
|
|
314
|
+
expectedType: agent === 'coder' ? 'code' : agent === 'documenter' || agent === 'api-docs' ? 'documentation' : 'analysis',
|
|
315
|
+
dependencies,
|
|
316
|
+
recommendedAgent: agent,
|
|
317
|
+
complexity: 0.3 + rng() * 0.5,
|
|
318
|
+
context: null,
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Generate sub-answers for sub-queries
|
|
324
|
+
*/
|
|
325
|
+
function generateSubAnswers(subQueries, rng, successRate = 0.9) {
|
|
326
|
+
return subQueries.map(sq => ({
|
|
327
|
+
subQueryId: sq.id,
|
|
328
|
+
content: `Completed ${sq.query}`,
|
|
329
|
+
confidence: 0.7 + rng() * 0.3,
|
|
330
|
+
agent: sq.recommendedAgent,
|
|
331
|
+
latencyMs: Math.floor(100 + rng() * 2000),
|
|
332
|
+
quality: rng() < successRate ? 0.7 + rng() * 0.3 : 0.3 + rng() * 0.4,
|
|
333
|
+
success: rng() < successRate,
|
|
334
|
+
error: rng() >= successRate ? 'Partial completion' : null,
|
|
335
|
+
reasoning: null,
|
|
336
|
+
}));
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Generate dependencies based on strategy
|
|
341
|
+
*/
|
|
342
|
+
function generateDependencies(subQueries, strategy) {
|
|
343
|
+
switch (strategy) {
|
|
344
|
+
case 'sequential':
|
|
345
|
+
return subQueries.map((sq, i) => ({
|
|
346
|
+
...sq,
|
|
347
|
+
dependencies: i > 0 ? [i - 1] : [],
|
|
348
|
+
}));
|
|
349
|
+
|
|
350
|
+
case 'parallel':
|
|
351
|
+
return subQueries.map(sq => ({
|
|
352
|
+
...sq,
|
|
353
|
+
dependencies: [],
|
|
354
|
+
}));
|
|
355
|
+
|
|
356
|
+
case 'hierarchical':
|
|
357
|
+
// First query is root, others depend on it or previous
|
|
358
|
+
return subQueries.map((sq, i) => ({
|
|
359
|
+
...sq,
|
|
360
|
+
dependencies: i === 0 ? [] : i < 3 ? [0] : [Math.floor(i / 2)],
|
|
361
|
+
}));
|
|
362
|
+
|
|
363
|
+
case 'dag-based':
|
|
364
|
+
// Complex dependencies
|
|
365
|
+
return subQueries.map((sq, i) => {
|
|
366
|
+
const deps = [];
|
|
367
|
+
if (i > 0) deps.push(i - 1);
|
|
368
|
+
if (i > 2) deps.push(0);
|
|
369
|
+
return { ...sq, dependencies: deps };
|
|
370
|
+
});
|
|
371
|
+
|
|
372
|
+
case 'iterative':
|
|
373
|
+
return subQueries.map((sq, i) => ({
|
|
374
|
+
...sq,
|
|
375
|
+
dependencies: i > 0 ? [i - 1] : [],
|
|
376
|
+
}));
|
|
377
|
+
|
|
378
|
+
default:
|
|
379
|
+
return subQueries;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Generate a single RLM training example
|
|
385
|
+
*/
|
|
386
|
+
function generateExample(config, rng) {
|
|
387
|
+
const template = pickRandom(COMPLEX_QUERY_TEMPLATES, rng);
|
|
388
|
+
const query = fillTemplate(template.template, rng);
|
|
389
|
+
|
|
390
|
+
// Generate sub-queries
|
|
391
|
+
let subQueries = template.decomposition.map((step, i) =>
|
|
392
|
+
generateSubQuery(i, step, template.agents[i], rng)
|
|
393
|
+
);
|
|
394
|
+
|
|
395
|
+
// Add dependencies based on strategy
|
|
396
|
+
subQueries = generateDependencies(subQueries, template.strategy);
|
|
397
|
+
|
|
398
|
+
const strategyDef = DECOMPOSITION_STRATEGIES[template.strategy];
|
|
399
|
+
const totalComplexity = subQueries.reduce((sum, sq) => sum + sq.complexity, 0) * strategyDef.complexityWeight;
|
|
400
|
+
|
|
401
|
+
// Generate decomposition
|
|
402
|
+
const decomposition = {
|
|
403
|
+
subQueries,
|
|
404
|
+
strategy: template.strategy,
|
|
405
|
+
rationale: `Using ${template.strategy} strategy for optimal execution`,
|
|
406
|
+
totalComplexity,
|
|
407
|
+
success: true,
|
|
408
|
+
error: null,
|
|
409
|
+
};
|
|
410
|
+
|
|
411
|
+
// Generate sub-answers
|
|
412
|
+
const subAnswers = generateSubAnswers(subQueries, rng, template.quality);
|
|
413
|
+
|
|
414
|
+
// Calculate quality score
|
|
415
|
+
const avgSubAnswerQuality = subAnswers.length > 0
|
|
416
|
+
? subAnswers.reduce((sum, a) => sum + a.quality, 0) / subAnswers.length
|
|
417
|
+
: 0;
|
|
418
|
+
|
|
419
|
+
const success = subAnswers.every(a => a.success);
|
|
420
|
+
const qualityScore = success
|
|
421
|
+
? template.quality * (0.9 + rng() * 0.1)
|
|
422
|
+
: template.quality * (0.5 + rng() * 0.3);
|
|
423
|
+
|
|
424
|
+
// Generate trajectory metadata
|
|
425
|
+
const trajectory = {
|
|
426
|
+
sessionId: generateUUID(),
|
|
427
|
+
userId: null,
|
|
428
|
+
totalLatencyMs: subAnswers.reduce((sum, a) => sum + a.latencyMs, 0),
|
|
429
|
+
retries: success ? 0 : Math.floor(rng() * 3),
|
|
430
|
+
maxParallelism: template.strategy === 'parallel' ? subQueries.length : 1,
|
|
431
|
+
modelsUsed: ['ruvltra-0.5b'],
|
|
432
|
+
agentsInvoked: [...new Set(subQueries.map(sq => sq.recommendedAgent))],
|
|
433
|
+
toolsUsed: [],
|
|
434
|
+
attributes: {},
|
|
435
|
+
};
|
|
436
|
+
|
|
437
|
+
return {
|
|
438
|
+
id: generateUUID(),
|
|
439
|
+
query,
|
|
440
|
+
queryEmbedding: generateEmbedding(config.embeddingDim, rng),
|
|
441
|
+
decomposition,
|
|
442
|
+
subAnswers,
|
|
443
|
+
finalAnswer: `Successfully completed: ${query}`,
|
|
444
|
+
finalEmbedding: generateEmbedding(config.embeddingDim, rng),
|
|
445
|
+
qualityScore,
|
|
446
|
+
trajectory,
|
|
447
|
+
success,
|
|
448
|
+
lessons: success ? [] : ['Encountered challenges during execution'],
|
|
449
|
+
source: 'synthetic',
|
|
450
|
+
};
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/**
|
|
454
|
+
* Generate contrastive pairs from examples
|
|
455
|
+
*/
|
|
456
|
+
function generateContrastivePairs(examples, config, rng) {
|
|
457
|
+
const pairs = [];
|
|
458
|
+
const agents = Object.keys(AGENT_TRAINING_DATA);
|
|
459
|
+
|
|
460
|
+
for (const example of examples) {
|
|
461
|
+
for (const subQuery of example.decomposition.subQueries) {
|
|
462
|
+
if (!subQuery.recommendedAgent) continue;
|
|
463
|
+
|
|
464
|
+
const positiveAgent = subQuery.recommendedAgent;
|
|
465
|
+
|
|
466
|
+
for (const negativeAgent of agents) {
|
|
467
|
+
if (negativeAgent === positiveAgent) continue;
|
|
468
|
+
|
|
469
|
+
const isHard = isHardNegative(positiveAgent, negativeAgent);
|
|
470
|
+
|
|
471
|
+
// Apply hard negative ratio
|
|
472
|
+
const include = isHard
|
|
473
|
+
? rng() < config.hardNegativeRatio
|
|
474
|
+
: rng() < (1 - config.hardNegativeRatio) / agents.length;
|
|
475
|
+
|
|
476
|
+
if (include) {
|
|
477
|
+
pairs.push({
|
|
478
|
+
anchor: subQuery.query,
|
|
479
|
+
anchorEmbedding: example.queryEmbedding,
|
|
480
|
+
positiveAgent,
|
|
481
|
+
negativeAgent,
|
|
482
|
+
isHardNegative: isHard,
|
|
483
|
+
quality: example.qualityScore,
|
|
484
|
+
sourceId: example.id,
|
|
485
|
+
});
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
return pairs;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* Generate the complete RLM dataset
|
|
496
|
+
*/
|
|
497
|
+
function generateRlmDataset(config = {}) {
|
|
498
|
+
const finalConfig = { ...DEFAULT_CONFIG, ...config };
|
|
499
|
+
const rng = seededRandom(finalConfig.seed);
|
|
500
|
+
|
|
501
|
+
console.log('\n============================================================');
|
|
502
|
+
console.log(' RLM TRAINING DATASET GENERATOR');
|
|
503
|
+
console.log('============================================================\n');
|
|
504
|
+
|
|
505
|
+
console.log('Configuration:');
|
|
506
|
+
console.log(` Examples to generate: ${finalConfig.exampleCount}`);
|
|
507
|
+
console.log(` Minimum quality: ${finalConfig.minQuality}`);
|
|
508
|
+
console.log(` Hard negative ratio: ${finalConfig.hardNegativeRatio}`);
|
|
509
|
+
console.log(` Embedding dimension: ${finalConfig.embeddingDim}`);
|
|
510
|
+
console.log(` Random seed: ${finalConfig.seed}`);
|
|
511
|
+
console.log('');
|
|
512
|
+
|
|
513
|
+
// Generate examples
|
|
514
|
+
console.log('Generating examples...');
|
|
515
|
+
const examples = [];
|
|
516
|
+
let attempts = 0;
|
|
517
|
+
const maxAttempts = finalConfig.exampleCount * 2;
|
|
518
|
+
|
|
519
|
+
while (examples.length < finalConfig.exampleCount && attempts < maxAttempts) {
|
|
520
|
+
const example = generateExample(finalConfig, rng);
|
|
521
|
+
if (example.qualityScore >= finalConfig.minQuality) {
|
|
522
|
+
examples.push(example);
|
|
523
|
+
}
|
|
524
|
+
attempts++;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
console.log(` Generated ${examples.length} examples (${attempts} attempts)`);
|
|
528
|
+
|
|
529
|
+
// Generate contrastive pairs
|
|
530
|
+
console.log('\nGenerating contrastive pairs...');
|
|
531
|
+
const contrastivePairs = generateContrastivePairs(examples, finalConfig, rng);
|
|
532
|
+
console.log(` Generated ${contrastivePairs.length} contrastive pairs`);
|
|
533
|
+
|
|
534
|
+
// Calculate statistics
|
|
535
|
+
const stats = {
|
|
536
|
+
totalExamples: examples.length,
|
|
537
|
+
successfulExamples: examples.filter(e => e.success).length,
|
|
538
|
+
avgQuality: examples.reduce((sum, e) => sum + e.qualityScore, 0) / examples.length,
|
|
539
|
+
avgDecompositionDepth: examples.reduce((sum, e) => sum + e.decomposition.subQueries.length, 0) / examples.length,
|
|
540
|
+
strategyDistribution: {},
|
|
541
|
+
agentDistribution: {},
|
|
542
|
+
contrastivePairs: contrastivePairs.length,
|
|
543
|
+
hardNegativePairs: contrastivePairs.filter(p => p.isHardNegative).length,
|
|
544
|
+
};
|
|
545
|
+
|
|
546
|
+
// Calculate strategy distribution
|
|
547
|
+
for (const example of examples) {
|
|
548
|
+
const strategy = example.decomposition.strategy;
|
|
549
|
+
stats.strategyDistribution[strategy] = (stats.strategyDistribution[strategy] || 0) + 1;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
// Calculate agent distribution
|
|
553
|
+
for (const example of examples) {
|
|
554
|
+
for (const sq of example.decomposition.subQueries) {
|
|
555
|
+
const agent = sq.recommendedAgent;
|
|
556
|
+
stats.agentDistribution[agent] = (stats.agentDistribution[agent] || 0) + 1;
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
console.log('\n============================================================');
|
|
561
|
+
console.log(' DATASET STATISTICS');
|
|
562
|
+
console.log('============================================================\n');
|
|
563
|
+
console.log(`Total Examples: ${stats.totalExamples}`);
|
|
564
|
+
console.log(`Successful Examples: ${stats.successfulExamples}`);
|
|
565
|
+
console.log(`Average Quality: ${(stats.avgQuality * 100).toFixed(1)}%`);
|
|
566
|
+
console.log(`Avg Decomposition Depth: ${stats.avgDecompositionDepth.toFixed(1)} steps`);
|
|
567
|
+
console.log(`Contrastive Pairs: ${stats.contrastivePairs}`);
|
|
568
|
+
console.log(`Hard Negative Pairs: ${stats.hardNegativePairs} (${((stats.hardNegativePairs / stats.contrastivePairs) * 100).toFixed(1)}%)`);
|
|
569
|
+
|
|
570
|
+
console.log('\nStrategy Distribution:');
|
|
571
|
+
for (const [strategy, count] of Object.entries(stats.strategyDistribution)) {
|
|
572
|
+
console.log(` ${strategy.padEnd(15)} ${count} (${((count / stats.totalExamples) * 100).toFixed(1)}%)`);
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
console.log('\nAgent Distribution:');
|
|
576
|
+
const sortedAgents = Object.entries(stats.agentDistribution).sort((a, b) => b[1] - a[1]);
|
|
577
|
+
for (const [agent, count] of sortedAgents) {
|
|
578
|
+
console.log(` ${agent.padEnd(20)} ${count}`);
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
return {
|
|
582
|
+
examples,
|
|
583
|
+
contrastivePairs,
|
|
584
|
+
stats,
|
|
585
|
+
config: finalConfig,
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* Export dataset to JSON file
|
|
591
|
+
*/
|
|
592
|
+
function exportDataset(dataset, outputFile) {
|
|
593
|
+
const output = {
|
|
594
|
+
metadata: {
|
|
595
|
+
generatedAt: new Date().toISOString(),
|
|
596
|
+
version: '1.0.0',
|
|
597
|
+
config: dataset.config,
|
|
598
|
+
stats: dataset.stats,
|
|
599
|
+
},
|
|
600
|
+
examples: dataset.examples,
|
|
601
|
+
contrastivePairs: dataset.contrastivePairs,
|
|
602
|
+
};
|
|
603
|
+
|
|
604
|
+
fs.writeFileSync(outputFile, JSON.stringify(output, null, 2));
|
|
605
|
+
console.log(`\nDataset exported to: ${outputFile}`);
|
|
606
|
+
console.log(`File size: ${(fs.statSync(outputFile).size / 1024 / 1024).toFixed(2)} MB`);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
// =============================================================================
|
|
610
|
+
// CLI Interface
|
|
611
|
+
// =============================================================================
|
|
612
|
+
|
|
613
|
+
function parseArgs(args) {
|
|
614
|
+
const config = { ...DEFAULT_CONFIG };
|
|
615
|
+
|
|
616
|
+
for (let i = 0; i < args.length; i++) {
|
|
617
|
+
const arg = args[i];
|
|
618
|
+
|
|
619
|
+
if (arg === '--output' && args[i + 1]) {
|
|
620
|
+
config.outputFile = args[++i];
|
|
621
|
+
} else if (arg === '--count' && args[i + 1]) {
|
|
622
|
+
config.exampleCount = parseInt(args[++i], 10);
|
|
623
|
+
} else if (arg === '--quality' && args[i + 1]) {
|
|
624
|
+
config.minQuality = parseFloat(args[++i]);
|
|
625
|
+
} else if (arg === '--seed' && args[i + 1]) {
|
|
626
|
+
config.seed = parseInt(args[++i], 10);
|
|
627
|
+
} else if (arg === '--hard-ratio' && args[i + 1]) {
|
|
628
|
+
config.hardNegativeRatio = parseFloat(args[++i]);
|
|
629
|
+
} else if (arg === '--help' || arg === '-h') {
|
|
630
|
+
console.log(`
|
|
631
|
+
RLM Training Dataset Generator
|
|
632
|
+
|
|
633
|
+
Usage: node rlm-dataset.js [options]
|
|
634
|
+
|
|
635
|
+
Options:
|
|
636
|
+
--output <file> Output file path (default: rlm-training-dataset.json)
|
|
637
|
+
--count <n> Number of examples to generate (default: 500)
|
|
638
|
+
--quality <min> Minimum quality threshold (default: 0.6)
|
|
639
|
+
--seed <n> Random seed (default: 42)
|
|
640
|
+
--hard-ratio <r> Hard negative pair ratio (default: 0.3)
|
|
641
|
+
--help, -h Show this help message
|
|
642
|
+
|
|
643
|
+
Example:
|
|
644
|
+
node rlm-dataset.js --output my-dataset.json --count 1000 --quality 0.7
|
|
645
|
+
`);
|
|
646
|
+
process.exit(0);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
return config;
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
// Main execution
|
|
654
|
+
if (require.main === module) {
|
|
655
|
+
const config = parseArgs(process.argv.slice(2));
|
|
656
|
+
const dataset = generateRlmDataset(config);
|
|
657
|
+
exportDataset(dataset, config.outputFile);
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
// Module exports
|
|
661
|
+
module.exports = {
|
|
662
|
+
generateRlmDataset,
|
|
663
|
+
generateExample,
|
|
664
|
+
generateContrastivePairs,
|
|
665
|
+
exportDataset,
|
|
666
|
+
COMPLEX_QUERY_TEMPLATES,
|
|
667
|
+
DECOMPOSITION_STRATEGIES,
|
|
668
|
+
TEMPLATE_VALUES,
|
|
669
|
+
HARD_NEGATIVE_PAIRS,
|
|
670
|
+
DEFAULT_CONFIG,
|
|
671
|
+
};
|