@agentic-robotics/mcp 0.1.3 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark.d.ts +97 -0
- package/dist/benchmark.d.ts.map +1 -0
- package/dist/benchmark.js +330 -0
- package/dist/benchmark.js.map +1 -0
- package/dist/cli.d.ts +6 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +79 -0
- package/dist/cli.js.map +1 -0
- package/dist/enhanced-memory.d.ts +118 -0
- package/dist/enhanced-memory.d.ts.map +1 -0
- package/dist/enhanced-memory.js +296 -0
- package/dist/enhanced-memory.js.map +1 -0
- package/dist/enhanced-server.d.ts +122 -0
- package/dist/enhanced-server.d.ts.map +1 -0
- package/dist/enhanced-server.js +361 -0
- package/dist/enhanced-server.js.map +1 -0
- package/dist/flow-orchestrator.d.ts +75 -0
- package/dist/flow-orchestrator.d.ts.map +1 -0
- package/dist/flow-orchestrator.js +350 -0
- package/dist/flow-orchestrator.js.map +1 -0
- package/dist/hybrid-benchmark.d.ts +44 -0
- package/dist/hybrid-benchmark.d.ts.map +1 -0
- package/dist/hybrid-benchmark.js +232 -0
- package/dist/hybrid-benchmark.js.map +1 -0
- package/dist/hybrid-memory.d.ts +81 -0
- package/dist/hybrid-memory.d.ts.map +1 -0
- package/dist/hybrid-memory.js +296 -0
- package/dist/hybrid-memory.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/{src/index.ts → dist/index.js} +1 -1
- package/dist/index.js.map +1 -0
- package/dist/interface.d.ts +38 -0
- package/dist/interface.d.ts.map +1 -0
- package/dist/interface.js +75 -0
- package/dist/interface.js.map +1 -0
- package/dist/memory.d.ts +52 -0
- package/dist/memory.d.ts.map +1 -0
- package/dist/memory.js +230 -0
- package/dist/memory.js.map +1 -0
- package/dist/server.d.ts +55 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +147 -0
- package/dist/server.js.map +1 -0
- package/package.json +3 -2
- package/src/benchmark.ts +0 -434
- package/src/cli.ts +0 -89
- package/src/enhanced-memory.ts +0 -437
- package/src/enhanced-server.ts +0 -523
- package/src/flow-orchestrator.ts +0 -415
- package/src/hybrid-benchmark.ts +0 -291
- package/src/hybrid-memory.ts +0 -401
- package/src/interface.ts +0 -118
- package/src/memory.ts +0 -196
- package/src/optimized-benchmark.ts +0 -302
- package/src/optimized-memory.ts +0 -271
- package/src/server.ts +0 -213
- package/tsconfig.json +0 -20
package/src/flow-orchestrator.ts
DELETED
|
@@ -1,415 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Agentic Flow Orchestration Integration
|
|
3
|
-
*
|
|
4
|
-
* Integrates agentic-flow's 66 agents and 213 MCP tools for
|
|
5
|
-
* multi-robot coordination and complex task execution
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
import { spawn } from 'child_process';
|
|
9
|
-
import { promisify } from 'util';
|
|
10
|
-
import { exec } from 'child_process';
|
|
11
|
-
|
|
12
|
-
const execAsync = promisify(exec);
|
|
13
|
-
|
|
14
|
-
export interface AgentTask {
|
|
15
|
-
id: string;
|
|
16
|
-
type: string;
|
|
17
|
-
priority: 'low' | 'medium' | 'high' | 'critical';
|
|
18
|
-
params: Record<string, any>;
|
|
19
|
-
timeout?: number;
|
|
20
|
-
retries?: number;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
export interface AgentResult {
|
|
24
|
-
taskId: string;
|
|
25
|
-
success: boolean;
|
|
26
|
-
result: any;
|
|
27
|
-
executionTime: number;
|
|
28
|
-
agentsUsed: string[];
|
|
29
|
-
toolsUsed: string[];
|
|
30
|
-
reasoning?: string;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
export interface SwarmConfig {
|
|
34
|
-
numAgents?: number;
|
|
35
|
-
strategy?: 'parallel' | 'sequential' | 'adaptive' | 'swarm';
|
|
36
|
-
mcpTools?: string[];
|
|
37
|
-
reasoningEnabled?: boolean;
|
|
38
|
-
learningEnabled?: boolean;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
export interface OrchestrationMetrics {
|
|
42
|
-
totalTasks: number;
|
|
43
|
-
successfulTasks: number;
|
|
44
|
-
failedTasks: number;
|
|
45
|
-
avgExecutionTime: number;
|
|
46
|
-
agentUtilization: Record<string, number>;
|
|
47
|
-
toolUsage: Record<string, number>;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
export class FlowOrchestrator {
|
|
51
|
-
private config: SwarmConfig;
|
|
52
|
-
private metrics: OrchestrationMetrics;
|
|
53
|
-
private initialized: boolean = false;
|
|
54
|
-
|
|
55
|
-
constructor(config: SwarmConfig = {}) {
|
|
56
|
-
this.config = {
|
|
57
|
-
numAgents: config.numAgents || 66,
|
|
58
|
-
strategy: config.strategy || 'adaptive',
|
|
59
|
-
mcpTools: config.mcpTools || [],
|
|
60
|
-
reasoningEnabled: config.reasoningEnabled ?? true,
|
|
61
|
-
learningEnabled: config.learningEnabled ?? true,
|
|
62
|
-
};
|
|
63
|
-
|
|
64
|
-
this.metrics = {
|
|
65
|
-
totalTasks: 0,
|
|
66
|
-
successfulTasks: 0,
|
|
67
|
-
failedTasks: 0,
|
|
68
|
-
avgExecutionTime: 0,
|
|
69
|
-
agentUtilization: {},
|
|
70
|
-
toolUsage: {},
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
async initialize(): Promise<void> {
|
|
75
|
-
if (this.initialized) return;
|
|
76
|
-
|
|
77
|
-
const startTime = Date.now();
|
|
78
|
-
|
|
79
|
-
try {
|
|
80
|
-
// Initialize agentic-flow with configuration
|
|
81
|
-
const cmd = [
|
|
82
|
-
'npx agentic-flow init',
|
|
83
|
-
`--agents ${this.config.numAgents}`,
|
|
84
|
-
`--strategy ${this.config.strategy}`,
|
|
85
|
-
this.config.reasoningEnabled ? '--enable-reasoning' : '',
|
|
86
|
-
this.config.learningEnabled ? '--enable-learning' : '',
|
|
87
|
-
].filter(Boolean).join(' ');
|
|
88
|
-
|
|
89
|
-
await execAsync(cmd);
|
|
90
|
-
|
|
91
|
-
const initTime = Date.now() - startTime;
|
|
92
|
-
console.error(`🌊 Agentic Flow initialized (${this.config.numAgents} agents, ${initTime}ms)`);
|
|
93
|
-
this.initialized = true;
|
|
94
|
-
} catch (error: any) {
|
|
95
|
-
console.error('⚠️ Flow initialization warning:', error.message);
|
|
96
|
-
this.initialized = true;
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* Execute a single task using agentic-flow orchestration
|
|
102
|
-
*/
|
|
103
|
-
async executeTask(task: AgentTask): Promise<AgentResult> {
|
|
104
|
-
const startTime = Date.now();
|
|
105
|
-
this.metrics.totalTasks++;
|
|
106
|
-
|
|
107
|
-
try {
|
|
108
|
-
const cmd = [
|
|
109
|
-
'npx agentic-flow execute',
|
|
110
|
-
`--task-type "${task.type}"`,
|
|
111
|
-
`--priority ${task.priority}`,
|
|
112
|
-
`--params '${JSON.stringify(task.params)}'`,
|
|
113
|
-
task.timeout ? `--timeout ${task.timeout}` : '',
|
|
114
|
-
task.retries ? `--retries ${task.retries}` : '',
|
|
115
|
-
this.config.reasoningEnabled ? '--enable-reasoning' : '',
|
|
116
|
-
'--format json',
|
|
117
|
-
].filter(Boolean).join(' ');
|
|
118
|
-
|
|
119
|
-
const { stdout } = await execAsync(cmd, { timeout: task.timeout || 30000 });
|
|
120
|
-
|
|
121
|
-
const lines = stdout.trim().split('\n');
|
|
122
|
-
const jsonLine = lines.find(line => line.startsWith('{'));
|
|
123
|
-
|
|
124
|
-
if (jsonLine) {
|
|
125
|
-
const data = JSON.parse(jsonLine);
|
|
126
|
-
|
|
127
|
-
const executionTime = Date.now() - startTime;
|
|
128
|
-
this.metrics.successfulTasks++;
|
|
129
|
-
this.updateMetrics(data, executionTime);
|
|
130
|
-
|
|
131
|
-
console.error(`✅ Task ${task.id} completed in ${executionTime}ms`);
|
|
132
|
-
|
|
133
|
-
return {
|
|
134
|
-
taskId: task.id,
|
|
135
|
-
success: true,
|
|
136
|
-
result: data.result,
|
|
137
|
-
executionTime,
|
|
138
|
-
agentsUsed: data.agents_used || [],
|
|
139
|
-
toolsUsed: data.tools_used || [],
|
|
140
|
-
reasoning: data.reasoning,
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
throw new Error('No valid JSON response from agentic-flow');
|
|
145
|
-
} catch (error: any) {
|
|
146
|
-
this.metrics.failedTasks++;
|
|
147
|
-
const executionTime = Date.now() - startTime;
|
|
148
|
-
|
|
149
|
-
console.error(`❌ Task ${task.id} failed: ${error.message}`);
|
|
150
|
-
|
|
151
|
-
return {
|
|
152
|
-
taskId: task.id,
|
|
153
|
-
success: false,
|
|
154
|
-
result: { error: error.message },
|
|
155
|
-
executionTime,
|
|
156
|
-
agentsUsed: [],
|
|
157
|
-
toolsUsed: [],
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
/**
|
|
163
|
-
* Execute multiple tasks in parallel using swarm intelligence
|
|
164
|
-
*/
|
|
165
|
-
async executeSwarm(tasks: AgentTask[]): Promise<AgentResult[]> {
|
|
166
|
-
const startTime = Date.now();
|
|
167
|
-
|
|
168
|
-
console.error(`🐝 Executing swarm with ${tasks.length} tasks`);
|
|
169
|
-
|
|
170
|
-
try {
|
|
171
|
-
// Use agentic-flow's swarm coordination
|
|
172
|
-
const tasksJson = JSON.stringify(tasks);
|
|
173
|
-
const cmd = [
|
|
174
|
-
'npx agentic-flow swarm',
|
|
175
|
-
`--tasks '${tasksJson}'`,
|
|
176
|
-
`--strategy ${this.config.strategy}`,
|
|
177
|
-
`--num-agents ${this.config.numAgents}`,
|
|
178
|
-
this.config.reasoningEnabled ? '--enable-reasoning' : '',
|
|
179
|
-
'--format json',
|
|
180
|
-
].filter(Boolean).join(' ');
|
|
181
|
-
|
|
182
|
-
const { stdout } = await execAsync(cmd, { timeout: 60000 });
|
|
183
|
-
|
|
184
|
-
const lines = stdout.trim().split('\n');
|
|
185
|
-
const jsonLine = lines.find(line => line.startsWith('[') || line.startsWith('{'));
|
|
186
|
-
|
|
187
|
-
if (jsonLine) {
|
|
188
|
-
const data = JSON.parse(jsonLine);
|
|
189
|
-
const results = Array.isArray(data) ? data : [data];
|
|
190
|
-
|
|
191
|
-
const swarmTime = Date.now() - startTime;
|
|
192
|
-
console.error(`🎯 Swarm completed ${results.length} tasks in ${swarmTime}ms`);
|
|
193
|
-
|
|
194
|
-
return results.map((r, i) => ({
|
|
195
|
-
taskId: tasks[i].id,
|
|
196
|
-
success: r.success ?? true,
|
|
197
|
-
result: r.result,
|
|
198
|
-
executionTime: r.execution_time || swarmTime / results.length,
|
|
199
|
-
agentsUsed: r.agents_used || [],
|
|
200
|
-
toolsUsed: r.tools_used || [],
|
|
201
|
-
reasoning: r.reasoning,
|
|
202
|
-
}));
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
throw new Error('No valid JSON response from swarm execution');
|
|
206
|
-
} catch (error: any) {
|
|
207
|
-
console.error(`❌ Swarm execution failed: ${error.message}`);
|
|
208
|
-
|
|
209
|
-
// Return individual failures
|
|
210
|
-
return tasks.map(task => ({
|
|
211
|
-
taskId: task.id,
|
|
212
|
-
success: false,
|
|
213
|
-
result: { error: error.message },
|
|
214
|
-
executionTime: 0,
|
|
215
|
-
agentsUsed: [],
|
|
216
|
-
toolsUsed: [],
|
|
217
|
-
}));
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
/**
|
|
222
|
-
* Coordinate multiple robots using agentic-flow
|
|
223
|
-
*/
|
|
224
|
-
async coordinateRobots(
|
|
225
|
-
robots: string[],
|
|
226
|
-
mission: {
|
|
227
|
-
type: string;
|
|
228
|
-
objectives: string[];
|
|
229
|
-
constraints?: Record<string, any>;
|
|
230
|
-
}
|
|
231
|
-
): Promise<{
|
|
232
|
-
success: boolean;
|
|
233
|
-
assignments: Record<string, string[]>;
|
|
234
|
-
executionPlan: any;
|
|
235
|
-
estimatedTime: number;
|
|
236
|
-
}> {
|
|
237
|
-
const startTime = Date.now();
|
|
238
|
-
|
|
239
|
-
try {
|
|
240
|
-
const cmd = [
|
|
241
|
-
'npx agentic-flow coordinate',
|
|
242
|
-
`--robots '${JSON.stringify(robots)}'`,
|
|
243
|
-
`--mission-type "${mission.type}"`,
|
|
244
|
-
`--objectives '${JSON.stringify(mission.objectives)}'`,
|
|
245
|
-
mission.constraints ? `--constraints '${JSON.stringify(mission.constraints)}'` : '',
|
|
246
|
-
'--format json',
|
|
247
|
-
].filter(Boolean).join(' ');
|
|
248
|
-
|
|
249
|
-
const { stdout } = await execAsync(cmd, { timeout: 30000 });
|
|
250
|
-
|
|
251
|
-
const lines = stdout.trim().split('\n');
|
|
252
|
-
const jsonLine = lines.find(line => line.startsWith('{'));
|
|
253
|
-
|
|
254
|
-
if (jsonLine) {
|
|
255
|
-
const data = JSON.parse(jsonLine);
|
|
256
|
-
|
|
257
|
-
const coordTime = Date.now() - startTime;
|
|
258
|
-
console.error(`🤝 Robot coordination completed in ${coordTime}ms`);
|
|
259
|
-
|
|
260
|
-
return {
|
|
261
|
-
success: true,
|
|
262
|
-
assignments: data.assignments || {},
|
|
263
|
-
executionPlan: data.execution_plan || {},
|
|
264
|
-
estimatedTime: data.estimated_time || 0,
|
|
265
|
-
};
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
throw new Error('No valid JSON response from coordination');
|
|
269
|
-
} catch (error: any) {
|
|
270
|
-
console.error(`❌ Coordination failed: ${error.message}`);
|
|
271
|
-
|
|
272
|
-
return {
|
|
273
|
-
success: false,
|
|
274
|
-
assignments: {},
|
|
275
|
-
executionPlan: {},
|
|
276
|
-
estimatedTime: 0,
|
|
277
|
-
};
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
/**
|
|
282
|
-
* Use ReasoningBank for complex decision making
|
|
283
|
-
*/
|
|
284
|
-
async reasonAboutTask(
|
|
285
|
-
context: string,
|
|
286
|
-
options: {
|
|
287
|
-
useMemory?: boolean;
|
|
288
|
-
synthesizeStrategy?: boolean;
|
|
289
|
-
explainReasoning?: boolean;
|
|
290
|
-
} = {}
|
|
291
|
-
): Promise<{
|
|
292
|
-
decision: string;
|
|
293
|
-
reasoning: string;
|
|
294
|
-
confidence: number;
|
|
295
|
-
alternatives: string[];
|
|
296
|
-
}> {
|
|
297
|
-
const startTime = Date.now();
|
|
298
|
-
|
|
299
|
-
try {
|
|
300
|
-
const cmd = [
|
|
301
|
-
'npx agentic-flow reason',
|
|
302
|
-
`--context "${context}"`,
|
|
303
|
-
options.useMemory ? '--use-memory' : '',
|
|
304
|
-
options.synthesizeStrategy ? '--synthesize-strategy' : '',
|
|
305
|
-
options.explainReasoning ? '--explain-reasoning' : '',
|
|
306
|
-
'--format json',
|
|
307
|
-
].filter(Boolean).join(' ');
|
|
308
|
-
|
|
309
|
-
const { stdout } = await execAsync(cmd, { timeout: 15000 });
|
|
310
|
-
|
|
311
|
-
const lines = stdout.trim().split('\n');
|
|
312
|
-
const jsonLine = lines.find(line => line.startsWith('{'));
|
|
313
|
-
|
|
314
|
-
if (jsonLine) {
|
|
315
|
-
const data = JSON.parse(jsonLine);
|
|
316
|
-
|
|
317
|
-
const reasonTime = Date.now() - startTime;
|
|
318
|
-
console.error(`🧠 Reasoning completed in ${reasonTime}ms`);
|
|
319
|
-
|
|
320
|
-
return {
|
|
321
|
-
decision: data.decision || '',
|
|
322
|
-
reasoning: data.reasoning || '',
|
|
323
|
-
confidence: data.confidence || 0,
|
|
324
|
-
alternatives: data.alternatives || [],
|
|
325
|
-
};
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
throw new Error('No valid JSON response from reasoning');
|
|
329
|
-
} catch (error: any) {
|
|
330
|
-
console.error(`❌ Reasoning failed: ${error.message}`);
|
|
331
|
-
|
|
332
|
-
return {
|
|
333
|
-
decision: '',
|
|
334
|
-
reasoning: error.message,
|
|
335
|
-
confidence: 0,
|
|
336
|
-
alternatives: [],
|
|
337
|
-
};
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
/**
|
|
342
|
-
* Get available MCP tools from agentic-flow
|
|
343
|
-
*/
|
|
344
|
-
async getAvailableTools(): Promise<string[]> {
|
|
345
|
-
try {
|
|
346
|
-
const { stdout } = await execAsync('npx agentic-flow list-tools --format json');
|
|
347
|
-
|
|
348
|
-
const lines = stdout.trim().split('\n');
|
|
349
|
-
const jsonLine = lines.find(line => line.startsWith('['));
|
|
350
|
-
|
|
351
|
-
if (jsonLine) {
|
|
352
|
-
const tools = JSON.parse(jsonLine);
|
|
353
|
-
console.error(`🔧 Found ${tools.length}/213 MCP tools available`);
|
|
354
|
-
return tools;
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
return [];
|
|
358
|
-
} catch (error: any) {
|
|
359
|
-
console.error('⚠️ Could not list tools:', error.message);
|
|
360
|
-
return [];
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
/**
|
|
365
|
-
* Get metrics and statistics
|
|
366
|
-
*/
|
|
367
|
-
getMetrics(): OrchestrationMetrics {
|
|
368
|
-
return {
|
|
369
|
-
...this.metrics,
|
|
370
|
-
avgExecutionTime: this.metrics.totalTasks > 0
|
|
371
|
-
? this.metrics.avgExecutionTime / this.metrics.totalTasks
|
|
372
|
-
: 0,
|
|
373
|
-
};
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
/**
|
|
377
|
-
* Update internal metrics
|
|
378
|
-
*/
|
|
379
|
-
private updateMetrics(data: any, executionTime: number): void {
|
|
380
|
-
// Update execution time
|
|
381
|
-
this.metrics.avgExecutionTime += executionTime;
|
|
382
|
-
|
|
383
|
-
// Track agent utilization
|
|
384
|
-
if (data.agents_used) {
|
|
385
|
-
for (const agent of data.agents_used) {
|
|
386
|
-
this.metrics.agentUtilization[agent] = (this.metrics.agentUtilization[agent] || 0) + 1;
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
// Track tool usage
|
|
391
|
-
if (data.tools_used) {
|
|
392
|
-
for (const tool of data.tools_used) {
|
|
393
|
-
this.metrics.toolUsage[tool] = (this.metrics.toolUsage[tool] || 0) + 1;
|
|
394
|
-
}
|
|
395
|
-
}
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
/**
|
|
399
|
-
* Reset metrics
|
|
400
|
-
*/
|
|
401
|
-
resetMetrics(): void {
|
|
402
|
-
this.metrics = {
|
|
403
|
-
totalTasks: 0,
|
|
404
|
-
successfulTasks: 0,
|
|
405
|
-
failedTasks: 0,
|
|
406
|
-
avgExecutionTime: 0,
|
|
407
|
-
agentUtilization: {},
|
|
408
|
-
toolUsage: {},
|
|
409
|
-
};
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
async close(): Promise<void> {
|
|
413
|
-
this.initialized = false;
|
|
414
|
-
}
|
|
415
|
-
}
|
package/src/hybrid-benchmark.ts
DELETED
|
@@ -1,291 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Hybrid Benchmark Suite
|
|
3
|
-
*
|
|
4
|
-
* Demonstrates performance improvements with direct SQL storage
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
import { HybridAgentDBMemory } from './hybrid-memory.js';
|
|
8
|
-
import type { Episode } from './memory.js';
|
|
9
|
-
|
|
10
|
-
interface BenchmarkResult {
|
|
11
|
-
name: string;
|
|
12
|
-
iterations: number;
|
|
13
|
-
totalTime: number;
|
|
14
|
-
avgTime: number;
|
|
15
|
-
minTime: number;
|
|
16
|
-
maxTime: number;
|
|
17
|
-
opsPerSec: number;
|
|
18
|
-
p50: number;
|
|
19
|
-
p95: number;
|
|
20
|
-
p99: number;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
export class HybridBenchmark {
|
|
24
|
-
private memory: HybridAgentDBMemory;
|
|
25
|
-
|
|
26
|
-
constructor() {
|
|
27
|
-
this.memory = new HybridAgentDBMemory('./hybrid-benchmark.db');
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
async initialize(): Promise<void> {
|
|
31
|
-
console.log('🏁 Initializing hybrid benchmark suite...\n');
|
|
32
|
-
await this.memory.initialize();
|
|
33
|
-
console.log('✅ Benchmark suite ready\n');
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Run complete benchmark suite
|
|
38
|
-
*/
|
|
39
|
-
async runAll(): Promise<void> {
|
|
40
|
-
const suiteStartTime = Date.now();
|
|
41
|
-
|
|
42
|
-
console.log('┌─────────────────────────────────────────────────────────┐');
|
|
43
|
-
console.log('│ Hybrid Integration Benchmark Suite │');
|
|
44
|
-
console.log('│ Direct SQL + CLI Vector Search Performance Test │');
|
|
45
|
-
console.log('└─────────────────────────────────────────────────────────┘\n');
|
|
46
|
-
|
|
47
|
-
// Hybrid Benchmarks
|
|
48
|
-
console.log('📊 HYBRID BENCHMARKS (SQL Storage + CLI Search)\n');
|
|
49
|
-
|
|
50
|
-
const storeEpisode = await this.benchmarkStoreEpisode(1000);
|
|
51
|
-
this.printResult('Store Episode (SQL)', storeEpisode);
|
|
52
|
-
|
|
53
|
-
const bulkStore = await this.benchmarkBulkStore(1000, 100);
|
|
54
|
-
this.printResult('Bulk Store (SQL Transaction)', bulkStore);
|
|
55
|
-
|
|
56
|
-
const retrieveMemories = await this.benchmarkRetrieveMemories(100);
|
|
57
|
-
this.printResult('Retrieve Memories (SQL Fallback)', retrieveMemories);
|
|
58
|
-
|
|
59
|
-
const queryWithContext = await this.benchmarkQueryWithContext(100);
|
|
60
|
-
this.printResult('Query with Context (Hybrid)', queryWithContext);
|
|
61
|
-
|
|
62
|
-
const searchSkills = await this.benchmarkSearchSkills(200);
|
|
63
|
-
this.printResult('Search Skills (SQL)', searchSkills);
|
|
64
|
-
|
|
65
|
-
const totalTime = Date.now() - suiteStartTime;
|
|
66
|
-
|
|
67
|
-
console.log('\n┌─────────────────────────────────────────────────────────┐');
|
|
68
|
-
console.log('│ BENCHMARK COMPLETE │');
|
|
69
|
-
console.log(`│ Total Time: ${(totalTime / 1000).toFixed(2)}s │`);
|
|
70
|
-
console.log('└─────────────────────────────────────────────────────────┘\n');
|
|
71
|
-
|
|
72
|
-
// Print performance metrics
|
|
73
|
-
const metrics = this.memory.getMetrics();
|
|
74
|
-
console.log('📈 PERFORMANCE METRICS\n');
|
|
75
|
-
console.log(` Store Operations: ${metrics.storeCount}`);
|
|
76
|
-
console.log(` Avg Store Time: ${metrics.avgStoreTime.toFixed(3)}ms`);
|
|
77
|
-
console.log(` Retrieve Operations: ${metrics.retrieveCount}`);
|
|
78
|
-
console.log(` Avg Retrieve Time: ${metrics.avgRetrieveTime.toFixed(3)}ms\n`);
|
|
79
|
-
|
|
80
|
-
// Print database stats
|
|
81
|
-
const stats = await this.memory.getStats();
|
|
82
|
-
console.log('💾 DATABASE STATISTICS\n');
|
|
83
|
-
console.log(` Total Episodes: ${stats.total_episodes}`);
|
|
84
|
-
console.log(` Successful: ${stats.successful_episodes}`);
|
|
85
|
-
console.log(` Failed: ${stats.failed_episodes}`);
|
|
86
|
-
console.log(` Unique Tasks: ${stats.unique_tasks}`);
|
|
87
|
-
console.log(` Avg Confidence: ${stats.avg_confidence?.toFixed(3)}\n`);
|
|
88
|
-
|
|
89
|
-
// Optimization test
|
|
90
|
-
console.log('🔧 Running database optimization...');
|
|
91
|
-
await this.memory.optimize();
|
|
92
|
-
console.log('');
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
* Benchmark: Store Episode (Direct SQL)
|
|
97
|
-
*/
|
|
98
|
-
private async benchmarkStoreEpisode(iterations: number): Promise<BenchmarkResult> {
|
|
99
|
-
const times: number[] = [];
|
|
100
|
-
|
|
101
|
-
for (let i = 0; i < iterations; i++) {
|
|
102
|
-
const episode: Episode = {
|
|
103
|
-
sessionId: `bench-${i}`,
|
|
104
|
-
taskName: 'navigation',
|
|
105
|
-
confidence: Math.random(),
|
|
106
|
-
success: Math.random() > 0.3,
|
|
107
|
-
outcome: 'Completed navigation task',
|
|
108
|
-
strategy: 'adaptive_planning',
|
|
109
|
-
metadata: { x: i, y: i * 2, latency: Math.random() * 100 },
|
|
110
|
-
};
|
|
111
|
-
|
|
112
|
-
const start = performance.now();
|
|
113
|
-
await this.memory.storeEpisode(episode);
|
|
114
|
-
times.push(performance.now() - start);
|
|
115
|
-
|
|
116
|
-
if (i % 100 === 0 && i > 0) {
|
|
117
|
-
process.stderr.write(` Progress: ${i}/${iterations}\r`);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
return this.calculateStats('Store Episode', times);
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
/**
|
|
125
|
-
* Benchmark: Bulk Store (SQL Transaction)
|
|
126
|
-
*/
|
|
127
|
-
private async benchmarkBulkStore(totalEpisodes: number, batchSize: number): Promise<BenchmarkResult> {
|
|
128
|
-
const times: number[] = [];
|
|
129
|
-
const numBatches = Math.ceil(totalEpisodes / batchSize);
|
|
130
|
-
|
|
131
|
-
for (let i = 0; i < numBatches; i++) {
|
|
132
|
-
const batch: Episode[] = [];
|
|
133
|
-
for (let j = 0; j < batchSize; j++) {
|
|
134
|
-
const idx = i * batchSize + j;
|
|
135
|
-
batch.push({
|
|
136
|
-
sessionId: `bulk-${idx}`,
|
|
137
|
-
taskName: 'navigation',
|
|
138
|
-
confidence: Math.random(),
|
|
139
|
-
success: Math.random() > 0.3,
|
|
140
|
-
outcome: 'Completed navigation task',
|
|
141
|
-
strategy: 'adaptive_planning',
|
|
142
|
-
metadata: { x: idx, y: idx * 2 },
|
|
143
|
-
});
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
const start = performance.now();
|
|
147
|
-
await this.memory.bulkStoreEpisodes(batch);
|
|
148
|
-
times.push(performance.now() - start);
|
|
149
|
-
|
|
150
|
-
if (i % 5 === 0 && i > 0) {
|
|
151
|
-
process.stderr.write(` Progress: ${i * batchSize}/${totalEpisodes}\r`);
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
// Convert to per-episode times
|
|
156
|
-
const perEpisodeTimes = times.map(t => t / batchSize);
|
|
157
|
-
return this.calculateStats('Bulk Store', perEpisodeTimes);
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/**
|
|
161
|
-
* Benchmark: Retrieve Memories (SQL fallback)
|
|
162
|
-
*/
|
|
163
|
-
private async benchmarkRetrieveMemories(iterations: number): Promise<BenchmarkResult> {
|
|
164
|
-
const times: number[] = [];
|
|
165
|
-
|
|
166
|
-
const queries = [
|
|
167
|
-
'navigation task',
|
|
168
|
-
'object detection',
|
|
169
|
-
'path planning',
|
|
170
|
-
'sensor fusion',
|
|
171
|
-
'motor control',
|
|
172
|
-
];
|
|
173
|
-
|
|
174
|
-
for (let i = 0; i < iterations; i++) {
|
|
175
|
-
const query = queries[i % queries.length];
|
|
176
|
-
|
|
177
|
-
const start = performance.now();
|
|
178
|
-
await this.memory.retrieveMemories(query, 5, {
|
|
179
|
-
onlySuccesses: true,
|
|
180
|
-
});
|
|
181
|
-
times.push(performance.now() - start);
|
|
182
|
-
|
|
183
|
-
if (i % 20 === 0 && i > 0) {
|
|
184
|
-
process.stderr.write(` Progress: ${i}/${iterations}\r`);
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
return this.calculateStats('Retrieve Memories', times);
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
/**
|
|
192
|
-
* Benchmark: Query with Context
|
|
193
|
-
*/
|
|
194
|
-
private async benchmarkQueryWithContext(iterations: number): Promise<BenchmarkResult> {
|
|
195
|
-
const times: number[] = [];
|
|
196
|
-
|
|
197
|
-
for (let i = 0; i < iterations; i++) {
|
|
198
|
-
const start = performance.now();
|
|
199
|
-
await this.memory.queryWithContext('warehouse navigation', {
|
|
200
|
-
k: 10,
|
|
201
|
-
minConfidence: 0.5,
|
|
202
|
-
});
|
|
203
|
-
times.push(performance.now() - start);
|
|
204
|
-
|
|
205
|
-
if (i % 20 === 0 && i > 0) {
|
|
206
|
-
process.stderr.write(` Progress: ${i}/${iterations}\r`);
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
return this.calculateStats('Query with Context', times);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
/**
|
|
214
|
-
* Benchmark: Search Skills (Direct SQL)
|
|
215
|
-
*/
|
|
216
|
-
private async benchmarkSearchSkills(iterations: number): Promise<BenchmarkResult> {
|
|
217
|
-
const times: number[] = [];
|
|
218
|
-
|
|
219
|
-
for (let i = 0; i < iterations; i++) {
|
|
220
|
-
const start = performance.now();
|
|
221
|
-
await this.memory.searchSkills('navigation', {
|
|
222
|
-
k: 5,
|
|
223
|
-
minSuccessRate: 0.5,
|
|
224
|
-
sortBy: 'success_rate',
|
|
225
|
-
});
|
|
226
|
-
times.push(performance.now() - start);
|
|
227
|
-
|
|
228
|
-
if (i % 20 === 0 && i > 0) {
|
|
229
|
-
process.stderr.write(` Progress: ${i}/${iterations}\r`);
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
return this.calculateStats('Search Skills', times);
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
/**
|
|
237
|
-
* Calculate statistics from timing data
|
|
238
|
-
*/
|
|
239
|
-
private calculateStats(name: string, times: number[]): BenchmarkResult {
|
|
240
|
-
const sorted = [...times].sort((a, b) => a - b);
|
|
241
|
-
const total = times.reduce((a, b) => a + b, 0);
|
|
242
|
-
|
|
243
|
-
return {
|
|
244
|
-
name,
|
|
245
|
-
iterations: times.length,
|
|
246
|
-
totalTime: total,
|
|
247
|
-
avgTime: total / times.length,
|
|
248
|
-
minTime: sorted[0],
|
|
249
|
-
maxTime: sorted[sorted.length - 1],
|
|
250
|
-
opsPerSec: (times.length / total) * 1000,
|
|
251
|
-
p50: sorted[Math.floor(sorted.length * 0.50)],
|
|
252
|
-
p95: sorted[Math.floor(sorted.length * 0.95)],
|
|
253
|
-
p99: sorted[Math.floor(sorted.length * 0.99)],
|
|
254
|
-
};
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
/**
|
|
258
|
-
* Print benchmark result
|
|
259
|
-
*/
|
|
260
|
-
private printResult(name: string, result: BenchmarkResult): void {
|
|
261
|
-
const speedup = 2300 / result.avgTime; // Compare to CLI baseline (2.3s)
|
|
262
|
-
|
|
263
|
-
console.log(`${name}:`);
|
|
264
|
-
console.log(` Iterations: ${result.iterations.toLocaleString()}`);
|
|
265
|
-
console.log(` Avg Time: ${result.avgTime.toFixed(3)}ms`);
|
|
266
|
-
console.log(` Min/Max: ${result.minTime.toFixed(3)}ms / ${result.maxTime.toFixed(3)}ms`);
|
|
267
|
-
console.log(` p50/p95/p99: ${result.p50.toFixed(3)}ms / ${result.p95.toFixed(3)}ms / ${result.p99.toFixed(3)}ms`);
|
|
268
|
-
console.log(` Ops/sec: ${result.opsPerSec.toFixed(0)}`);
|
|
269
|
-
console.log(` Speedup: ${speedup.toFixed(0)}x faster than CLI`);
|
|
270
|
-
console.log('');
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
async close(): Promise<void> {
|
|
274
|
-
await this.memory.close();
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
// CLI entry point
|
|
279
|
-
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
280
|
-
const benchmark = new HybridBenchmark();
|
|
281
|
-
|
|
282
|
-
try {
|
|
283
|
-
await benchmark.initialize();
|
|
284
|
-
await benchmark.runAll();
|
|
285
|
-
} catch (error) {
|
|
286
|
-
console.error('❌ Benchmark failed:', error);
|
|
287
|
-
process.exit(1);
|
|
288
|
-
} finally {
|
|
289
|
-
await benchmark.close();
|
|
290
|
-
}
|
|
291
|
-
}
|