nodebench-mcp 2.47.0 → 2.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmarks/agentValidation.d.ts +47 -0
- package/dist/benchmarks/agentValidation.js +417 -0
- package/dist/benchmarks/agentValidation.js.map +1 -0
- package/dist/benchmarks/testProviderBus.d.ts +7 -0
- package/dist/benchmarks/testProviderBus.js +272 -0
- package/dist/benchmarks/testProviderBus.js.map +1 -0
- package/dist/benchmarks/testSupermemory.d.ts +8 -0
- package/dist/benchmarks/testSupermemory.js +187 -0
- package/dist/benchmarks/testSupermemory.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Agent Validation Harness — Simulates how real AI agents (Claude Code,
|
|
3
|
+
* OpenClaw, Cursor, Windsurf, generic MCP clients) interact with the
|
|
4
|
+
* NodeBench MCP server and scores the experience.
|
|
5
|
+
*
|
|
6
|
+
* Validates: preset loading, tool count limits, progressive discovery,
|
|
7
|
+
* toolset expansion, and workflow completion for each persona.
|
|
8
|
+
*
|
|
9
|
+
* All scoring is deterministic and heuristic-based (no LLM calls).
|
|
10
|
+
*/
|
|
11
|
+
import type { McpTool } from "../types.js";
|
|
12
|
+
export interface AgentPersona {
|
|
13
|
+
name: string;
|
|
14
|
+
maxTools: number;
|
|
15
|
+
supportsWebSocket: boolean;
|
|
16
|
+
supportsOAuth: boolean;
|
|
17
|
+
preferredPreset: string;
|
|
18
|
+
typicalWorkflow: string;
|
|
19
|
+
}
|
|
20
|
+
export interface PersonaScores {
|
|
21
|
+
toolDiscovery: number;
|
|
22
|
+
workflowCompletion: number;
|
|
23
|
+
presetFit: number;
|
|
24
|
+
errorRate: number;
|
|
25
|
+
}
|
|
26
|
+
export interface PersonaResult {
|
|
27
|
+
name: string;
|
|
28
|
+
scores: PersonaScores;
|
|
29
|
+
passed: boolean;
|
|
30
|
+
errors: string[];
|
|
31
|
+
toolCount: number;
|
|
32
|
+
presetUsed: string;
|
|
33
|
+
durationMs: number;
|
|
34
|
+
}
|
|
35
|
+
export interface AgentValidationResult {
|
|
36
|
+
personas: PersonaResult[];
|
|
37
|
+
overallPassRate: number;
|
|
38
|
+
recommendations: string[];
|
|
39
|
+
totalDurationMs: number;
|
|
40
|
+
}
|
|
41
|
+
export interface WebMcpReadiness {
|
|
42
|
+
ready: boolean;
|
|
43
|
+
missingItems: string[];
|
|
44
|
+
}
|
|
45
|
+
export declare function runAgentValidation(personaFilter?: string): Promise<AgentValidationResult>;
|
|
46
|
+
export declare function checkWebMcpReadiness(): WebMcpReadiness;
|
|
47
|
+
export declare const validateAgentCompatibilityTool: McpTool;
|
|
@@ -0,0 +1,417 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Agent Validation Harness — Simulates how real AI agents (Claude Code,
|
|
3
|
+
* OpenClaw, Cursor, Windsurf, generic MCP clients) interact with the
|
|
4
|
+
* NodeBench MCP server and scores the experience.
|
|
5
|
+
*
|
|
6
|
+
* Validates: preset loading, tool count limits, progressive discovery,
|
|
7
|
+
* toolset expansion, and workflow completion for each persona.
|
|
8
|
+
*
|
|
9
|
+
* All scoring is deterministic and heuristic-based (no LLM calls).
|
|
10
|
+
*/
|
|
11
|
+
import { getDb, genId } from "../db.js";
|
|
12
|
+
import { loadToolsets, ALL_DOMAIN_KEYS } from "../toolsetRegistry.js";
|
|
13
|
+
import { hybridSearch, ALL_REGISTRY_ENTRIES, WORKFLOW_CHAINS, } from "../tools/toolRegistry.js";
|
|
14
|
+
/* ================================================================== */
|
|
15
|
+
/* Persona Definitions */
|
|
16
|
+
/* ================================================================== */
|
|
17
|
+
const AGENT_PERSONAS = [
|
|
18
|
+
{
|
|
19
|
+
name: "claude_code",
|
|
20
|
+
maxTools: Infinity,
|
|
21
|
+
supportsWebSocket: true,
|
|
22
|
+
supportsOAuth: false,
|
|
23
|
+
preferredPreset: "founder",
|
|
24
|
+
typicalWorkflow: "weekly_reset",
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
name: "openclaw",
|
|
28
|
+
maxTools: Infinity,
|
|
29
|
+
supportsWebSocket: true,
|
|
30
|
+
supportsOAuth: false,
|
|
31
|
+
preferredPreset: "core",
|
|
32
|
+
typicalWorkflow: "research",
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
name: "cursor",
|
|
36
|
+
maxTools: 40,
|
|
37
|
+
supportsWebSocket: false,
|
|
38
|
+
supportsOAuth: false,
|
|
39
|
+
preferredPreset: "cursor",
|
|
40
|
+
typicalWorkflow: "code_review",
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
name: "windsurf",
|
|
44
|
+
maxTools: Infinity,
|
|
45
|
+
supportsWebSocket: false,
|
|
46
|
+
supportsOAuth: false,
|
|
47
|
+
preferredPreset: "core",
|
|
48
|
+
typicalWorkflow: "code_review",
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
name: "generic_mcp",
|
|
52
|
+
maxTools: Infinity,
|
|
53
|
+
supportsWebSocket: true,
|
|
54
|
+
supportsOAuth: false,
|
|
55
|
+
preferredPreset: "starter",
|
|
56
|
+
typicalWorkflow: "company_search",
|
|
57
|
+
},
|
|
58
|
+
];
|
|
59
|
+
/* ================================================================== */
|
|
60
|
+
/* Workflow → search queries mapping */
|
|
61
|
+
/* ================================================================== */
|
|
62
|
+
const WORKFLOW_QUERIES = {
|
|
63
|
+
weekly_reset: [
|
|
64
|
+
"weekly review",
|
|
65
|
+
"action tracking",
|
|
66
|
+
"milestone",
|
|
67
|
+
"session history",
|
|
68
|
+
],
|
|
69
|
+
research: [
|
|
70
|
+
"research",
|
|
71
|
+
"web search",
|
|
72
|
+
"knowledge",
|
|
73
|
+
"analysis",
|
|
74
|
+
],
|
|
75
|
+
code_review: [
|
|
76
|
+
"verification",
|
|
77
|
+
"quality gate",
|
|
78
|
+
"gap",
|
|
79
|
+
"test",
|
|
80
|
+
],
|
|
81
|
+
company_search: [
|
|
82
|
+
"company",
|
|
83
|
+
"search",
|
|
84
|
+
"deep sim",
|
|
85
|
+
"decision",
|
|
86
|
+
],
|
|
87
|
+
};
|
|
88
|
+
/* ================================================================== */
|
|
89
|
+
/* Preset → domain mapping (mirrors index.ts PRESETS) */
|
|
90
|
+
/* ================================================================== */
|
|
91
|
+
const PRESET_DOMAINS = {
|
|
92
|
+
starter: ["deep_sim"],
|
|
93
|
+
default: ["deep_sim"],
|
|
94
|
+
core: [
|
|
95
|
+
"verification", "eval", "quality_gate", "learning", "flywheel",
|
|
96
|
+
"recon", "security", "boilerplate", "skill_update", "context_sandbox",
|
|
97
|
+
"observability", "execution_trace", "mission_harness", "deep_sim", "founder",
|
|
98
|
+
],
|
|
99
|
+
founder: ["deep_sim", "founder", "learning", "local_dashboard"],
|
|
100
|
+
cursor: ["deep_sim", "quality_gate", "learning", "session_memory", "web", "toon"],
|
|
101
|
+
full: ALL_DOMAIN_KEYS,
|
|
102
|
+
};
|
|
103
|
+
/* ================================================================== */
|
|
104
|
+
/* Schema bootstrap (idempotent) */
|
|
105
|
+
/* ================================================================== */
|
|
106
|
+
let _schemaReady = false;
|
|
107
|
+
function ensureSchema() {
|
|
108
|
+
if (_schemaReady)
|
|
109
|
+
return;
|
|
110
|
+
const db = getDb();
|
|
111
|
+
db.exec(`
|
|
112
|
+
CREATE TABLE IF NOT EXISTS agent_validation_runs (
|
|
113
|
+
id TEXT PRIMARY KEY,
|
|
114
|
+
timestamp TEXT NOT NULL,
|
|
115
|
+
personaName TEXT NOT NULL,
|
|
116
|
+
presetUsed TEXT NOT NULL,
|
|
117
|
+
toolCount INTEGER NOT NULL,
|
|
118
|
+
toolDiscovery REAL NOT NULL,
|
|
119
|
+
workflowCompletion REAL NOT NULL,
|
|
120
|
+
presetFit REAL NOT NULL,
|
|
121
|
+
errorRate REAL NOT NULL,
|
|
122
|
+
passed INTEGER NOT NULL,
|
|
123
|
+
errors TEXT,
|
|
124
|
+
durationMs INTEGER NOT NULL
|
|
125
|
+
);
|
|
126
|
+
CREATE INDEX IF NOT EXISTS idx_agent_validation_persona
|
|
127
|
+
ON agent_validation_runs(personaName);
|
|
128
|
+
CREATE INDEX IF NOT EXISTS idx_agent_validation_timestamp
|
|
129
|
+
ON agent_validation_runs(timestamp);
|
|
130
|
+
`);
|
|
131
|
+
_schemaReady = true;
|
|
132
|
+
}
|
|
133
|
+
/* ================================================================== */
|
|
134
|
+
/* Per-persona validation */
|
|
135
|
+
/* ================================================================== */
|
|
136
|
+
async function validatePersona(persona) {
|
|
137
|
+
const startMs = Date.now();
|
|
138
|
+
const errors = [];
|
|
139
|
+
const scores = {
|
|
140
|
+
toolDiscovery: 0,
|
|
141
|
+
workflowCompletion: 0,
|
|
142
|
+
presetFit: 0,
|
|
143
|
+
errorRate: 1, // start optimistic
|
|
144
|
+
};
|
|
145
|
+
/* ── Step 1: Load the persona's preferred preset ──────────────── */
|
|
146
|
+
const domains = PRESET_DOMAINS[persona.preferredPreset];
|
|
147
|
+
if (!domains) {
|
|
148
|
+
errors.push(`Unknown preset: ${persona.preferredPreset}`);
|
|
149
|
+
scores.errorRate = 0;
|
|
150
|
+
return {
|
|
151
|
+
name: persona.name,
|
|
152
|
+
scores,
|
|
153
|
+
passed: false,
|
|
154
|
+
errors,
|
|
155
|
+
toolCount: 0,
|
|
156
|
+
presetUsed: persona.preferredPreset,
|
|
157
|
+
durationMs: Date.now() - startMs,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
let tools;
|
|
161
|
+
try {
|
|
162
|
+
tools = await loadToolsets(domains);
|
|
163
|
+
}
|
|
164
|
+
catch (err) {
|
|
165
|
+
errors.push(`loadToolsets failed: ${String(err)}`);
|
|
166
|
+
scores.errorRate = 0;
|
|
167
|
+
return {
|
|
168
|
+
name: persona.name,
|
|
169
|
+
scores,
|
|
170
|
+
passed: false,
|
|
171
|
+
errors,
|
|
172
|
+
toolCount: 0,
|
|
173
|
+
presetUsed: persona.preferredPreset,
|
|
174
|
+
durationMs: Date.now() - startMs,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
const toolCount = tools.length;
|
|
178
|
+
/* ── Step 2: Check tool count fits within maxTools ─────────────── */
|
|
179
|
+
if (toolCount <= persona.maxTools) {
|
|
180
|
+
scores.presetFit = 1;
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
// Partial credit: how close is the fit?
|
|
184
|
+
scores.presetFit = Math.max(0, persona.maxTools / toolCount);
|
|
185
|
+
errors.push(`Preset '${persona.preferredPreset}' loads ${toolCount} tools but ${persona.name} supports max ${persona.maxTools}`);
|
|
186
|
+
}
|
|
187
|
+
/* ── Step 3: Simulate tool discovery via hybridSearch ──────────── */
|
|
188
|
+
const queries = WORKFLOW_QUERIES[persona.typicalWorkflow] ?? ["tools"];
|
|
189
|
+
let discoveredCount = 0;
|
|
190
|
+
let totalExpected = 0;
|
|
191
|
+
for (const query of queries) {
|
|
192
|
+
totalExpected++;
|
|
193
|
+
try {
|
|
194
|
+
const allToolDescs = ALL_REGISTRY_ENTRIES.map((e) => ({
|
|
195
|
+
name: e.name,
|
|
196
|
+
description: `${e.category} ${e.tags.join(" ")}`,
|
|
197
|
+
}));
|
|
198
|
+
const results = hybridSearch(query, allToolDescs, {
|
|
199
|
+
limit: 5,
|
|
200
|
+
mode: "hybrid",
|
|
201
|
+
});
|
|
202
|
+
if (results.length > 0) {
|
|
203
|
+
discoveredCount++;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
catch (err) {
|
|
207
|
+
errors.push(`discover_tools('${query}') failed: ${String(err)}`);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
scores.toolDiscovery = totalExpected > 0 ? discoveredCount / totalExpected : 0;
|
|
211
|
+
/* ── Step 4: Check workflow chain availability ─────────────────── */
|
|
212
|
+
const chainKeys = Object.keys(WORKFLOW_CHAINS);
|
|
213
|
+
// Search for a chain matching the persona's typical workflow
|
|
214
|
+
const workflowKeywords = persona.typicalWorkflow.split("_");
|
|
215
|
+
const matchingChain = chainKeys.find((key) => workflowKeywords.some((kw) => key.toLowerCase().includes(kw)));
|
|
216
|
+
if (matchingChain) {
|
|
217
|
+
scores.workflowCompletion = 1;
|
|
218
|
+
}
|
|
219
|
+
else {
|
|
220
|
+
// Partial credit: can we at least find tools relevant to the workflow?
|
|
221
|
+
const relevantTools = ALL_REGISTRY_ENTRIES.filter((e) => workflowKeywords.some((kw) => e.name.includes(kw) ||
|
|
222
|
+
e.tags.some((t) => t.includes(kw)) ||
|
|
223
|
+
e.category.includes(kw)));
|
|
224
|
+
scores.workflowCompletion = relevantTools.length > 0
|
|
225
|
+
? Math.min(1, relevantTools.length / 3)
|
|
226
|
+
: 0;
|
|
227
|
+
}
|
|
228
|
+
/* ── Step 5: Simulate load_toolset expansion ──────────────────── */
|
|
229
|
+
// Verify that loading an additional domain works without error
|
|
230
|
+
const expansionDomain = domains.includes("learning") ? "web" : "learning";
|
|
231
|
+
try {
|
|
232
|
+
await loadToolsets([expansionDomain]);
|
|
233
|
+
// Expansion succeeded — no error
|
|
234
|
+
}
|
|
235
|
+
catch (err) {
|
|
236
|
+
errors.push(`Toolset expansion to '${expansionDomain}' failed: ${String(err)}`);
|
|
237
|
+
}
|
|
238
|
+
/* ── Compute error rate ───────────────────────────────────────── */
|
|
239
|
+
const totalChecks = 4; // preset load, discovery, workflow, expansion
|
|
240
|
+
scores.errorRate = Math.max(0, 1 - errors.length / totalChecks);
|
|
241
|
+
/* ── Pass/fail threshold ──────────────────────────────────────── */
|
|
242
|
+
const avgScore = (scores.toolDiscovery +
|
|
243
|
+
scores.workflowCompletion +
|
|
244
|
+
scores.presetFit +
|
|
245
|
+
scores.errorRate) /
|
|
246
|
+
4;
|
|
247
|
+
const passed = avgScore >= 0.6;
|
|
248
|
+
return {
|
|
249
|
+
name: persona.name,
|
|
250
|
+
scores,
|
|
251
|
+
passed,
|
|
252
|
+
errors,
|
|
253
|
+
toolCount,
|
|
254
|
+
presetUsed: persona.preferredPreset,
|
|
255
|
+
durationMs: Date.now() - startMs,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
/* ================================================================== */
|
|
259
|
+
/* Run all personas */
|
|
260
|
+
/* ================================================================== */
|
|
261
|
+
export async function runAgentValidation(personaFilter) {
|
|
262
|
+
ensureSchema();
|
|
263
|
+
const overallStart = Date.now();
|
|
264
|
+
const personas = personaFilter
|
|
265
|
+
? AGENT_PERSONAS.filter((p) => p.name === personaFilter)
|
|
266
|
+
: AGENT_PERSONAS;
|
|
267
|
+
if (personas.length === 0 && personaFilter) {
|
|
268
|
+
return {
|
|
269
|
+
personas: [],
|
|
270
|
+
overallPassRate: 0,
|
|
271
|
+
recommendations: [`Unknown agent persona: '${personaFilter}'. Available: ${AGENT_PERSONAS.map((p) => p.name).join(", ")}`],
|
|
272
|
+
totalDurationMs: Date.now() - overallStart,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
const results = [];
|
|
276
|
+
for (const persona of personas) {
|
|
277
|
+
const result = await validatePersona(persona);
|
|
278
|
+
results.push(result);
|
|
279
|
+
// Persist to SQLite
|
|
280
|
+
const db = getDb();
|
|
281
|
+
db.prepare(`
|
|
282
|
+
INSERT INTO agent_validation_runs
|
|
283
|
+
(id, timestamp, personaName, presetUsed, toolCount,
|
|
284
|
+
toolDiscovery, workflowCompletion, presetFit, errorRate,
|
|
285
|
+
passed, errors, durationMs)
|
|
286
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
287
|
+
`).run(genId("av"), new Date().toISOString(), result.name, result.presetUsed, result.toolCount, result.scores.toolDiscovery, result.scores.workflowCompletion, result.scores.presetFit, result.scores.errorRate, result.passed ? 1 : 0, JSON.stringify(result.errors), result.durationMs);
|
|
288
|
+
}
|
|
289
|
+
/* ── Generate recommendations ─────────────────────────────────── */
|
|
290
|
+
const recommendations = [];
|
|
291
|
+
const passedCount = results.filter((r) => r.passed).length;
|
|
292
|
+
const overallPassRate = results.length > 0 ? passedCount / results.length : 0;
|
|
293
|
+
// Analyze failure patterns
|
|
294
|
+
const lowDiscovery = results.filter((r) => r.scores.toolDiscovery < 0.5);
|
|
295
|
+
if (lowDiscovery.length > 0) {
|
|
296
|
+
recommendations.push(`Tool discovery scored low for: ${lowDiscovery.map((r) => r.name).join(", ")}. ` +
|
|
297
|
+
`Consider adding more tags/aliases to tool registry entries for their workflow queries.`);
|
|
298
|
+
}
|
|
299
|
+
const lowPresetFit = results.filter((r) => r.scores.presetFit < 1);
|
|
300
|
+
if (lowPresetFit.length > 0) {
|
|
301
|
+
recommendations.push(`Preset overflow for: ${lowPresetFit.map((r) => `${r.name} (${r.toolCount}/${AGENT_PERSONAS.find((p) => p.name === r.name)?.maxTools ?? "?"})`).join(", ")}. ` +
|
|
302
|
+
`Create a slimmer preset or enable progressive discovery for these agents.`);
|
|
303
|
+
}
|
|
304
|
+
const lowWorkflow = results.filter((r) => r.scores.workflowCompletion < 0.5);
|
|
305
|
+
if (lowWorkflow.length > 0) {
|
|
306
|
+
recommendations.push(`Missing workflow chains for: ${lowWorkflow.map((r) => `${r.name} (${AGENT_PERSONAS.find((p) => p.name === r.name)?.typicalWorkflow})`).join(", ")}. ` +
|
|
307
|
+
`Add WORKFLOW_CHAINS entries in toolRegistry.ts for these workflows.`);
|
|
308
|
+
}
|
|
309
|
+
const highErrorRate = results.filter((r) => r.scores.errorRate < 0.75);
|
|
310
|
+
if (highErrorRate.length > 0) {
|
|
311
|
+
recommendations.push(`High error rate for: ${highErrorRate.map((r) => `${r.name} (${r.errors.length} errors)`).join(", ")}. ` +
|
|
312
|
+
`Review error details for root cause.`);
|
|
313
|
+
}
|
|
314
|
+
if (overallPassRate === 1) {
|
|
315
|
+
recommendations.push("All personas passed. Consider adding edge-case personas or stricter thresholds.");
|
|
316
|
+
}
|
|
317
|
+
return {
|
|
318
|
+
personas: results,
|
|
319
|
+
overallPassRate,
|
|
320
|
+
recommendations,
|
|
321
|
+
totalDurationMs: Date.now() - overallStart,
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
/* ================================================================== */
|
|
325
|
+
/* WebMCP Readiness Check */
|
|
326
|
+
/* ================================================================== */
|
|
327
|
+
export function checkWebMcpReadiness() {
|
|
328
|
+
const missingItems = [];
|
|
329
|
+
// Check for WebMCP meta tags in index.html
|
|
330
|
+
// These are not yet present — this is a placeholder for future integration
|
|
331
|
+
missingItems.push("navigator.modelContext meta tag in index.html");
|
|
332
|
+
missingItems.push("WebMCP tool declarations (<tool> elements)");
|
|
333
|
+
missingItems.push("WebMCP manifest (/.well-known/mcp.json)");
|
|
334
|
+
missingItems.push("Content-Security-Policy header for WebMCP origins");
|
|
335
|
+
return {
|
|
336
|
+
ready: missingItems.length === 0,
|
|
337
|
+
missingItems,
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
/* ================================================================== */
|
|
341
|
+
/* MCP Tool: validate_agent_compatibility */
|
|
342
|
+
/* ================================================================== */
|
|
343
|
+
export const validateAgentCompatibilityTool = {
|
|
344
|
+
name: "validate_agent_compatibility",
|
|
345
|
+
description: "Run the agent validation harness — simulates how AI agents (Claude Code, " +
|
|
346
|
+
"OpenClaw, Cursor, Windsurf, generic MCP) interact with NodeBench tools. " +
|
|
347
|
+
"Scores tool discovery, workflow completion, preset fit, and error rate " +
|
|
348
|
+
"per persona. Pass agentName to run a single persona or omit for all 5.",
|
|
349
|
+
inputSchema: {
|
|
350
|
+
type: "object",
|
|
351
|
+
properties: {
|
|
352
|
+
agentName: {
|
|
353
|
+
type: "string",
|
|
354
|
+
description: "Run a single persona: claude_code, openclaw, cursor, windsurf, or generic_mcp. Omit to run all.",
|
|
355
|
+
enum: ["claude_code", "openclaw", "cursor", "windsurf", "generic_mcp"],
|
|
356
|
+
},
|
|
357
|
+
},
|
|
358
|
+
},
|
|
359
|
+
annotations: {
|
|
360
|
+
readOnlyHint: true,
|
|
361
|
+
},
|
|
362
|
+
handler: async (args) => {
|
|
363
|
+
const result = await runAgentValidation(args.agentName);
|
|
364
|
+
const webMcp = checkWebMcpReadiness();
|
|
365
|
+
return {
|
|
366
|
+
...result,
|
|
367
|
+
webMcpReadiness: webMcp,
|
|
368
|
+
summary: `${result.personas.length} persona(s) tested. ` +
|
|
369
|
+
`Pass rate: ${(result.overallPassRate * 100).toFixed(0)}%. ` +
|
|
370
|
+
`WebMCP: ${webMcp.ready ? "ready" : `not ready (${webMcp.missingItems.length} items missing)`}. ` +
|
|
371
|
+
`Duration: ${result.totalDurationMs}ms.`,
|
|
372
|
+
};
|
|
373
|
+
},
|
|
374
|
+
};
|
|
375
|
+
/* ================================================================== */
|
|
376
|
+
/* CLI entry point */
|
|
377
|
+
/* ================================================================== */
|
|
378
|
+
if (process.argv[1]?.endsWith("agentValidation.ts") || process.argv[1]?.endsWith("agentValidation.js")) {
|
|
379
|
+
const personaArg = process.argv[2];
|
|
380
|
+
console.log("=== NodeBench Agent Validation Harness ===\n");
|
|
381
|
+
runAgentValidation(personaArg)
|
|
382
|
+
.then((result) => {
|
|
383
|
+
for (const p of result.personas) {
|
|
384
|
+
const status = p.passed ? "PASS" : "FAIL";
|
|
385
|
+
console.log(`[${status}] ${p.name} (preset: ${p.presetUsed}, tools: ${p.toolCount})`);
|
|
386
|
+
console.log(` toolDiscovery: ${p.scores.toolDiscovery.toFixed(2)}`);
|
|
387
|
+
console.log(` workflowCompletion: ${p.scores.workflowCompletion.toFixed(2)}`);
|
|
388
|
+
console.log(` presetFit: ${p.scores.presetFit.toFixed(2)}`);
|
|
389
|
+
console.log(` errorRate: ${p.scores.errorRate.toFixed(2)}`);
|
|
390
|
+
if (p.errors.length > 0) {
|
|
391
|
+
console.log(` errors: ${p.errors.join("; ")}`);
|
|
392
|
+
}
|
|
393
|
+
console.log();
|
|
394
|
+
}
|
|
395
|
+
console.log(`Overall pass rate: ${(result.overallPassRate * 100).toFixed(0)}%`);
|
|
396
|
+
console.log(`Duration: ${result.totalDurationMs}ms\n`);
|
|
397
|
+
if (result.recommendations.length > 0) {
|
|
398
|
+
console.log("Recommendations:");
|
|
399
|
+
for (const rec of result.recommendations) {
|
|
400
|
+
console.log(` - ${rec}`);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
const webMcp = checkWebMcpReadiness();
|
|
404
|
+
console.log(`\nWebMCP readiness: ${webMcp.ready ? "READY" : "NOT READY"}`);
|
|
405
|
+
if (webMcp.missingItems.length > 0) {
|
|
406
|
+
console.log(" Missing:");
|
|
407
|
+
for (const item of webMcp.missingItems) {
|
|
408
|
+
console.log(` - ${item}`);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
})
|
|
412
|
+
.catch((err) => {
|
|
413
|
+
console.error("Validation failed:", err);
|
|
414
|
+
process.exit(1);
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
//# sourceMappingURL=agentValidation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agentValidation.js","sourceRoot":"","sources":["../../src/benchmarks/agentValidation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACxC,OAAO,EAAE,YAAY,EAAe,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACnF,OAAO,EACL,YAAY,EACZ,oBAAoB,EAEpB,eAAe,GAChB,MAAM,0BAA0B,CAAC;AA4ClC,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE,MAAM,cAAc,GAAmB;IACrC;QACE,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,QAAQ;QAClB,iBAAiB,EAAE,IAAI;QACvB,aAAa,EAAE,KAAK;QACpB,eAAe,EAAE,SAAS;QAC1B,eAAe,EAAE,cAAc;KAChC;IACD;QACE,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE,QAAQ;QAClB,iBAAiB,EAAE,IAAI;QACvB,aAAa,EAAE,KAAK;QACpB,eAAe,EAAE,MAAM;QACvB,eAAe,EAAE,UAAU;KAC5B;IACD;QACE,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,EAAE;QACZ,iBAAiB,EAAE,KAAK;QACxB,aAAa,EAAE,KAAK;QACpB,eAAe,EAAE,QAAQ;QACzB,eAAe,EAAE,aAAa;KAC/B;IACD;QACE,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE,QAAQ;QAClB,iBAAiB,EAAE,KAAK;QACxB,aAAa,EAAE,KAAK;QACpB,eAAe,EAAE,MAAM;QACvB,eAAe,EAAE,aAAa;KAC/B;IACD;QACE,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,QAAQ;QAClB,iBAAiB,EAAE,IAAI;QACvB,aAAa,EAAE,KAAK;QACpB,eAAe,EAAE,SAAS;QAC1B,eAAe,EAAE,gBAAgB;KAClC;CACF,CAAC;AAEF,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE,MAAM,gBAAgB,GAA6B;IACjD,YAAY,EAAE;QACZ,eAAe;QACf,iBAAiB;QACjB,WAAW;QACX,iBAAiB;KAClB;IACD,QAAQ,EAAE;QACR,UAAU;QACV,YAAY;QACZ,WAAW;QACX,UAAU;KACX;IACD,WAAW,EAAE;QACX,cAAc;QACd,cAAc;QACd,KAAK;QACL,MAAM;KACP;IACD,cAAc,EAAE;QACd,SAAS;QACT,QAAQ;QACR,UAAU;QACV,UAAU;KACX;CACF,CAAC;AAEF,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE,MAAM,cAAc,GAA6B;IAC/C,OAAO,EAAE,CAAC,UAAU,CAAC;IACrB,OAAO,EAAE,CAAC,UAAU,CAAC;IACrB,IAAI,EAAE;QACJ,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU;QAC9D,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,cAAc,EAAE,iBAAiB;QACrE,eAAe,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS;KAC7E;IACD,OAAO,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,iBAAiB,CAAC;IAC/D,MAAM,EAAE,CAAC,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,CAAC;IACjF,IAAI,EAAE,eAAe;CACtB,CAAC;AAEF,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE,IAAI,YAAY,GAAG,KAAK,CAAC;AAEzB,SAAS,YAAY;IACnB,IAAI,YAAY;QAAE,OAAO;IACzB,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;GAmBP,CAAC,CAAC;IACH,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC;AAED,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE,KAAK,UAAU,eAAe,CAAC,OAAqB;IAClD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAkB;QAC5B,aAAa,EAAE,CAAC;QAChB,kBAAkB,EAAE,CAAC;QACrB,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC,EAAE,mBAAmB;KAClC,CAAC;IAEF,qEAAqE;IAErE,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IACxD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,mBAAmB,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;QAC1D,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;QACrB,OAAO;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM;YACN,MAAM,EAAE,KAAK;YACb,MAAM;YACN,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,OAAO,CAAC,eAAe;YACnC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAC;IACJ,CAAC;IAED,IAAI,KAAgB,CAAC;IACrB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnD,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;QACrB,OAAO;YACL,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM;YACN,MAAM,EAAE,KAAK;YACb,MAAM;YACN,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,OAAO,CAAC,eAAe;YACnC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC;IAE/B,sEAAsE;IAEtE,IAAI,SAAS,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QAClC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,wCAAwC;QACxC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;QAC7D,MAAM,CAAC,IAAI,CACT,WAAW,OAAO,CAAC,eAAe,WAAW,SAAS,cAAc,OAAO,CAAC,IAAI,iBAAiB,OAAO,CAAC,QAAQ,EAAE,CACpH,CAAC;IACJ,CAAC;IAED,sEAAsE;IAEtE,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvE,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,aAAa,EAAE,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACpD,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,WAAW,EAAE,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;aACjD,CAAC,CAAC,CAAC;YACJ,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE;gBAChD,KAAK,EAAE,CAAC;gBACR,IAAI,EAAE,QAAQ;aACf,CAAC,CAAC;YACH,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,mBAAmB,KAAK,cAAc,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,MAAM,CAAC,aAAa,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/E,sEAAsE;IAEtE,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC/C,6DAA6D;IAC7D,MAAM,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC5D,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAC3C,gBAAgB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAC9D,CAAC;IAEF,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,CAAC,kBAAkB,GAAG,CAAC,CAAC;IAChC,CAAC;SAAM,CAAC;QACN,uEAAuE;QACvE,MAAM,aAAa,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACtD,gBAAgB,CAAC,IAAI,CACnB,CAAC,EAAE,EAAE,EAAE,CACL,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAClC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAC1B,CACF,CAAC;QACF,MAAM,CAAC,kBAAkB,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC;YAClD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC,CAAC;IACR,CAAC;IAED,qEAAqE;IAErE,+DAA+D;IAC/D,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;IAC1E,IAAI,CAAC;QACH,MAAM,YAAY,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QACtC,iCAAiC;IACnC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,yBAAyB,eAAe,aAAa,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,qEAAqE;IAErE,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,8CAA8C;IACrE,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;IAEhE,qEAAqE;IAErE,MAAM,QAAQ,GACZ,CAAC,MAAM,CAAC,aAAa;QACnB,MAAM,CAAC,kBAAkB;QACzB,MAAM,CAAC,SAAS;QAChB,MAAM,CAAC,SAAS,CAAC;QACnB,CAAC,CAAC;IACJ,MAAM,MAAM,GAAG,QAAQ,IAAI,GAAG,CAAC;IAE/B,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM;QACN,MAAM;QACN,MAAM;QACN,SAAS;QACT,UAAU,EAAE,OAAO,CAAC,eAAe;QACnC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;KACjC,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,aAAsB;IAEtB,YAAY,EAAE,CAAC;IACf,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEhC,MAAM,QAAQ,GAAG,aAAa;QAC5B,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC;QACxD,CAAC,CAAC,cAAc,CAAC;IAEnB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;QAC3C,OAAO;YACL,QAAQ,EAAE,EAAE;YACZ,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,CAAC,2BAA2B,aAAa,iBAAiB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1H,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY;SAC3C,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAoB,EAAE,CAAC;IAEpC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAErB,oBAAoB;QACpB,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;QACnB,EAAE,CAAC,OAAO,CAAC;;;;;;KAMV,CAAC,CAAC,GAAG,CACJ,KAAK,CAAC,IAAI,CAAC,EACX,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EACxB,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,MAAM,CAAC,aAAa,EAC3B,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAChC,MAAM,CAAC,MAAM,CAAC,SAAS,EACvB,MAAM,CAAC,MAAM,CAAC,SAAS,EACvB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACrB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAC7B,MAAM,CAAC,UAAU,CAClB,CAAC;IACJ,CAAC;IAED,qEAAqE;IAErE,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE9E,2BAA2B;IAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC;IACzE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,eAAe,CAAC,IAAI,CAClB,kCAAkC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YAC9E,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IACnE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,eAAe,CAAC,IAAI,CAClB,wBAAwB,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,SAAS,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,QAAQ,IAAI,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YAC5J,2EAA2E,CAC9E,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC;IAC7E,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,eAAe,CAAC,IAAI,CAClB,gCAAgC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,eAAe,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACpJ,qEAAqE,CACxE,CAAC;IACJ,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACvE,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,eAAe,CAAC,IAAI,CAClB,wBAAwB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACtG,sCAAsC,CACzC,CAAC;IACJ,CAAC;IAED,IAAI,eAAe,KAAK,CAAC,EAAE,CAAC;QAC1B,eAAe,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAC;IAC1G,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,OAAO;QACjB,eAAe;QACf,eAAe;QACf,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY;KAC3C,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE,MAAM,UAAU,oBAAoB;IAClC,MAAM,YAAY,GAAa,EAAE,CAAC;IAElC,2CAA2C;IAC3C,2EAA2E;IAC3E,YAAY,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;IACnE,YAAY,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IAChE,YAAY,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;IAC7D,YAAY,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;IAEvE,OAAO;QACL,KAAK,EAAE,YAAY,CAAC,MAAM,KAAK,CAAC;QAChC,YAAY;KACb,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE,MAAM,CAAC,MAAM,8BAA8B,GAAY;IACrD,IAAI,EAAE,8BAA8B;IACpC,WAAW,EACT,2EAA2E;QAC3E,0EAA0E;QAC1E,yEAAyE;QACzE,wEAAwE;IAC1E,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,iGAAiG;gBACnG,IAAI,EAAE,CAAC,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,CAAC;aACvE;SACF;KACF;IACD,WAAW,EAAE;QACX,YAAY,EAAE,IAAI;KACnB;IACD,OAAO,EAAE,KAAK,EAAE,IAA4B,EAAE,EAAE;QAC9C,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAEtC,OAAO;YACL,GAAG,MAAM;YACT,eAAe,EAAE,MAAM;YACvB,OAAO,EACL,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,sBAAsB;gBAC/C,cAAc,CAAC,MAAM,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;gBAC5D,WAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,MAAM,CAAC,YAAY,CAAC,MAAM,iBAAiB,IAAI;gBACjG,aAAa,MAAM,CAAC,eAAe,KAAK;SAC3C,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,wEAAwE;AACxE,yEAAyE;AACzE,wEAAwE;AAExE,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;IACvG,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAE5D,kBAAkB,CAAC,UAAU,CAAC;SAC3B,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;QACf,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,UAAU,YAAY,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;YACtF,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1E,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC/E,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACtE,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACtE,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClD,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChF,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,eAAe,MAAM,CAAC,CAAC;QAEvD,IAAI,MAAM,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YAChC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,oBAAoB,EAAE,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAC3E,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBACvC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;QACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testProviderBus.ts — Integration test for ProviderBus WebSocket event bus.
|
|
3
|
+
*
|
|
4
|
+
* Tests: auth, registration, event routing, disconnect cleanup.
|
|
5
|
+
* Run: npx tsx src/benchmarks/testProviderBus.ts
|
|
6
|
+
*/
|
|
7
|
+
import { createServer } from "node:http";
|
|
8
|
+
import { WebSocket } from "ws";
|
|
9
|
+
import { ProviderBus, } from "../../../../server/providerBus.js";
|
|
10
|
+
import { generateApiKey, getMemoryStore, } from "../../../../server/mcpAuth.js";
|
|
11
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
12
|
+
// Test helpers
|
|
13
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
14
|
+
const results = [];
|
|
15
|
+
function record(name, pass, detail) {
|
|
16
|
+
results.push({ name, pass, detail });
|
|
17
|
+
const icon = pass ? "PASS" : "FAIL";
|
|
18
|
+
console.log(` [${icon}] ${name}${detail ? ` — ${detail}` : ""}`);
|
|
19
|
+
}
|
|
20
|
+
/** Wait for a WS message matching a predicate, with timeout. */
|
|
21
|
+
function waitForMessage(ws, predicate, timeoutMs = 5000) {
|
|
22
|
+
return new Promise((resolve, reject) => {
|
|
23
|
+
const timer = setTimeout(() => {
|
|
24
|
+
ws.removeListener("message", handler);
|
|
25
|
+
reject(new Error(`Timed out waiting for message (${timeoutMs}ms)`));
|
|
26
|
+
}, timeoutMs);
|
|
27
|
+
function handler(data) {
|
|
28
|
+
try {
|
|
29
|
+
const parsed = JSON.parse(typeof data === "string" ? data : data.toString("utf-8"));
|
|
30
|
+
if (predicate(parsed)) {
|
|
31
|
+
clearTimeout(timer);
|
|
32
|
+
ws.removeListener("message", handler);
|
|
33
|
+
resolve(parsed);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
// ignore parse errors, keep waiting
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
ws.on("message", handler);
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
/** Connect a WS client to the bus with Bearer auth. Returns the ws + first message. */
|
|
44
|
+
function connectClient(port, apiKey) {
|
|
45
|
+
return new Promise((resolve, reject) => {
|
|
46
|
+
const ws = new WebSocket(`ws://127.0.0.1:${port}/bus`, {
|
|
47
|
+
headers: { Authorization: `Bearer ${apiKey}` },
|
|
48
|
+
});
|
|
49
|
+
let firstMessage = null;
|
|
50
|
+
let resolved = false;
|
|
51
|
+
ws.on("message", (data) => {
|
|
52
|
+
if (!firstMessage) {
|
|
53
|
+
try {
|
|
54
|
+
firstMessage = JSON.parse(typeof data === "string" ? data : data.toString("utf-8"));
|
|
55
|
+
}
|
|
56
|
+
catch { /* ignore */ }
|
|
57
|
+
if (!resolved) {
|
|
58
|
+
resolved = true;
|
|
59
|
+
resolve({ ws, firstMessage });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
ws.on("open", () => {
|
|
64
|
+
// Give a brief window for the welcome message to arrive
|
|
65
|
+
setTimeout(() => {
|
|
66
|
+
if (!resolved) {
|
|
67
|
+
resolved = true;
|
|
68
|
+
resolve({ ws, firstMessage });
|
|
69
|
+
}
|
|
70
|
+
}, 500);
|
|
71
|
+
});
|
|
72
|
+
ws.on("error", (err) => {
|
|
73
|
+
if (!resolved) {
|
|
74
|
+
resolved = true;
|
|
75
|
+
reject(err);
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
81
|
+
// Setup: HTTP server + ProviderBus + seed an API key
|
|
82
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
83
|
+
async function run() {
|
|
84
|
+
console.log("\n=== ProviderBus Integration Tests ===\n");
|
|
85
|
+
// 1. Generate a valid API key and seed it into the in-memory store
|
|
86
|
+
const generated = generateApiKey();
|
|
87
|
+
const rawKey = generated.rawKey;
|
|
88
|
+
const store = getMemoryStore();
|
|
89
|
+
const keyRecord = {
|
|
90
|
+
keyHash: generated.keyHash,
|
|
91
|
+
keyHashPrefix: generated.keyHashPrefix,
|
|
92
|
+
userId: "test-user-1",
|
|
93
|
+
permissions: ["tools:read", "tools:execute"],
|
|
94
|
+
rateLimits: { perMinute: 100, perDay: 10_000 },
|
|
95
|
+
createdAt: Date.now(),
|
|
96
|
+
lastUsedAt: Date.now(),
|
|
97
|
+
revokedAt: null,
|
|
98
|
+
};
|
|
99
|
+
store.set(generated.keyHashPrefix, keyRecord);
|
|
100
|
+
// 2. Create HTTP server
|
|
101
|
+
const httpServer = createServer((_req, res) => {
|
|
102
|
+
res.writeHead(404);
|
|
103
|
+
res.end();
|
|
104
|
+
});
|
|
105
|
+
// 3. Create ProviderBus with short timeouts for testing
|
|
106
|
+
const busConfig = {
|
|
107
|
+
maxConnections: 10,
|
|
108
|
+
heartbeatIntervalMs: 60_000, // long — we don't need heartbeats during test
|
|
109
|
+
idleTimeoutMs: 60_000,
|
|
110
|
+
rateLimitPerMinute: 100,
|
|
111
|
+
};
|
|
112
|
+
const bus = new ProviderBus(busConfig);
|
|
113
|
+
bus.attachToServer(httpServer, "/bus");
|
|
114
|
+
// Wire upgrade handling
|
|
115
|
+
httpServer.on("upgrade", (req, socket, head) => {
|
|
116
|
+
const url = new URL(req.url ?? "/", `http://127.0.0.1`);
|
|
117
|
+
if (url.pathname === "/bus") {
|
|
118
|
+
bus.handleUpgrade(req, socket, head).catch((err) => {
|
|
119
|
+
console.error(` [ERROR] handleUpgrade error:`, err);
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
socket.destroy();
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
// 4. Start listening
|
|
127
|
+
const port = await new Promise((resolve) => {
|
|
128
|
+
httpServer.listen(0, "127.0.0.1", () => {
|
|
129
|
+
const addr = httpServer.address();
|
|
130
|
+
if (typeof addr === "object" && addr !== null) {
|
|
131
|
+
resolve(addr.port);
|
|
132
|
+
}
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
console.log(` Server listening on 127.0.0.1:${port}`);
|
|
136
|
+
console.log("");
|
|
137
|
+
try {
|
|
138
|
+
// ── Test 1: Connect with valid API key ──────────────────────────────
|
|
139
|
+
let ws;
|
|
140
|
+
try {
|
|
141
|
+
const conn = await connectClient(port, rawKey);
|
|
142
|
+
ws = conn.ws;
|
|
143
|
+
const welcome = conn.firstMessage;
|
|
144
|
+
record("Connect with valid API key", welcome !== null && welcome.type === "heartbeat" && typeof welcome.providerId === "string", welcome ? `providerId=${welcome.providerId}` : "no welcome message received");
|
|
145
|
+
}
|
|
146
|
+
catch (err) {
|
|
147
|
+
record("Connect with valid API key", false, String(err));
|
|
148
|
+
// Can't continue without a connection
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
// ── Test 2: Connect with invalid API key ─────────────────────────────
|
|
152
|
+
try {
|
|
153
|
+
const badConn = await connectClient(port, "nb_key_00000000000000000000000000000000");
|
|
154
|
+
// If we get here the connection opened — wait briefly for a close
|
|
155
|
+
await new Promise((resolve, reject) => {
|
|
156
|
+
badConn.ws.on("close", () => resolve());
|
|
157
|
+
badConn.ws.on("error", () => resolve());
|
|
158
|
+
setTimeout(() => {
|
|
159
|
+
badConn.ws.close();
|
|
160
|
+
reject(new Error("Bad key connection was not rejected"));
|
|
161
|
+
}, 2000);
|
|
162
|
+
});
|
|
163
|
+
record("Reject invalid API key", true);
|
|
164
|
+
}
|
|
165
|
+
catch (err) {
|
|
166
|
+
// Connection refused or error is also acceptable (socket destroyed before upgrade completes)
|
|
167
|
+
const msg = String(err);
|
|
168
|
+
if (msg.includes("not rejected")) {
|
|
169
|
+
record("Reject invalid API key", false, msg);
|
|
170
|
+
}
|
|
171
|
+
else {
|
|
172
|
+
record("Reject invalid API key", true, "Connection rejected/errored as expected");
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
// ── Test 3: Register as a provider ───────────────────────────────────
|
|
176
|
+
try {
|
|
177
|
+
ws.send(JSON.stringify({
|
|
178
|
+
type: "register",
|
|
179
|
+
registration: {
|
|
180
|
+
providerName: "test-agent",
|
|
181
|
+
providerType: "custom",
|
|
182
|
+
capabilities: ["chat", "tool_execution"],
|
|
183
|
+
version: "1.0.0",
|
|
184
|
+
},
|
|
185
|
+
}));
|
|
186
|
+
const registered = await waitForMessage(ws, (m) => m.type === "registered", 3000);
|
|
187
|
+
record("Register as provider", registered.type === "registered" && typeof registered.providerId === "string", `providerId=${registered.providerId}`);
|
|
188
|
+
}
|
|
189
|
+
catch (err) {
|
|
190
|
+
record("Register as provider", false, String(err));
|
|
191
|
+
}
|
|
192
|
+
// ── Test 4: Send an event ────────────────────────────────────────────
|
|
193
|
+
try {
|
|
194
|
+
const eventPromise = new Promise((resolve) => {
|
|
195
|
+
bus.once("event", (evt) => {
|
|
196
|
+
resolve(evt.type === "chat.message" && evt.payload?.text === "hello");
|
|
197
|
+
});
|
|
198
|
+
setTimeout(() => resolve(false), 3000);
|
|
199
|
+
});
|
|
200
|
+
ws.send(JSON.stringify({
|
|
201
|
+
type: "event",
|
|
202
|
+
event: {
|
|
203
|
+
type: "chat.message",
|
|
204
|
+
payload: { text: "hello" },
|
|
205
|
+
},
|
|
206
|
+
}));
|
|
207
|
+
const eventReceived = await eventPromise;
|
|
208
|
+
record("Send event and bus emits it", eventReceived);
|
|
209
|
+
// Verify event log
|
|
210
|
+
const log = bus.getEventLog(10);
|
|
211
|
+
const found = log.some((e) => e.type === "chat.message");
|
|
212
|
+
record("Event logged in event log", found, `log size=${log.length}`);
|
|
213
|
+
}
|
|
214
|
+
catch (err) {
|
|
215
|
+
record("Send event and bus emits it", false, String(err));
|
|
216
|
+
}
|
|
217
|
+
// ── Test 5: Health snapshot ──────────────────────────────────────────
|
|
218
|
+
try {
|
|
219
|
+
const health = bus.getHealthSnapshot();
|
|
220
|
+
record("Health snapshot returns data", health.status === "healthy" && health.providers.connected >= 1, `connected=${health.providers.connected}, registered=${health.providers.registered}`);
|
|
221
|
+
}
|
|
222
|
+
catch (err) {
|
|
223
|
+
record("Health snapshot returns data", false, String(err));
|
|
224
|
+
}
|
|
225
|
+
// ── Test 6: Connected providers list ─────────────────────────────────
|
|
226
|
+
try {
|
|
227
|
+
const providers = bus.getConnectedProviders();
|
|
228
|
+
const testProvider = providers.find((p) => p.providerName === "test-agent");
|
|
229
|
+
record("Connected providers list includes test-agent", !!testProvider && testProvider.providerType === "custom", testProvider
|
|
230
|
+
? `capabilities=${testProvider.capabilities.join(",")}`
|
|
231
|
+
: "not found");
|
|
232
|
+
}
|
|
233
|
+
catch (err) {
|
|
234
|
+
record("Connected providers list includes test-agent", false, String(err));
|
|
235
|
+
}
|
|
236
|
+
// ── Test 7: Disconnect and cleanup ──────────────────────────────────
|
|
237
|
+
try {
|
|
238
|
+
const disconnectPromise = new Promise((resolve) => {
|
|
239
|
+
bus.once("provider:disconnected", () => resolve(true));
|
|
240
|
+
setTimeout(() => resolve(false), 3000);
|
|
241
|
+
});
|
|
242
|
+
ws.close();
|
|
243
|
+
const disconnected = await disconnectPromise;
|
|
244
|
+
record("Disconnect fires provider:disconnected", disconnected);
|
|
245
|
+
// Verify cleanup
|
|
246
|
+
const afterProviders = bus.getConnectedProviders();
|
|
247
|
+
record("Provider removed from connected list after disconnect", afterProviders.length === 0, `remaining=${afterProviders.length}`);
|
|
248
|
+
}
|
|
249
|
+
catch (err) {
|
|
250
|
+
record("Disconnect and cleanup", false, String(err));
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
finally {
|
|
254
|
+
// Cleanup
|
|
255
|
+
bus.shutdown();
|
|
256
|
+
store.delete(generated.keyHashPrefix);
|
|
257
|
+
httpServer.close();
|
|
258
|
+
// Summary
|
|
259
|
+
console.log("\n=== Summary ===");
|
|
260
|
+
const passed = results.filter((r) => r.pass).length;
|
|
261
|
+
const total = results.length;
|
|
262
|
+
console.log(` ${passed}/${total} tests passed\n`);
|
|
263
|
+
if (passed < total) {
|
|
264
|
+
process.exit(1);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
run().catch((err) => {
|
|
269
|
+
console.error("Fatal error:", err);
|
|
270
|
+
process.exit(1);
|
|
271
|
+
});
|
|
272
|
+
//# sourceMappingURL=testProviderBus.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"testProviderBus.js","sourceRoot":"","sources":["../../src/benchmarks/testProviderBus.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAA6B,MAAM,WAAW,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC/B,OAAO,EACL,WAAW,GAEZ,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,cAAc,EACd,cAAc,GAIf,MAAM,+BAA+B,CAAC;AAEvC,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E,MAAM,OAAO,GAA4D,EAAE,CAAC;AAE5E,SAAS,MAAM,CAAC,IAAY,EAAE,IAAa,EAAE,MAAe;IAC1D,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,gEAAgE;AAChE,SAAS,cAAc,CACrB,EAAa,EACb,SAAoD,EACpD,SAAS,GAAG,IAAI;IAEhB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,EAAE,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,KAAK,CAAC,kCAAkC,SAAS,KAAK,CAAC,CAAC,CAAC;QACtE,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,SAAS,OAAO,CAAC,IAAqB;YACpC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CACvB,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAC9B,CAAC;gBAC7B,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;oBACtB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,EAAE,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;oBACtC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,oCAAoC;YACtC,CAAC;QACH,CAAC;QACD,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uFAAuF;AACvF,SAAS,aAAa,CACpB,IAAY,EACZ,MAAc;IAEd,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,kBAAkB,IAAI,MAAM,EAAE;YACrD,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,MAAM,EAAE,EAAE;SAC/C,CAAC,CAAC;QACH,IAAI,YAAY,GAAmC,IAAI,CAAC;QACxD,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAqB,EAAE,EAAE;YACzC,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,IAAI,CAAC;oBACH,YAAY,GAAG,IAAI,CAAC,KAAK,CACvB,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAC9B,CAAC;gBAC/B,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;gBACxB,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,IAAI,CAAC;oBAChB,OAAO,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACjB,wDAAwD;YACxD,UAAU,CAAC,GAAG,EAAE;gBACd,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,IAAI,CAAC;oBAChB,OAAO,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC,EAAE,GAAG,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACrB,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAE9E,KAAK,UAAU,GAAG;IAChB,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;IAEzD,mEAAmE;IACnE,MAAM,SAAS,GAAG,cAAc,EAAE,CAAC;IACnC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;IAChC,MAAM,KAAK,GAAG,cAAc,EAAE,CAAC;IAE/B,MAAM,SAAS,GAAiB;QAC9B,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,aAAa,EAAE,SAAS,CAAC,aAAa;QACtC,MAAM,EAAE,aAAa;QACrB,WAAW,EAAE,CAAC,YAAY,EAAE,eAAe,CAAC;QAC5C,UAAU,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE;QAC9C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;QACtB,SAAS,EAAE,IAAI;KAChB,CAAC;IACF,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IAE9C,wBAAwB;IACxB,MAAM,UAAU,GAAe,YAAY,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;QACxD,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACnB,GAAG,CAAC,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IAEH,wDAAwD;IACxD,MAAM,SAAS,GAAsB;QACnC,cAAc,EAAE,EAAE;QAClB,mBAAmB,EAAE,MAAM,EAAE,8CAA8C;QAC3E,aAAa,EAAE,MAAM;QACrB,kBAAkB,EAAE,GAAG;KACxB,CAAC;IACF,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,CAAC;IACvC,GAAG,CAAC,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAEvC,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;QAC7C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,kBAAkB,CAAC,CAAC;QACxD,IAAI,GAAG,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;YAC5B,GAAG,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACjD,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;YACvD,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,qBAAqB;IACrB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,EAAE;QACjD,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE;YACrC,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAC9C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;IAEvD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,IAAI,CAAC;QACH,uEAAuE;QACvE,IAAI,EAAa,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC/C,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC;YAClC,MAAM,CACJ,4BAA4B,EAC5B,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,OAAO,CAAC,UAAU,KAAK,QAAQ,EAC1F,OAAO,CAAC,CAAC,CAAC,cAAc,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,6BAA6B,CAC7E,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,4BAA4B,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACzD,sCAAsC;YACtC,OAAO;QACT,CAAC;QAED,wEAAwE;QACxE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,yCAAyC,CAAC,CAAC;YACrF,kEAAkE;YAClE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC1C,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxC,UAAU,CAAC,GAAG,EAAE;oBACd,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;oBACnB,MAAM,CAAC,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC,CAAC;gBAC3D,CAAC,EAAE,IAAI,CAAC,CAAC;YACX,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6FAA6F;YAC7F,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YACxB,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,wBAAwB,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,wBAAwB,EAAE,IAAI,EAAE,yCAAyC,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;QAED,wEAAwE;QACxE,IAAI,CAAC;YACH,EAAE,CAAC,IAAI,CACL,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE;oBACZ,YAAY,EAAE,YAAY;oBAC1B,YAAY,EAAE,QAAQ;oBACtB,YAAY,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC;oBACxC,OAAO,EAAE,OAAO;iBACjB;aACF,CAAC,CACH,CAAC;YAEF,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,IAAI,CAAC,CAAC;YAClF,MAAM,CACJ,sBAAsB,EACtB,UAAU,CAAC,IAAI,KAAK,YAAY,IAAI,OAAO,UAAU,CAAC,UAAU,KAAK,QAAQ,EAC7E,cAAc,UAAU,CAAC,UAAU,EAAE,CACtC,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,sBAAsB,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACrD,CAAC;QAED,wEAAwE;QACxE,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,IAAI,OAAO,CAAU,CAAC,OAAO,EAAE,EAAE;gBACpD,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;oBACxB,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,KAAK,OAAO,CAAC,CAAC;gBACxE,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,IAAI,CACL,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE;oBACL,IAAI,EAAE,cAAc;oBACpB,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;iBAC3B;aACF,CAAC,CACH,CAAC;YAEF,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC;YACzC,MAAM,CAAC,6BAA6B,EAAE,aAAa,CAAC,CAAC;YAErD,mBAAmB;YACnB,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC;YACzD,MAAM,CAAC,2BAA2B,EAAE,KAAK,EAAE,YAAY,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,6BAA6B,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5D,CAAC;QAED,wEAAwE;QACxE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,GAAG,CAAC,iBAAiB,EAAE,CAAC;YACvC,MAAM,CACJ,8BAA8B,EAC9B,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,IAAI,CAAC,EAC9D,aAAa,MAAM,CAAC,SAAS,CAAC,SAAS,gBAAgB,MAAM,CAAC,SAAS,CAAC,UAAU,EAAE,CACrF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,8BAA8B,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7D,CAAC;QAED,wEAAwE;QACxE,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,GAAG,CAAC,qBAAqB,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,YAAY,CAAC,CAAC;YAC5E,MAAM,CACJ,8CAA8C,EAC9C,CAAC,CAAC,YAAY,IAAI,YAAY,CAAC,YAAY,KAAK,QAAQ,EACxD,YAAY;gBACV,CAAC,CAAC,gBAAgB,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBACvD,CAAC,CAAC,WAAW,CAChB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,8CAA8C,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,uEAAuE;QACvE,IAAI,CAAC;YACH,MAAM,iBAAiB,GAAG,IAAI,OAAO,CAAU,CAAC,OAAO,EAAE,EAAE;gBACzD,GAAG,CAAC,IAAI,CAAC,uBAAuB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;gBACvD,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC;YAC7C,MAAM,CAAC,wCAAwC,EAAE,YAAY,CAAC,CAAC;YAE/D,iBAAiB;YACjB,MAAM,cAAc,GAAG,GAAG,CAAC,qBAAqB,EAAE,CAAC;YACnD,MAAM,CACJ,uDAAuD,EACvD,cAAc,CAAC,MAAM,KAAK,CAAC,EAC3B,aAAa,cAAc,CAAC,MAAM,EAAE,CACrC,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,wBAAwB,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;YAAS,CAAC;QACT,UAAU;QACV,GAAG,CAAC,QAAQ,EAAE,CAAC;QACf,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACtC,UAAU,CAAC,KAAK,EAAE,CAAC;QAEnB,UAAU;QACV,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QACpD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,IAAI,KAAK,iBAAiB,CAAC,CAAC;QAEnD,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;AACH,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IAClB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testSupermemory.ts — Unit test for SupermemoryProvider.
|
|
3
|
+
*
|
|
4
|
+
* Tests WITHOUT a real API key: verifies graceful error handling,
|
|
5
|
+
* interface compliance, and error class hierarchy.
|
|
6
|
+
* Run: npx tsx src/benchmarks/testSupermemory.ts
|
|
7
|
+
*/
|
|
8
|
+
import { SupermemoryProvider, SupermemoryError, SupermemoryAuthError, SupermemoryRateLimitError, } from "../providers/supermemoryProvider.js";
|
|
9
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
10
|
+
// Test helpers
|
|
11
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
12
|
+
const results = [];
|
|
13
|
+
function record(name, pass, detail) {
|
|
14
|
+
results.push({ name, pass, detail });
|
|
15
|
+
const icon = pass ? "PASS" : "FAIL";
|
|
16
|
+
console.log(` [${icon}] ${name}${detail ? ` — ${detail}` : ""}`);
|
|
17
|
+
}
|
|
18
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
19
|
+
// Tests
|
|
20
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
21
|
+
async function run() {
|
|
22
|
+
console.log("\n=== SupermemoryProvider Tests ===\n");
|
|
23
|
+
// ── Test 1: Constructor creates a valid instance ───────────────────────
|
|
24
|
+
const provider = new SupermemoryProvider();
|
|
25
|
+
record("Constructor creates instance", provider instanceof SupermemoryProvider);
|
|
26
|
+
// ── Test 2: Implements MemoryProvider interface (all 12 methods + 2 props)
|
|
27
|
+
const requiredMethods = [
|
|
28
|
+
"connect",
|
|
29
|
+
"disconnect",
|
|
30
|
+
"isConnected",
|
|
31
|
+
"store",
|
|
32
|
+
"update",
|
|
33
|
+
"delete",
|
|
34
|
+
"recall",
|
|
35
|
+
"get",
|
|
36
|
+
"list",
|
|
37
|
+
"relate",
|
|
38
|
+
"getProfile",
|
|
39
|
+
"sync",
|
|
40
|
+
];
|
|
41
|
+
const requiredProps = ["name", "type"];
|
|
42
|
+
let allMethodsPresent = true;
|
|
43
|
+
const missingMethods = [];
|
|
44
|
+
for (const method of requiredMethods) {
|
|
45
|
+
if (typeof provider[method] !== "function") {
|
|
46
|
+
allMethodsPresent = false;
|
|
47
|
+
missingMethods.push(method);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
record(`Implements all ${requiredMethods.length} MemoryProvider methods`, allMethodsPresent, missingMethods.length > 0 ? `missing: ${missingMethods.join(", ")}` : undefined);
|
|
51
|
+
let allPropsPresent = true;
|
|
52
|
+
const missingProps = [];
|
|
53
|
+
for (const prop of requiredProps) {
|
|
54
|
+
if (provider[prop] === undefined) {
|
|
55
|
+
allPropsPresent = false;
|
|
56
|
+
missingProps.push(prop);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
record(`Has required readonly properties (name, type)`, allPropsPresent, missingProps.length > 0 ? `missing: ${missingProps.join(", ")}` : undefined);
|
|
60
|
+
// ── Test 3: name and type values ─────────────────────────────────────
|
|
61
|
+
record("name is 'supermemory'", provider.name === "supermemory", `actual: "${provider.name}"`);
|
|
62
|
+
record("type is 'supermemory'", provider.type === "supermemory", `actual: "${provider.type}"`);
|
|
63
|
+
// ── Test 4: isConnected() before connect() ────────────────────────────
|
|
64
|
+
record("isConnected() is false before connect()", provider.isConnected() === false);
|
|
65
|
+
// ── Test 5: connect() with invalid key — should not throw ─────────────
|
|
66
|
+
// The provider marks itself as connected even if the API returns errors
|
|
67
|
+
// (except for explicit 401 from the validation request).
|
|
68
|
+
// With a fake key hitting a real endpoint, we may get a network error
|
|
69
|
+
// or a 401. The connect() method catches network errors and still marks connected.
|
|
70
|
+
try {
|
|
71
|
+
await provider.connect({ apiKey: "test-invalid-key" });
|
|
72
|
+
// If connect succeeds (network error caught internally), isConnected should be true
|
|
73
|
+
record("connect() with invalid key does not throw", true);
|
|
74
|
+
record("isConnected() returns true after connect()", provider.isConnected() === true);
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
// connect() only rethrows SupermemoryAuthError (401).
|
|
78
|
+
// A real 401 from the API is expected behavior.
|
|
79
|
+
if (err instanceof SupermemoryAuthError) {
|
|
80
|
+
record("connect() with invalid key throws SupermemoryAuthError", true, "401 from API");
|
|
81
|
+
record("isConnected() returns false after auth failure", provider.isConnected() === false);
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
record("connect() with invalid key does not throw", false, String(err));
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// ── Test 6: store() with invalid key — should throw ───────────────────
|
|
88
|
+
// Re-connect to ensure connected state for store/recall tests
|
|
89
|
+
const provider2 = new SupermemoryProvider();
|
|
90
|
+
try {
|
|
91
|
+
await provider2.connect({ apiKey: "test-invalid-key-2" });
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// If 401 from connect, we still test store behavior
|
|
95
|
+
}
|
|
96
|
+
if (provider2.isConnected()) {
|
|
97
|
+
// store() should fail with auth error from the API
|
|
98
|
+
try {
|
|
99
|
+
await provider2.store({ content: "test memory content" });
|
|
100
|
+
record("store() throws on invalid API key", false, "did not throw");
|
|
101
|
+
}
|
|
102
|
+
catch (err) {
|
|
103
|
+
if (err instanceof SupermemoryAuthError) {
|
|
104
|
+
record("store() throws SupermemoryAuthError on invalid key", true);
|
|
105
|
+
record("SupermemoryAuthError has statusCode 401", err.statusCode === 401, `actual: ${err.statusCode}`);
|
|
106
|
+
record("SupermemoryAuthError.name is correct", err.name === "SupermemoryAuthError");
|
|
107
|
+
}
|
|
108
|
+
else if (err instanceof SupermemoryError) {
|
|
109
|
+
// Network error or other API error — still a valid failure mode
|
|
110
|
+
record("store() throws SupermemoryError on invalid key", true, `${err.name}: ${err.message}`);
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
record("store() throws on invalid API key", false, String(err));
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
// ── Test 7: recall() with invalid key — should throw ────────────────
|
|
117
|
+
try {
|
|
118
|
+
await provider2.recall("test query");
|
|
119
|
+
record("recall() throws on invalid API key", false, "did not throw");
|
|
120
|
+
}
|
|
121
|
+
catch (err) {
|
|
122
|
+
if (err instanceof SupermemoryAuthError) {
|
|
123
|
+
record("recall() throws SupermemoryAuthError on invalid key", true);
|
|
124
|
+
}
|
|
125
|
+
else if (err instanceof SupermemoryError) {
|
|
126
|
+
record("recall() throws SupermemoryError on invalid key", true, `${err.name}: ${err.message}`);
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
record("recall() throws on invalid API key", false, String(err));
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
record("store()/recall() tests skipped", true, "provider not connected (401 on connect)");
|
|
135
|
+
}
|
|
136
|
+
// ── Test 8: Error class hierarchy ─────────────────────────────────────
|
|
137
|
+
const authErr = new SupermemoryAuthError();
|
|
138
|
+
record("SupermemoryAuthError extends SupermemoryError", authErr instanceof SupermemoryError);
|
|
139
|
+
record("SupermemoryAuthError extends Error", authErr instanceof Error);
|
|
140
|
+
record("SupermemoryAuthError.statusCode is 401", authErr.statusCode === 401);
|
|
141
|
+
const rateLimitErr = new SupermemoryRateLimitError(5000);
|
|
142
|
+
record("SupermemoryRateLimitError extends SupermemoryError", rateLimitErr instanceof SupermemoryError);
|
|
143
|
+
record("SupermemoryRateLimitError.statusCode is 429", rateLimitErr.statusCode === 429);
|
|
144
|
+
record("SupermemoryRateLimitError.retryAfterMs is set", rateLimitErr.retryAfterMs === 5000);
|
|
145
|
+
// ── Test 9: sync() returns no-op result ───────────────────────────────
|
|
146
|
+
const provider3 = new SupermemoryProvider();
|
|
147
|
+
try {
|
|
148
|
+
await provider3.connect({ apiKey: "test-sync-key" });
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
// ignore auth error
|
|
152
|
+
}
|
|
153
|
+
if (provider3.isConnected()) {
|
|
154
|
+
try {
|
|
155
|
+
const syncResult = await provider3.sync("both");
|
|
156
|
+
record("sync() returns no-op SyncResult", syncResult.pushed === 0 &&
|
|
157
|
+
syncResult.pulled === 0 &&
|
|
158
|
+
syncResult.conflicts === 0 &&
|
|
159
|
+
syncResult.direction === "both", `direction=${syncResult.direction}, pushed=${syncResult.pushed}`);
|
|
160
|
+
}
|
|
161
|
+
catch (err) {
|
|
162
|
+
record("sync() returns no-op SyncResult", false, String(err));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
// ── Test 10: disconnect() ─────────────────────────────────────────────
|
|
166
|
+
if (provider3.isConnected()) {
|
|
167
|
+
await provider3.disconnect();
|
|
168
|
+
record("disconnect() sets isConnected to false", provider3.isConnected() === false);
|
|
169
|
+
}
|
|
170
|
+
// ── Test 11: TypeScript structural compatibility ──────────────────────
|
|
171
|
+
// Verify the provider satisfies MemoryProvider at the type level
|
|
172
|
+
const _typeCheck = new SupermemoryProvider();
|
|
173
|
+
record("Structural type compatibility with MemoryProvider", true);
|
|
174
|
+
// Summary
|
|
175
|
+
console.log("\n=== Summary ===");
|
|
176
|
+
const passed = results.filter((r) => r.pass).length;
|
|
177
|
+
const total = results.length;
|
|
178
|
+
console.log(` ${passed}/${total} tests passed\n`);
|
|
179
|
+
if (passed < total) {
|
|
180
|
+
process.exit(1);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
run().catch((err) => {
|
|
184
|
+
console.error("Fatal error:", err);
|
|
185
|
+
process.exit(1);
|
|
186
|
+
});
|
|
187
|
+
//# sourceMappingURL=testSupermemory.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"testSupermemory.js","sourceRoot":"","sources":["../../src/benchmarks/testSupermemory.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,EACpB,yBAAyB,GAC1B,MAAM,qCAAqC,CAAC;AAG7C,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E,MAAM,OAAO,GAA4D,EAAE,CAAC;AAE5E,SAAS,MAAM,CAAC,IAAY,EAAE,IAAa,EAAE,MAAe;IAC1D,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,KAAK,UAAU,GAAG;IAChB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IAErD,0EAA0E;IAC1E,MAAM,QAAQ,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC3C,MAAM,CAAC,8BAA8B,EAAE,QAAQ,YAAY,mBAAmB,CAAC,CAAC;IAEhF,4EAA4E;IAC5E,MAAM,eAAe,GAAG;QACtB,SAAS;QACT,YAAY;QACZ,aAAa;QACb,OAAO;QACP,QAAQ;QACR,QAAQ;QACR,QAAQ;QACR,KAAK;QACL,MAAM;QACN,QAAQ;QACR,YAAY;QACZ,MAAM;KACE,CAAC;IAEX,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,MAAM,CAAU,CAAC;IAEhD,IAAI,iBAAiB,GAAG,IAAI,CAAC;IAC7B,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;QACrC,IAAI,OAAQ,QAA+C,CAAC,MAAM,CAAC,KAAK,UAAU,EAAE,CAAC;YACnF,iBAAiB,GAAG,KAAK,CAAC;YAC1B,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,MAAM,CACJ,kBAAkB,eAAe,CAAC,MAAM,yBAAyB,EACjE,iBAAiB,EACjB,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAChF,CAAC;IAEF,IAAI,eAAe,GAAG,IAAI,CAAC;IAC3B,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,IAAK,QAA+C,CAAC,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACzE,eAAe,GAAG,KAAK,CAAC;YACxB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,MAAM,CACJ,+CAA+C,EAC/C,eAAe,EACf,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAC5E,CAAC;IAEF,wEAAwE;IACxE,MAAM,CACJ,uBAAuB,EACvB,QAAQ,CAAC,IAAI,KAAK,aAAa,EAC/B,YAAY,QAAQ,CAAC,IAAI,GAAG,CAC7B,CAAC;IACF,MAAM,CACJ,uBAAuB,EACvB,QAAQ,CAAC,IAAI,KAAK,aAAa,EAC/B,YAAY,QAAQ,CAAC,IAAI,GAAG,CAC7B,CAAC;IAEF,yEAAyE;IACzE,MAAM,CACJ,yCAAyC,EACzC,QAAQ,CAAC,WAAW,EAAE,KAAK,KAAK,CACjC,CAAC;IAEF,yEAAyE;IACzE,wEAAwE;IACxE,yDAAyD;IACzD,sEAAsE;IACtE,mFAAmF;IACnF,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACvD,oFAAoF;QACpF,MAAM,CAAC,2CAA2C,EAAE,IAAI,CAAC,CAAC;QAC1D,MAAM,CACJ,4CAA4C,EAC5C,QAAQ,CAAC,WAAW,EAAE,KAAK,IAAI,CAChC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,sDAAsD;QACtD,gDAAgD;QAChD,IAAI,GAAG,YAAY,oBAAoB,EAAE,CAAC;YACxC,MAAM,CAAC,wDAAwD,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;YACvF,MAAM,CACJ,gDAAgD,EAChD,QAAQ,CAAC,WAAW,EAAE,KAAK,KAAK,CACjC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,2CAA2C,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,8DAA8D;IAC9D,MAAM,SAAS,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC5C,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,oDAAoD;IACtD,CAAC;IAED,IAAI,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC;QAC5B,mDAAmD;QACnD,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,qBAAqB,EAAE,CAAC,CAAC;YAC1D,MAAM,CAAC,mCAAmC,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC;QACtE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,oBAAoB,EAAE,CAAC;gBACxC,MAAM,CAAC,oDAAoD,EAAE,IAAI,CAAC,CAAC;gBACnE,MAAM,CACJ,yCAAyC,EACzC,GAAG,CAAC,UAAU,KAAK,GAAG,EACtB,WAAW,GAAG,CAAC,UAAU,EAAE,CAC5B,CAAC;gBACF,MAAM,CACJ,sCAAsC,EACtC,GAAG,CAAC,IAAI,KAAK,sBAAsB,CACpC,CAAC;YACJ,CAAC;iBAAM,IAAI,GAAG,YAAY,gBAAgB,EAAE,CAAC;gBAC3C,gEAAgE;gBAChE,MAAM,CACJ,gDAAgD,EAChD,IAAI,EACJ,GAAG,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO,EAAE,CAC9B,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,mCAAmC,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACrC,MAAM,CAAC,oCAAoC,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,oBAAoB,EAAE,CAAC;gBACxC,MAAM,CAAC,qDAAqD,EAAE,IAAI,CAAC,CAAC;YACtE,CAAC;iBAAM,IAAI,GAAG,YAAY,gBAAgB,EAAE,CAAC;gBAC3C,MAAM,CACJ,iDAAiD,EACjD,IAAI,EACJ,GAAG,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO,EAAE,CAC9B,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,oCAAoC,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,gCAAgC,EAAE,IAAI,EAAE,yCAAyC,CAAC,CAAC;IAC5F,CAAC;IAED,yEAAyE;IACzE,MAAM,OAAO,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC3C,MAAM,CACJ,+CAA+C,EAC/C,OAAO,YAAY,gBAAgB,CACpC,CAAC;IACF,MAAM,CACJ,oCAAoC,EACpC,OAAO,YAAY,KAAK,CACzB,CAAC;IACF,MAAM,CACJ,wCAAwC,EACxC,OAAO,CAAC,UAAU,KAAK,GAAG,CAC3B,CAAC;IAEF,MAAM,YAAY,GAAG,IAAI,yBAAyB,CAAC,IAAI,CAAC,CAAC;IACzD,MAAM,CACJ,oDAAoD,EACpD,YAAY,YAAY,gBAAgB,CACzC,CAAC;IACF,MAAM,CACJ,6CAA6C,EAC7C,YAAY,CAAC,UAAU,KAAK,GAAG,CAChC,CAAC;IACF,MAAM,CACJ,+CAA+C,EAC/C,YAAY,CAAC,YAAY,KAAK,IAAI,CACnC,CAAC;IAEF,yEAAyE;IACzE,MAAM,SAAS,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC5C,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,oBAAoB;IACtB,CAAC;IAED,IAAI,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChD,MAAM,CACJ,iCAAiC,EACjC,UAAU,CAAC,MAAM,KAAK,CAAC;gBACrB,UAAU,CAAC,MAAM,KAAK,CAAC;gBACvB,UAAU,CAAC,SAAS,KAAK,CAAC;gBAC1B,UAAU,CAAC,SAAS,KAAK,MAAM,EACjC,aAAa,UAAU,CAAC,SAAS,YAAY,UAAU,CAAC,MAAM,EAAE,CACjE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,iCAAiC,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,IAAI,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC;QAC5B,MAAM,SAAS,CAAC,UAAU,EAAE,CAAC;QAC7B,MAAM,CACJ,wCAAwC,EACxC,SAAS,CAAC,WAAW,EAAE,KAAK,KAAK,CAClC,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,iEAAiE;IACjE,MAAM,UAAU,GAAmB,IAAI,mBAAmB,EAAE,CAAC;IAC7D,MAAM,CAAC,mDAAmD,EAAE,IAAI,CAAC,CAAC;IAElE,UAAU;IACV,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACpD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,IAAI,KAAK,iBAAiB,CAAC,CAAC;IAEnD,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IAClB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nodebench-mcp",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.49.0",
|
|
4
4
|
"mcpName": "io.github.HomenShum/nodebench",
|
|
5
5
|
"description": "341 MCP tools across 55 domains. Operating intelligence for founders: decision packets, structured research, agent verification, quality gates, progressive discovery with 8-mode hybrid search + neural embeddings, TOON encoding, Agent-as-a-Graph bipartite search. Starter: 15 tools. Core: 81 tools. Full: 341 tools.",
|
|
6
6
|
"type": "module",
|