@polderlabs/bizar 4.8.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bizar-dash/dist/assets/icons-CFqu2M-c.js +656 -0
- package/bizar-dash/dist/assets/icons-CFqu2M-c.js.map +1 -0
- package/bizar-dash/dist/assets/index-DmpSFPJY.js +9 -0
- package/bizar-dash/dist/assets/index-DmpSFPJY.js.map +1 -0
- package/bizar-dash/dist/assets/main-Dl8yY5_H.js +16 -0
- package/bizar-dash/dist/assets/main-Dl8yY5_H.js.map +1 -0
- package/bizar-dash/dist/assets/{main-DX_Jh8Wc.css → main-ZAfGKENE.css} +1 -1
- package/bizar-dash/dist/assets/markdown-DIquRulQ.js +29 -0
- package/bizar-dash/dist/assets/markdown-DIquRulQ.js.map +1 -0
- package/bizar-dash/dist/assets/mobile-C2gysFOZ.js +2 -0
- package/bizar-dash/dist/assets/mobile-C2gysFOZ.js.map +1 -0
- package/bizar-dash/dist/assets/mobile-DHXXbn1A.js +1 -0
- package/bizar-dash/dist/assets/{mobile-Chvf9u_B.js.map → mobile-DHXXbn1A.js.map} +1 -1
- package/bizar-dash/dist/assets/react-vendor-DZRUXSPQ.js +40 -0
- package/bizar-dash/dist/assets/react-vendor-DZRUXSPQ.js.map +1 -0
- package/bizar-dash/dist/index.html +6 -3
- package/bizar-dash/dist/mobile.html +5 -2
- package/bizar-dash/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json +1 -1
- package/bizar-dash/skills/eval/SKILL.md +237 -0
- package/bizar-dash/src/server/api.mjs +28 -0
- package/bizar-dash/src/server/auth.mjs +155 -1
- package/bizar-dash/src/server/eval-store.mjs +226 -0
- package/bizar-dash/src/server/eval.mjs +347 -0
- package/bizar-dash/src/server/memory-lightrag.mjs +109 -0
- package/bizar-dash/src/server/memory-store.mjs +121 -0
- package/bizar-dash/src/server/ocr.mjs +55 -0
- package/bizar-dash/src/server/otel.mjs +133 -0
- package/bizar-dash/src/server/plugins/registry.mjs +363 -0
- package/bizar-dash/src/server/plugins/sandbox.mjs +655 -0
- package/bizar-dash/src/server/plugins/store.mjs +659 -0
- package/bizar-dash/src/server/routes/chat.mjs +246 -170
- package/bizar-dash/src/server/routes/clipboard.mjs +173 -0
- package/bizar-dash/src/server/routes/eval.mjs +147 -0
- package/bizar-dash/src/server/routes/memory.mjs +46 -0
- package/bizar-dash/src/server/routes/ocr.mjs +182 -0
- package/bizar-dash/src/server/routes/opencode-sessions.mjs +82 -48
- package/bizar-dash/src/server/routes/plugins.mjs +220 -0
- package/bizar-dash/src/server/routes/users.mjs +84 -0
- package/bizar-dash/src/server/routes/voice.mjs +131 -0
- package/bizar-dash/src/server/routes/workspaces.mjs +204 -0
- package/bizar-dash/src/server/server.mjs +40 -0
- package/bizar-dash/src/server/voice-store.mjs +202 -0
- package/bizar-dash/src/server/voice-transcribe.mjs +72 -0
- package/bizar-dash/src/server/workspaces.mjs +626 -0
- package/bizar-dash/src/web/components/InviteDialog.tsx +205 -0
- package/bizar-dash/src/web/components/ScreenshotCapture.tsx +138 -0
- package/bizar-dash/src/web/components/ScreenshotOCR.tsx +42 -0
- package/bizar-dash/src/web/components/SettingsSearch.tsx +204 -89
- package/bizar-dash/src/web/components/VoiceNotesPanel.tsx +182 -0
- package/bizar-dash/src/web/components/VoiceRecorder.tsx +104 -0
- package/bizar-dash/src/web/components/WorkspaceSelector.tsx +158 -0
- package/bizar-dash/src/web/lib/search.ts +115 -0
- package/bizar-dash/src/web/mobile/views/MobileSettings.tsx +10 -35
- package/bizar-dash/src/web/mobile/views/QrCodePanel.tsx +69 -0
- package/bizar-dash/src/web/styles/memory.css +166 -1
- package/bizar-dash/src/web/styles/settings.css +80 -0
- package/bizar-dash/src/web/views/Memory.tsx +22 -2
- package/bizar-dash/src/web/views/Settings.tsx +99 -0
- package/bizar-dash/src/web/views/Workspace.tsx +294 -0
- package/bizar-dash/src/web/views/memory/FromScreenshotPanel.tsx +23 -0
- package/bizar-dash/src/web/views/memory/MemoryGraphLegend.tsx +29 -0
- package/bizar-dash/src/web/views/memory/MemoryGraphPanel.tsx +192 -0
- package/bizar-dash/src/web/views/memory/MemoryGraphView.tsx +336 -0
- package/bizar-dash/src/web/views/memory/VaultFromClipboardPanel.tsx +101 -0
- package/bizar-dash/src/web/views/settings/WorkspacesSection.tsx +189 -0
- package/bizar-dash/tests/bundle-analysis.test.mjs +73 -0
- package/bizar-dash/tests/clipboard.test.mjs +147 -0
- package/bizar-dash/tests/components/screenshot-ocr.test.tsx +75 -0
- package/bizar-dash/tests/components/settings-search.test.tsx +180 -0
- package/bizar-dash/tests/components/workspace-selector.test.tsx +73 -0
- package/bizar-dash/tests/deploy-templates.test.mjs +100 -0
- package/bizar-dash/tests/docker-build.test.mjs +96 -0
- package/bizar-dash/tests/eval/fixtures.test.mjs +141 -0
- package/bizar-dash/tests/eval/report.test.mjs +284 -0
- package/bizar-dash/tests/eval/runner.test.mjs +471 -0
- package/bizar-dash/tests/lib/search-fuzzy.test.ts +149 -0
- package/bizar-dash/tests/memory-graph-view.test.tsx +69 -0
- package/bizar-dash/tests/memory-graph.test.mjs +95 -0
- package/bizar-dash/tests/ocr.test.mjs +87 -0
- package/bizar-dash/tests/otel.test.mjs +188 -0
- package/bizar-dash/tests/plugins-registry.test.mjs +387 -0
- package/bizar-dash/tests/plugins-sandbox.test.mjs +374 -0
- package/bizar-dash/tests/plugins-store.test.mjs +455 -0
- package/bizar-dash/tests/users.test.mjs +108 -0
- package/bizar-dash/tests/voice-recorder.test.tsx +95 -0
- package/bizar-dash/tests/voice-store.test.mjs +148 -0
- package/bizar-dash/tests/voice-transcribe.test.mjs +87 -0
- package/bizar-dash/tests/workspaces.test.mjs +527 -0
- package/cli/bin.mjs +72 -2
- package/cli/commands/clip.mjs +146 -0
- package/cli/commands/dash.mjs +6 -0
- package/cli/commands/deploy/cloudflare.mjs +250 -0
- package/cli/commands/deploy/docker.mjs +221 -0
- package/cli/commands/deploy/fly.mjs +161 -0
- package/cli/commands/deploy/vercel.mjs +225 -0
- package/cli/commands/deploy.mjs +240 -0
- package/cli/commands/eval.mjs +378 -0
- package/cli/commands/marketplace.mjs +64 -0
- package/cli/commands/ocr.mjs +165 -0
- package/cli/commands/plugin.mjs +358 -0
- package/cli/commands/voice.mjs +211 -0
- package/cli/commands/workspace.mjs +247 -0
- package/package.json +12 -2
- package/templates/deploy/cloudflare/README.md +32 -0
- package/templates/deploy/cloudflare/functions-index.template.js +15 -0
- package/templates/deploy/cloudflare/wrangler.toml.template +9 -0
- package/templates/deploy/docker/.env.template +16 -0
- package/templates/deploy/docker/README.md +58 -0
- package/templates/deploy/docker/docker-compose.template.yml +23 -0
- package/templates/deploy/fly/README.md +35 -0
- package/templates/deploy/fly/fly.toml.template +28 -0
- package/templates/deploy/vercel/README.md +29 -0
- package/templates/deploy/vercel/api-index.template.js +18 -0
- package/templates/deploy/vercel/vercel.json.template +16 -0
- package/templates/eval-fixtures/README.md +58 -0
- package/templates/eval-fixtures/code-search-basic.json +28 -0
- package/templates/eval-fixtures/latency-bounds.json +16 -0
- package/templates/eval-fixtures/regression-suite.json +79 -0
- package/templates/eval-fixtures/response-format.json +30 -0
- package/templates/eval-fixtures/tool-call-correctness.json +24 -0
- package/templates/plugin-template/README.md +121 -0
- package/templates/plugin-template/index.js +66 -0
- package/templates/plugin-template/plugin.json +42 -0
- package/templates/plugin-template/tests/plugin.test.js +83 -0
- package/bizar-dash/dist/assets/main-DHZmbnxQ.js +0 -361
- package/bizar-dash/dist/assets/main-DHZmbnxQ.js.map +0 -1
- package/bizar-dash/dist/assets/mobile-BK8-ythT.js +0 -351
- package/bizar-dash/dist/assets/mobile-BK8-ythT.js.map +0 -1
- package/bizar-dash/dist/assets/mobile-Chvf9u_B.js +0 -1
|
@@ -0,0 +1,471 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tests/eval/runner.test.mjs
|
|
3
|
+
*
|
|
4
|
+
* v5.0.0 — Tests for the eval runner.
|
|
5
|
+
*
|
|
6
|
+
* Verifies:
|
|
7
|
+
* - runFixture returns correct shape
|
|
8
|
+
* - All check types work (contains, notContains, regex, jsonSchema, maxTokens, maxLatencyMs)
|
|
9
|
+
* - Latency tracking
|
|
10
|
+
* - Timeout enforcement
|
|
11
|
+
* - Suite parallelism
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { describe, it, beforeEach, afterEach } from 'node:test';
|
|
15
|
+
import assert from 'node:assert';
|
|
16
|
+
import { tmpdir } from 'node:os';
|
|
17
|
+
import { join, dirname } from 'node:path';
|
|
18
|
+
import { mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
19
|
+
import { fileURLToPath } from 'node:url';
|
|
20
|
+
|
|
21
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
22
|
+
const EVAL = await import('../../src/server/eval.mjs');
|
|
23
|
+
|
|
24
|
+
// ── Mock LLM call helpers ──────────────────────────────────────────────────────
|
|
25
|
+
|
|
26
|
+
function mockLlmCall({ content = 'ok', latencyMs = 10, tokens = 100 } = {}) {
|
|
27
|
+
return async (prompt, opts) => {
|
|
28
|
+
await new Promise((r) => setTimeout(r, latencyMs));
|
|
29
|
+
return {
|
|
30
|
+
content,
|
|
31
|
+
usage: {
|
|
32
|
+
inputTokens: tokens,
|
|
33
|
+
outputTokens: tokens,
|
|
34
|
+
totalTokens: tokens * 2,
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function failingLlmCall() {
|
|
41
|
+
return async (prompt, opts) => {
|
|
42
|
+
throw new Error('LLM error');
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// ── Fixtures ───────────────────────────────────────────────────────────────────
|
|
47
|
+
|
|
48
|
+
describe('eval runner', () => {
|
|
49
|
+
|
|
50
|
+
describe('runFixture', () => {
|
|
51
|
+
|
|
52
|
+
it('returns correct shape', async () => {
|
|
53
|
+
const fixture = {
|
|
54
|
+
id: 'test-fixture',
|
|
55
|
+
name: 'Test',
|
|
56
|
+
description: 'Test fixture',
|
|
57
|
+
agent: 'thor',
|
|
58
|
+
prompt: 'Say hello',
|
|
59
|
+
expected: {},
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const result = await EVAL.runFixture(fixture, {
|
|
63
|
+
llmCall: mockLlmCall({ content: 'hello world' }),
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
assert.strictEqual(result.fixtureId, 'test-fixture');
|
|
67
|
+
assert.strictEqual(typeof result.ok, 'boolean');
|
|
68
|
+
assert.ok(Array.isArray(result.checks));
|
|
69
|
+
assert.ok(typeof result.latencyMs === 'number');
|
|
70
|
+
assert.strictEqual(result.content, 'hello world');
|
|
71
|
+
assert.ok(result.usage);
|
|
72
|
+
assert.strictEqual(result.usage.totalTokens, 200);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
describe('contains check', () => {
|
|
76
|
+
it('passes when content includes expected substring', async () => {
|
|
77
|
+
const fixture = {
|
|
78
|
+
id: 'test',
|
|
79
|
+
agent: 'thor',
|
|
80
|
+
prompt: 'test',
|
|
81
|
+
expected: { contains: ['hello', 'world'] },
|
|
82
|
+
};
|
|
83
|
+
const result = await EVAL.runFixture(fixture, {
|
|
84
|
+
llmCall: mockLlmCall({ content: 'hello world' }),
|
|
85
|
+
});
|
|
86
|
+
assert.strictEqual(result.ok, true);
|
|
87
|
+
const containsCheck = result.checks.find((c) => c.kind === 'contains');
|
|
88
|
+
assert.ok(containsCheck);
|
|
89
|
+
assert.strictEqual(containsCheck.ok, true);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it('fails when content does not include expected substring', async () => {
|
|
93
|
+
const fixture = {
|
|
94
|
+
id: 'test',
|
|
95
|
+
agent: 'thor',
|
|
96
|
+
prompt: 'test',
|
|
97
|
+
expected: { contains: ['goodbye'] },
|
|
98
|
+
};
|
|
99
|
+
const result = await EVAL.runFixture(fixture, {
|
|
100
|
+
llmCall: mockLlmCall({ content: 'hello world' }),
|
|
101
|
+
});
|
|
102
|
+
assert.strictEqual(result.ok, false);
|
|
103
|
+
const containsCheck = result.checks.find((c) => c.kind === 'contains');
|
|
104
|
+
assert.ok(containsCheck);
|
|
105
|
+
assert.strictEqual(containsCheck.ok, false);
|
|
106
|
+
assert.ok(containsCheck.message.includes('goodbye'));
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
describe('notContains check', () => {
|
|
111
|
+
it('passes when content does NOT include forbidden substring', async () => {
|
|
112
|
+
const fixture = {
|
|
113
|
+
id: 'test',
|
|
114
|
+
agent: 'thor',
|
|
115
|
+
prompt: 'test',
|
|
116
|
+
expected: { notContains: ['error', 'fail'] },
|
|
117
|
+
};
|
|
118
|
+
const result = await EVAL.runFixture(fixture, {
|
|
119
|
+
llmCall: mockLlmCall({ content: 'hello world' }),
|
|
120
|
+
});
|
|
121
|
+
assert.strictEqual(result.ok, true);
|
|
122
|
+
const notContainsCheck = result.checks.find((c) => c.kind === 'notContains');
|
|
123
|
+
assert.ok(notContainsCheck);
|
|
124
|
+
assert.strictEqual(notContainsCheck.ok, true);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it('fails when content includes forbidden substring', async () => {
|
|
128
|
+
const fixture = {
|
|
129
|
+
id: 'test',
|
|
130
|
+
agent: 'thor',
|
|
131
|
+
prompt: 'test',
|
|
132
|
+
expected: { notContains: ['error'] },
|
|
133
|
+
};
|
|
134
|
+
const result = await EVAL.runFixture(fixture, {
|
|
135
|
+
llmCall: mockLlmCall({ content: 'hello error world' }),
|
|
136
|
+
});
|
|
137
|
+
assert.strictEqual(result.ok, false);
|
|
138
|
+
const notContainsCheck = result.checks.find((c) => c.kind === 'notContains');
|
|
139
|
+
assert.ok(notContainsCheck);
|
|
140
|
+
assert.strictEqual(notContainsCheck.ok, false);
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
describe('regex check', () => {
|
|
145
|
+
it('passes when regex matches', async () => {
|
|
146
|
+
const fixture = {
|
|
147
|
+
id: 'test',
|
|
148
|
+
agent: 'thor',
|
|
149
|
+
prompt: 'test',
|
|
150
|
+
expected: { regex: ['hello\\s+world'] },
|
|
151
|
+
};
|
|
152
|
+
const result = await EVAL.runFixture(fixture, {
|
|
153
|
+
llmCall: mockLlmCall({ content: 'hello world' }),
|
|
154
|
+
});
|
|
155
|
+
assert.strictEqual(result.ok, true);
|
|
156
|
+
const regexCheck = result.checks.find((c) => c.kind === 'regex');
|
|
157
|
+
assert.ok(regexCheck);
|
|
158
|
+
assert.strictEqual(regexCheck.ok, true);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
it('fails when regex does not match', async () => {
|
|
162
|
+
const fixture = {
|
|
163
|
+
id: 'test',
|
|
164
|
+
agent: 'thor',
|
|
165
|
+
prompt: 'test',
|
|
166
|
+
expected: { regex: ['goodbye\\s+world'] },
|
|
167
|
+
};
|
|
168
|
+
const result = await EVAL.runFixture(fixture, {
|
|
169
|
+
llmCall: mockLlmCall({ content: 'hello world' }),
|
|
170
|
+
});
|
|
171
|
+
assert.strictEqual(result.ok, false);
|
|
172
|
+
const regexCheck = result.checks.find((c) => c.kind === 'regex');
|
|
173
|
+
assert.ok(regexCheck);
|
|
174
|
+
assert.strictEqual(regexCheck.ok, false);
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
it('fails gracefully on invalid regex', async () => {
|
|
178
|
+
const fixture = {
|
|
179
|
+
id: 'test',
|
|
180
|
+
agent: 'thor',
|
|
181
|
+
prompt: 'test',
|
|
182
|
+
expected: { regex: ['[invalid'] },
|
|
183
|
+
};
|
|
184
|
+
const result = await EVAL.runFixture(fixture, {
|
|
185
|
+
llmCall: mockLlmCall({ content: 'hello' }),
|
|
186
|
+
});
|
|
187
|
+
assert.strictEqual(result.ok, false);
|
|
188
|
+
const regexCheck = result.checks.find((c) => c.kind === 'regex');
|
|
189
|
+
assert.ok(regexCheck);
|
|
190
|
+
assert.strictEqual(regexCheck.ok, false);
|
|
191
|
+
assert.ok(regexCheck.message.includes('invalid regex'));
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
describe('jsonSchema check', () => {
|
|
196
|
+
it('passes when response is valid JSON matching schema', async () => {
|
|
197
|
+
const fixture = {
|
|
198
|
+
id: 'test',
|
|
199
|
+
agent: 'thor',
|
|
200
|
+
prompt: 'test',
|
|
201
|
+
expected: {
|
|
202
|
+
jsonSchema: {
|
|
203
|
+
type: 'object',
|
|
204
|
+
required: ['status'],
|
|
205
|
+
properties: { status: { type: 'string' } },
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
};
|
|
209
|
+
const result = await EVAL.runFixture(fixture, {
|
|
210
|
+
llmCall: mockLlmCall({ content: '{"status": "ok"}' }),
|
|
211
|
+
});
|
|
212
|
+
assert.strictEqual(result.ok, true);
|
|
213
|
+
const schemaCheck = result.checks.find((c) => c.kind === 'jsonSchema');
|
|
214
|
+
assert.ok(schemaCheck);
|
|
215
|
+
assert.strictEqual(schemaCheck.ok, true);
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
it('fails when response is not valid JSON', async () => {
|
|
219
|
+
const fixture = {
|
|
220
|
+
id: 'test',
|
|
221
|
+
agent: 'thor',
|
|
222
|
+
prompt: 'test',
|
|
223
|
+
expected: { jsonSchema: { type: 'object' } },
|
|
224
|
+
};
|
|
225
|
+
const result = await EVAL.runFixture(fixture, {
|
|
226
|
+
llmCall: mockLlmCall({ content: 'not json' }),
|
|
227
|
+
});
|
|
228
|
+
assert.strictEqual(result.ok, false);
|
|
229
|
+
const schemaCheck = result.checks.find((c) => c.kind === 'jsonSchema');
|
|
230
|
+
assert.ok(schemaCheck);
|
|
231
|
+
assert.strictEqual(schemaCheck.ok, false);
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
it('fails when required property is missing', async () => {
|
|
235
|
+
const fixture = {
|
|
236
|
+
id: 'test',
|
|
237
|
+
agent: 'thor',
|
|
238
|
+
prompt: 'test',
|
|
239
|
+
expected: {
|
|
240
|
+
jsonSchema: {
|
|
241
|
+
type: 'object',
|
|
242
|
+
required: ['version'],
|
|
243
|
+
properties: { version: { type: 'string' } },
|
|
244
|
+
},
|
|
245
|
+
},
|
|
246
|
+
};
|
|
247
|
+
const result = await EVAL.runFixture(fixture, {
|
|
248
|
+
llmCall: mockLlmCall({ content: '{"status": "ok"}' }),
|
|
249
|
+
});
|
|
250
|
+
assert.strictEqual(result.ok, false);
|
|
251
|
+
const schemaCheck = result.checks.find((c) => c.kind === 'jsonSchema');
|
|
252
|
+
assert.ok(schemaCheck);
|
|
253
|
+
assert.strictEqual(schemaCheck.ok, false);
|
|
254
|
+
assert.ok(schemaCheck.message.includes('missing required property'));
|
|
255
|
+
});
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
describe('maxTokens check', () => {
|
|
259
|
+
it('passes when token count is under limit', async () => {
|
|
260
|
+
const fixture = {
|
|
261
|
+
id: 'test',
|
|
262
|
+
agent: 'thor',
|
|
263
|
+
prompt: 'test',
|
|
264
|
+
expected: { maxTokens: 1000 },
|
|
265
|
+
};
|
|
266
|
+
const result = await EVAL.runFixture(fixture, {
|
|
267
|
+
llmCall: mockLlmCall({ tokens: 100 }),
|
|
268
|
+
});
|
|
269
|
+
assert.strictEqual(result.ok, true);
|
|
270
|
+
const tokenCheck = result.checks.find((c) => c.kind === 'maxTokens');
|
|
271
|
+
assert.ok(tokenCheck);
|
|
272
|
+
assert.strictEqual(tokenCheck.ok, true);
|
|
273
|
+
});
|
|
274
|
+
|
|
275
|
+
it('fails when token count exceeds limit', async () => {
|
|
276
|
+
const fixture = {
|
|
277
|
+
id: 'test',
|
|
278
|
+
agent: 'thor',
|
|
279
|
+
prompt: 'test',
|
|
280
|
+
expected: { maxTokens: 50 },
|
|
281
|
+
};
|
|
282
|
+
const result = await EVAL.runFixture(fixture, {
|
|
283
|
+
llmCall: mockLlmCall({ tokens: 100 }),
|
|
284
|
+
});
|
|
285
|
+
assert.strictEqual(result.ok, false);
|
|
286
|
+
const tokenCheck = result.checks.find((c) => c.kind === 'maxTokens');
|
|
287
|
+
assert.ok(tokenCheck);
|
|
288
|
+
assert.strictEqual(tokenCheck.ok, false);
|
|
289
|
+
assert.ok(tokenCheck.message.includes('exceeds max'));
|
|
290
|
+
});
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
describe('maxLatencyMs check', () => {
|
|
294
|
+
it('passes when latency is under limit', async () => {
|
|
295
|
+
const fixture = {
|
|
296
|
+
id: 'test',
|
|
297
|
+
agent: 'thor',
|
|
298
|
+
prompt: 'test',
|
|
299
|
+
expected: { maxLatencyMs: 5000 },
|
|
300
|
+
};
|
|
301
|
+
const result = await EVAL.runFixture(fixture, {
|
|
302
|
+
llmCall: mockLlmCall({ latencyMs: 10 }),
|
|
303
|
+
});
|
|
304
|
+
assert.strictEqual(result.ok, true);
|
|
305
|
+
const latencyCheck = result.checks.find((c) => c.kind === 'maxLatencyMs');
|
|
306
|
+
assert.ok(latencyCheck);
|
|
307
|
+
assert.strictEqual(latencyCheck.ok, true);
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
it('fails when latency exceeds limit', async () => {
|
|
311
|
+
const fixture = {
|
|
312
|
+
id: 'test',
|
|
313
|
+
agent: 'thor',
|
|
314
|
+
prompt: 'test',
|
|
315
|
+
expected: { maxLatencyMs: 5 },
|
|
316
|
+
};
|
|
317
|
+
const result = await EVAL.runFixture(fixture, {
|
|
318
|
+
llmCall: mockLlmCall({ latencyMs: 100 }),
|
|
319
|
+
});
|
|
320
|
+
assert.strictEqual(result.ok, false);
|
|
321
|
+
const latencyCheck = result.checks.find((c) => c.kind === 'maxLatencyMs');
|
|
322
|
+
assert.ok(latencyCheck);
|
|
323
|
+
assert.strictEqual(latencyCheck.ok, false);
|
|
324
|
+
assert.ok(latencyCheck.message.includes('exceeds max'));
|
|
325
|
+
});
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
describe('timeout enforcement', () => {
|
|
329
|
+
it('returns timeout check on timeout', async () => {
|
|
330
|
+
const fixture = {
|
|
331
|
+
id: 'test',
|
|
332
|
+
agent: 'thor',
|
|
333
|
+
prompt: 'test',
|
|
334
|
+
expected: {},
|
|
335
|
+
};
|
|
336
|
+
const result = await EVAL.runFixture(fixture, {
|
|
337
|
+
llmCall: async () => {
|
|
338
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
339
|
+
return { content: 'ok', usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 } };
|
|
340
|
+
},
|
|
341
|
+
timeoutMs: 50,
|
|
342
|
+
});
|
|
343
|
+
assert.strictEqual(result.ok, false);
|
|
344
|
+
const timeoutCheck = result.checks.find((c) => c.kind === 'timeout');
|
|
345
|
+
assert.ok(timeoutCheck);
|
|
346
|
+
assert.strictEqual(timeoutCheck.ok, false);
|
|
347
|
+
});
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
describe('error handling', () => {
|
|
351
|
+
it('returns error check when LLM call throws', async () => {
|
|
352
|
+
const fixture = {
|
|
353
|
+
id: 'test',
|
|
354
|
+
agent: 'thor',
|
|
355
|
+
prompt: 'test',
|
|
356
|
+
expected: {},
|
|
357
|
+
};
|
|
358
|
+
const result = await EVAL.runFixture(fixture, {
|
|
359
|
+
llmCall: failingLlmCall(),
|
|
360
|
+
});
|
|
361
|
+
assert.strictEqual(result.ok, false);
|
|
362
|
+
const errorCheck = result.checks.find((c) => c.kind === 'error');
|
|
363
|
+
assert.ok(errorCheck);
|
|
364
|
+
assert.strictEqual(errorCheck.ok, false);
|
|
365
|
+
assert.ok(errorCheck.message.includes('LLM error'));
|
|
366
|
+
});
|
|
367
|
+
});
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
describe('runSuite', () => {
|
|
371
|
+
const suiteDir = join(tmpdir(), `eval-suite-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
372
|
+
|
|
373
|
+
beforeEach(() => {
|
|
374
|
+
mkdirSync(suiteDir, { recursive: true });
|
|
375
|
+
writeFileSync(join(suiteDir, 'fix1.json'), JSON.stringify({
|
|
376
|
+
id: 'fix1',
|
|
377
|
+
name: 'Fix 1',
|
|
378
|
+
description: 'First fixture',
|
|
379
|
+
agent: 'thor',
|
|
380
|
+
prompt: 'Say hello',
|
|
381
|
+
expected: { contains: ['hello'] },
|
|
382
|
+
}));
|
|
383
|
+
writeFileSync(join(suiteDir, 'fix2.json'), JSON.stringify({
|
|
384
|
+
id: 'fix2',
|
|
385
|
+
name: 'Fix 2',
|
|
386
|
+
description: 'Second fixture',
|
|
387
|
+
agent: 'thor',
|
|
388
|
+
prompt: 'Say goodbye',
|
|
389
|
+
expected: { contains: ['goodbye'] },
|
|
390
|
+
}));
|
|
391
|
+
});
|
|
392
|
+
|
|
393
|
+
afterEach(() => {
|
|
394
|
+
rmSync(suiteDir, { recursive: true, force: true });
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
it('runs all fixtures in a directory', async () => {
|
|
398
|
+
const result = await EVAL.runSuite(suiteDir, {
|
|
399
|
+
llmCall: mockLlmCall({ content: 'hello and goodbye' }),
|
|
400
|
+
concurrency: 5,
|
|
401
|
+
});
|
|
402
|
+
assert.strictEqual(result.total, 2);
|
|
403
|
+
assert.strictEqual(result.passed, 2);
|
|
404
|
+
assert.strictEqual(result.failed, 0);
|
|
405
|
+
assert.strictEqual(result.results.length, 2);
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
it('reports failures correctly', async () => {
|
|
409
|
+
const result = await EVAL.runSuite(suiteDir, {
|
|
410
|
+
llmCall: mockLlmCall({ content: 'hello only' }),
|
|
411
|
+
concurrency: 5,
|
|
412
|
+
});
|
|
413
|
+
assert.strictEqual(result.total, 2);
|
|
414
|
+
assert.strictEqual(result.passed, 1);
|
|
415
|
+
assert.strictEqual(result.failed, 1);
|
|
416
|
+
});
|
|
417
|
+
|
|
418
|
+
it('runs fixtures in parallel batches', async () => {
|
|
419
|
+
const start = Date.now();
|
|
420
|
+
// Create 6 fixtures with unique IDs to test batching
|
|
421
|
+
for (let i = 0; i < 6; i++) {
|
|
422
|
+
writeFileSync(join(suiteDir, `batch${i}.json`), JSON.stringify({
|
|
423
|
+
id: `batch${i}`,
|
|
424
|
+
name: `Batch ${i}`,
|
|
425
|
+
description: 'Test fixture',
|
|
426
|
+
agent: 'thor',
|
|
427
|
+
prompt: 'Say hello',
|
|
428
|
+
expected: { contains: ['hello'] },
|
|
429
|
+
}));
|
|
430
|
+
}
|
|
431
|
+
const result = await EVAL.runSuite(suiteDir, {
|
|
432
|
+
llmCall: mockLlmCall({ latencyMs: 50 }),
|
|
433
|
+
concurrency: 3,
|
|
434
|
+
});
|
|
435
|
+
// 2 original (fix1, fix2) + 6 new (batch0-5) = 8
|
|
436
|
+
assert.ok(result.total >= 8, `Expected at least 8 fixtures, got ${result.total}`);
|
|
437
|
+
// Parallel batches should complete faster than sequential
|
|
438
|
+
const elapsed = Date.now() - start;
|
|
439
|
+
assert.ok(elapsed < 500, `Should complete quickly, took ${elapsed}ms`);
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
it('handles suite with no fixtures', async () => {
|
|
443
|
+
const emptyDir = join(tmpdir(), `eval-empty-${Date.now()}`);
|
|
444
|
+
mkdirSync(emptyDir, { recursive: true });
|
|
445
|
+
const result = await EVAL.runSuite(emptyDir, {
|
|
446
|
+
llmCall: mockLlmCall(),
|
|
447
|
+
concurrency: 5,
|
|
448
|
+
});
|
|
449
|
+
assert.strictEqual(result.total, 0);
|
|
450
|
+
assert.strictEqual(result.passed, 0);
|
|
451
|
+
assert.strictEqual(result.failed, 0);
|
|
452
|
+
rmSync(emptyDir, { recursive: true, force: true });
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
it('loads suite JSON with array of fixtures', async () => {
|
|
456
|
+
const suiteFile = join(suiteDir, 'suite.json');
|
|
457
|
+
writeFileSync(suiteFile, JSON.stringify({
|
|
458
|
+
id: 'my-suite',
|
|
459
|
+
fixtures: [
|
|
460
|
+
{ id: 'f1', name: 'F1', description: 'F1', agent: 'thor', prompt: 'hi', expected: { contains: ['hi'] } },
|
|
461
|
+
{ id: 'f2', name: 'F2', description: 'F2', agent: 'thor', prompt: 'bye', expected: { contains: ['bye'] } },
|
|
462
|
+
],
|
|
463
|
+
}));
|
|
464
|
+
const result = await EVAL.runSuite(suiteDir, {
|
|
465
|
+
llmCall: mockLlmCall({ content: 'hi and bye' }),
|
|
466
|
+
concurrency: 5,
|
|
467
|
+
});
|
|
468
|
+
assert.strictEqual(result.total, 4); // 2 individual + 2 in suite
|
|
469
|
+
});
|
|
470
|
+
});
|
|
471
|
+
});
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tests/lib/search-fuzzy.test.ts
|
|
3
|
+
*
|
|
4
|
+
* v4.9 — Unit tests for fuzzy search with typo tolerance.
|
|
5
|
+
*/
|
|
6
|
+
import { describe, it, expect } from 'vitest';
|
|
7
|
+
import {
|
|
8
|
+
levenshtein,
|
|
9
|
+
scoreField,
|
|
10
|
+
fuzzySearch,
|
|
11
|
+
type SearchableItem,
|
|
12
|
+
} from '../../src/web/lib/search';
|
|
13
|
+
|
|
14
|
+
/* ─── Levenshtein ─── */
|
|
15
|
+
|
|
16
|
+
describe('levenshtein', () => {
|
|
17
|
+
it('returns 0 for identical strings', () => {
|
|
18
|
+
expect(levenshtein('hello', 'hello')).toBe(0);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('returns length for completely different strings', () => {
|
|
22
|
+
expect(levenshtein('abc', 'xyz')).toBe(3);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('counts single-character insertion', () => {
|
|
26
|
+
expect(levenshtein('cat', 'cats')).toBe(1);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
it('counts single-character deletion', () => {
|
|
30
|
+
expect(levenshtein('cats', 'cat')).toBe(1);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it('counts single-character substitution', () => {
|
|
34
|
+
expect(levenshtein('cat', 'car')).toBe(1);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('handles empty strings', () => {
|
|
38
|
+
expect(levenshtein('', 'abc')).toBe(3);
|
|
39
|
+
expect(levenshtein('abc', '')).toBe(3);
|
|
40
|
+
expect(levenshtein('', '')).toBe(0);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it('handles transposition as two edits', () => {
|
|
44
|
+
expect(levenshtein('ab', 'ba')).toBe(2);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
/* ─── scoreField ─── */
|
|
49
|
+
|
|
50
|
+
describe('scoreField', () => {
|
|
51
|
+
const field = 'accent color';
|
|
52
|
+
|
|
53
|
+
it('exact match scores 100', () => {
|
|
54
|
+
expect(scoreField('accent color', field)).toBe(100);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it('prefix match scores 90', () => {
|
|
58
|
+
expect(scoreField('acc', field)).toBe(90);
|
|
59
|
+
expect(scoreField('accent', field)).toBe(90); // starts with "accent"
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('word boundary match scores 80', () => {
|
|
63
|
+
// "color" is a separate word in "accent color"
|
|
64
|
+
expect(scoreField('color', field)).toBe(80);
|
|
65
|
+
// "colo" starts at "color" word boundary, not field start
|
|
66
|
+
expect(scoreField('colo', field)).toBe(80);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('typo within 2 chars scores 40', () => {
|
|
70
|
+
// "accent" vs "axcent" (one char off)
|
|
71
|
+
expect(scoreField('axcent', 'accent color')).toBe(40);
|
|
72
|
+
// "accent" vs "accant" (one char off)
|
|
73
|
+
const score = scoreField('accant', 'accent color');
|
|
74
|
+
expect(score).toBe(40);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it('substring non-prefix, non-word-boundary match scores 30', () => {
|
|
78
|
+
// "cent" is not at a word boundary in "accent color"
|
|
79
|
+
expect(scoreField('cent', 'accent color')).toBe(30);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('returns 0 for no match', () => {
|
|
83
|
+
expect(scoreField('zzzzz', 'accent color')).toBe(0);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('handles case insensitivity via pre-lowered inputs', () => {
|
|
87
|
+
// Our callers lower case before passing — verify the scoring works
|
|
88
|
+
// "accent" is a prefix of "accent color" → scores 90
|
|
89
|
+
expect(scoreField('Accent'.toLowerCase(), 'Accent Color'.toLowerCase())).toBe(90);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
/* ─── fuzzySearch ─── */
|
|
94
|
+
|
|
95
|
+
describe('fuzzySearch', () => {
|
|
96
|
+
const items: SearchableItem[] = [
|
|
97
|
+
{ key: 'theme.accent', label: 'Accent color', section: 'theme', value: '#8b5cf6' },
|
|
98
|
+
{ key: 'theme.fontFamily', label: 'Font family', section: 'theme' },
|
|
99
|
+
{ key: 'ui.layout', label: 'UI layout', section: 'layout', value: 'topnav' },
|
|
100
|
+
{ key: 'defaultAgent', label: 'Default agent', section: 'general', value: 'odin' },
|
|
101
|
+
{ key: 'notifications.onAgentComplete', label: 'Notify on agent complete', section: 'notifications', description: 'Show toast when an agent finishes' },
|
|
102
|
+
];
|
|
103
|
+
|
|
104
|
+
it('returns empty array for empty query', () => {
|
|
105
|
+
expect(fuzzySearch('', items)).toEqual([]);
|
|
106
|
+
expect(fuzzySearch(' ', items)).toEqual([]);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it('finds exact match by label', () => {
|
|
110
|
+
const results = fuzzySearch('accent color', items);
|
|
111
|
+
expect(results.length).toBeGreaterThanOrEqual(1);
|
|
112
|
+
expect(results[0].key).toBe('theme.accent');
|
|
113
|
+
expect(results[0].score).toBe(100);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('finds match by value', () => {
|
|
117
|
+
const results = fuzzySearch('#8b5cf6', items);
|
|
118
|
+
expect(results.some((r) => r.key === 'theme.accent')).toBe(true);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it('finds match by description', () => {
|
|
122
|
+
const results = fuzzySearch('toast', items);
|
|
123
|
+
expect(results.some((r) => r.key === 'notifications.onAgentComplete')).toBe(true);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('finds match via typo tolerance', () => {
|
|
127
|
+
const results = fuzzySearch('axcent', items);
|
|
128
|
+
expect(results.some((r) => r.key === 'theme.accent')).toBe(true);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it('sorts by score descending', () => {
|
|
132
|
+
const results = fuzzySearch('agent', items);
|
|
133
|
+
for (let i = 1; i < results.length; i++) {
|
|
134
|
+
expect(results[i].score).toBeLessThanOrEqual(results[i - 1].score);
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it('returns empty when nothing matches', () => {
|
|
139
|
+
const results = fuzzySearch('zzzznonexistent', items);
|
|
140
|
+
expect(results).toEqual([]);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it('sets matchedField and matchedTerm on results', () => {
|
|
144
|
+
const results = fuzzySearch('accent', items);
|
|
145
|
+
expect(results.length).toBeGreaterThan(0);
|
|
146
|
+
expect(results[0].matchedField).toBe('label');
|
|
147
|
+
expect(results[0].matchedTerm).toBe('Accent color');
|
|
148
|
+
});
|
|
149
|
+
});
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tests/memory-graph-view.test.tsx
|
|
3
|
+
*
|
|
4
|
+
* Tests for MemoryGraphView (SVG force-directed graph).
|
|
5
|
+
*/
|
|
6
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
7
|
+
import { render, screen, fireEvent } from '@testing-library/react';
|
|
8
|
+
import React from 'react';
|
|
9
|
+
import { MemoryGraphView, type GraphData } from '../src/web/views/memory/MemoryGraphView';
|
|
10
|
+
|
|
11
|
+
const EMPTY_GRAPH: GraphData = { nodes: [], edges: [] };
|
|
12
|
+
|
|
13
|
+
const THREE_NODE_GRAPH: GraphData = {
|
|
14
|
+
nodes: [
|
|
15
|
+
{ id: 'notes/alpha.md', label: 'Alpha Note', type: 'note', size: 1, group: 'notes' },
|
|
16
|
+
{ id: 'notes/beta.md', label: 'Beta Note', type: 'note', size: 2, group: 'notes' },
|
|
17
|
+
{ id: 'decisions/dec.md', label: 'Decision', type: 'note', size: 1, group: 'decisions' },
|
|
18
|
+
],
|
|
19
|
+
edges: [
|
|
20
|
+
{ source: 'notes/alpha.md', target: 'notes/beta.md', type: 'links_to', weight: 1 },
|
|
21
|
+
{ source: 'notes/beta.md', target: 'decisions/dec.md', type: 'links_to', weight: 1 },
|
|
22
|
+
],
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
describe('MemoryGraphView', () => {
|
|
26
|
+
beforeEach(() => {
|
|
27
|
+
vi.clearAllMocks();
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('renders SVG with empty graph', () => {
|
|
31
|
+
render(<MemoryGraphView data={EMPTY_GRAPH} />);
|
|
32
|
+
const svg = document.querySelector('.memory-graph-canvas');
|
|
33
|
+
expect(svg).toBeInTheDocument();
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('renders nodes and edges', () => {
|
|
37
|
+
render(<MemoryGraphView data={THREE_NODE_GRAPH} />);
|
|
38
|
+
// SVG should have circles for nodes (one per node).
|
|
39
|
+
const circles = document.querySelectorAll('.memory-graph-node');
|
|
40
|
+
expect(circles.length).toBe(3);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it('calls onNodeClick when a node is clicked', () => {
|
|
44
|
+
const onNodeClick = vi.fn();
|
|
45
|
+
render(<MemoryGraphView data={THREE_NODE_GRAPH} onNodeClick={onNodeClick} />);
|
|
46
|
+
const circles = document.querySelectorAll('.memory-graph-node');
|
|
47
|
+
// Click the first node circle group.
|
|
48
|
+
if (circles.length > 0) {
|
|
49
|
+
fireEvent.click(circles[0]);
|
|
50
|
+
}
|
|
51
|
+
expect(onNodeClick).toHaveBeenCalledTimes(1);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it('renders filtered nodes when filter is applied via data prop', () => {
|
|
55
|
+
const filteredData = {
|
|
56
|
+
nodes: THREE_NODE_GRAPH.nodes.filter((n) => n.label.includes('Alpha')),
|
|
57
|
+
edges: [],
|
|
58
|
+
};
|
|
59
|
+
render(<MemoryGraphView data={filteredData} />);
|
|
60
|
+
const circles = document.querySelectorAll('.memory-graph-node');
|
|
61
|
+
expect(circles.length).toBe(1);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it('node has correct type and group fields', () => {
|
|
65
|
+
render(<MemoryGraphView data={THREE_NODE_GRAPH} />);
|
|
66
|
+
const circles = document.querySelectorAll('.memory-graph-node');
|
|
67
|
+
expect(circles.length).toBeGreaterThan(0);
|
|
68
|
+
});
|
|
69
|
+
});
|