@polderlabs/bizar 4.8.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/bizar-dash/dist/assets/icons-CFqu2M-c.js +656 -0
  2. package/bizar-dash/dist/assets/icons-CFqu2M-c.js.map +1 -0
  3. package/bizar-dash/dist/assets/index-DmpSFPJY.js +9 -0
  4. package/bizar-dash/dist/assets/index-DmpSFPJY.js.map +1 -0
  5. package/bizar-dash/dist/assets/main-Dl8yY5_H.js +16 -0
  6. package/bizar-dash/dist/assets/main-Dl8yY5_H.js.map +1 -0
  7. package/bizar-dash/dist/assets/{main-DX_Jh8Wc.css → main-ZAfGKENE.css} +1 -1
  8. package/bizar-dash/dist/assets/markdown-DIquRulQ.js +29 -0
  9. package/bizar-dash/dist/assets/markdown-DIquRulQ.js.map +1 -0
  10. package/bizar-dash/dist/assets/mobile-C2gysFOZ.js +2 -0
  11. package/bizar-dash/dist/assets/mobile-C2gysFOZ.js.map +1 -0
  12. package/bizar-dash/dist/assets/mobile-DHXXbn1A.js +1 -0
  13. package/bizar-dash/dist/assets/{mobile-Chvf9u_B.js.map → mobile-DHXXbn1A.js.map} +1 -1
  14. package/bizar-dash/dist/assets/react-vendor-DZRUXSPQ.js +40 -0
  15. package/bizar-dash/dist/assets/react-vendor-DZRUXSPQ.js.map +1 -0
  16. package/bizar-dash/dist/index.html +6 -3
  17. package/bizar-dash/dist/mobile.html +5 -2
  18. package/bizar-dash/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json +1 -1
  19. package/bizar-dash/skills/eval/SKILL.md +237 -0
  20. package/bizar-dash/src/server/api.mjs +28 -0
  21. package/bizar-dash/src/server/auth.mjs +155 -1
  22. package/bizar-dash/src/server/eval-store.mjs +226 -0
  23. package/bizar-dash/src/server/eval.mjs +347 -0
  24. package/bizar-dash/src/server/memory-lightrag.mjs +109 -0
  25. package/bizar-dash/src/server/memory-store.mjs +121 -0
  26. package/bizar-dash/src/server/ocr.mjs +55 -0
  27. package/bizar-dash/src/server/otel.mjs +133 -0
  28. package/bizar-dash/src/server/plugins/registry.mjs +363 -0
  29. package/bizar-dash/src/server/plugins/sandbox.mjs +655 -0
  30. package/bizar-dash/src/server/plugins/store.mjs +659 -0
  31. package/bizar-dash/src/server/routes/chat.mjs +246 -170
  32. package/bizar-dash/src/server/routes/clipboard.mjs +173 -0
  33. package/bizar-dash/src/server/routes/eval.mjs +147 -0
  34. package/bizar-dash/src/server/routes/memory.mjs +46 -0
  35. package/bizar-dash/src/server/routes/ocr.mjs +182 -0
  36. package/bizar-dash/src/server/routes/opencode-sessions.mjs +82 -48
  37. package/bizar-dash/src/server/routes/plugins.mjs +220 -0
  38. package/bizar-dash/src/server/routes/users.mjs +84 -0
  39. package/bizar-dash/src/server/routes/voice.mjs +131 -0
  40. package/bizar-dash/src/server/routes/workspaces.mjs +204 -0
  41. package/bizar-dash/src/server/server.mjs +40 -0
  42. package/bizar-dash/src/server/voice-store.mjs +202 -0
  43. package/bizar-dash/src/server/voice-transcribe.mjs +72 -0
  44. package/bizar-dash/src/server/workspaces.mjs +626 -0
  45. package/bizar-dash/src/web/components/InviteDialog.tsx +205 -0
  46. package/bizar-dash/src/web/components/ScreenshotCapture.tsx +138 -0
  47. package/bizar-dash/src/web/components/ScreenshotOCR.tsx +42 -0
  48. package/bizar-dash/src/web/components/SettingsSearch.tsx +204 -89
  49. package/bizar-dash/src/web/components/VoiceNotesPanel.tsx +182 -0
  50. package/bizar-dash/src/web/components/VoiceRecorder.tsx +104 -0
  51. package/bizar-dash/src/web/components/WorkspaceSelector.tsx +158 -0
  52. package/bizar-dash/src/web/lib/search.ts +115 -0
  53. package/bizar-dash/src/web/mobile/views/MobileSettings.tsx +10 -35
  54. package/bizar-dash/src/web/mobile/views/QrCodePanel.tsx +69 -0
  55. package/bizar-dash/src/web/styles/memory.css +166 -1
  56. package/bizar-dash/src/web/styles/settings.css +80 -0
  57. package/bizar-dash/src/web/views/Memory.tsx +22 -2
  58. package/bizar-dash/src/web/views/Settings.tsx +99 -0
  59. package/bizar-dash/src/web/views/Workspace.tsx +294 -0
  60. package/bizar-dash/src/web/views/memory/FromScreenshotPanel.tsx +23 -0
  61. package/bizar-dash/src/web/views/memory/MemoryGraphLegend.tsx +29 -0
  62. package/bizar-dash/src/web/views/memory/MemoryGraphPanel.tsx +192 -0
  63. package/bizar-dash/src/web/views/memory/MemoryGraphView.tsx +336 -0
  64. package/bizar-dash/src/web/views/memory/VaultFromClipboardPanel.tsx +101 -0
  65. package/bizar-dash/src/web/views/settings/WorkspacesSection.tsx +189 -0
  66. package/bizar-dash/tests/bundle-analysis.test.mjs +73 -0
  67. package/bizar-dash/tests/clipboard.test.mjs +147 -0
  68. package/bizar-dash/tests/components/screenshot-ocr.test.tsx +75 -0
  69. package/bizar-dash/tests/components/settings-search.test.tsx +180 -0
  70. package/bizar-dash/tests/components/workspace-selector.test.tsx +73 -0
  71. package/bizar-dash/tests/deploy-templates.test.mjs +100 -0
  72. package/bizar-dash/tests/docker-build.test.mjs +96 -0
  73. package/bizar-dash/tests/eval/fixtures.test.mjs +141 -0
  74. package/bizar-dash/tests/eval/report.test.mjs +284 -0
  75. package/bizar-dash/tests/eval/runner.test.mjs +471 -0
  76. package/bizar-dash/tests/lib/search-fuzzy.test.ts +149 -0
  77. package/bizar-dash/tests/memory-graph-view.test.tsx +69 -0
  78. package/bizar-dash/tests/memory-graph.test.mjs +95 -0
  79. package/bizar-dash/tests/ocr.test.mjs +87 -0
  80. package/bizar-dash/tests/otel.test.mjs +188 -0
  81. package/bizar-dash/tests/plugins-registry.test.mjs +387 -0
  82. package/bizar-dash/tests/plugins-sandbox.test.mjs +374 -0
  83. package/bizar-dash/tests/plugins-store.test.mjs +455 -0
  84. package/bizar-dash/tests/users.test.mjs +108 -0
  85. package/bizar-dash/tests/voice-recorder.test.tsx +95 -0
  86. package/bizar-dash/tests/voice-store.test.mjs +148 -0
  87. package/bizar-dash/tests/voice-transcribe.test.mjs +87 -0
  88. package/bizar-dash/tests/workspaces.test.mjs +527 -0
  89. package/cli/bin.mjs +72 -2
  90. package/cli/commands/clip.mjs +146 -0
  91. package/cli/commands/dash.mjs +6 -0
  92. package/cli/commands/deploy/cloudflare.mjs +250 -0
  93. package/cli/commands/deploy/docker.mjs +221 -0
  94. package/cli/commands/deploy/fly.mjs +161 -0
  95. package/cli/commands/deploy/vercel.mjs +225 -0
  96. package/cli/commands/deploy.mjs +240 -0
  97. package/cli/commands/eval.mjs +378 -0
  98. package/cli/commands/marketplace.mjs +64 -0
  99. package/cli/commands/ocr.mjs +165 -0
  100. package/cli/commands/plugin.mjs +358 -0
  101. package/cli/commands/voice.mjs +211 -0
  102. package/cli/commands/workspace.mjs +247 -0
  103. package/package.json +12 -2
  104. package/templates/deploy/cloudflare/README.md +32 -0
  105. package/templates/deploy/cloudflare/functions-index.template.js +15 -0
  106. package/templates/deploy/cloudflare/wrangler.toml.template +9 -0
  107. package/templates/deploy/docker/.env.template +16 -0
  108. package/templates/deploy/docker/README.md +58 -0
  109. package/templates/deploy/docker/docker-compose.template.yml +23 -0
  110. package/templates/deploy/fly/README.md +35 -0
  111. package/templates/deploy/fly/fly.toml.template +28 -0
  112. package/templates/deploy/vercel/README.md +29 -0
  113. package/templates/deploy/vercel/api-index.template.js +18 -0
  114. package/templates/deploy/vercel/vercel.json.template +16 -0
  115. package/templates/eval-fixtures/README.md +58 -0
  116. package/templates/eval-fixtures/code-search-basic.json +28 -0
  117. package/templates/eval-fixtures/latency-bounds.json +16 -0
  118. package/templates/eval-fixtures/regression-suite.json +79 -0
  119. package/templates/eval-fixtures/response-format.json +30 -0
  120. package/templates/eval-fixtures/tool-call-correctness.json +24 -0
  121. package/templates/plugin-template/README.md +121 -0
  122. package/templates/plugin-template/index.js +66 -0
  123. package/templates/plugin-template/plugin.json +42 -0
  124. package/templates/plugin-template/tests/plugin.test.js +83 -0
  125. package/bizar-dash/dist/assets/main-DHZmbnxQ.js +0 -361
  126. package/bizar-dash/dist/assets/main-DHZmbnxQ.js.map +0 -1
  127. package/bizar-dash/dist/assets/mobile-BK8-ythT.js +0 -351
  128. package/bizar-dash/dist/assets/mobile-BK8-ythT.js.map +0 -1
  129. package/bizar-dash/dist/assets/mobile-Chvf9u_B.js +0 -1
@@ -0,0 +1,73 @@
1
+ import { render, screen, waitFor, act } from '@testing-library/react';
2
+ import userEvent from '@testing-library/user-event';
3
+ import { describe, it, expect, vi, beforeEach } from 'vitest';
4
+ import React from 'react';
5
+ import { WorkspaceSelector } from '../../src/web/components/WorkspaceSelector';
6
+
7
+ // Mock the api module
8
+ vi.mock('../../src/web/lib/api', () => ({
9
+ api: {
10
+ get: vi.fn().mockResolvedValue({ workspaces: [] }),
11
+ },
12
+ }));
13
+
14
+ describe('WorkspaceSelector', () => {
15
+ beforeEach(() => {
16
+ vi.clearAllMocks();
17
+ });
18
+
19
+ it('renders with default text when no workspace is selected', () => {
20
+ render(
21
+ <WorkspaceSelector
22
+ currentWorkspaceId={null}
23
+ onWorkspaceChange={vi.fn()}
24
+ />,
25
+ );
26
+ expect(screen.getByText('Workspace')).toBeInTheDocument();
27
+ });
28
+
29
+ it('opens dropdown on click', async () => {
30
+ const user = userEvent.setup();
31
+ render(
32
+ <WorkspaceSelector
33
+ currentWorkspaceId={null}
34
+ onWorkspaceChange={vi.fn()}
35
+ />,
36
+ );
37
+ await user.click(screen.getByRole('button'));
38
+ expect(screen.getByRole('menu')).toBeInTheDocument();
39
+ });
40
+
41
+ it('shows empty state in dropdown when no workspaces', async () => {
42
+ const user = userEvent.setup();
43
+ render(
44
+ <WorkspaceSelector
45
+ currentWorkspaceId={null}
46
+ onWorkspaceChange={vi.fn()}
47
+ />,
48
+ );
49
+ await user.click(screen.getByRole('button'));
50
+ expect(screen.getByText('No workspaces')).toBeInTheDocument();
51
+ });
52
+
53
+ it('renders workspace name when provided and API returns data', async () => {
54
+ const { api } = await import('../../src/web/lib/api');
55
+ (api.get as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
56
+ workspaces: [
57
+ { workspace: { id: 'ws_abc', name: 'Engineering' }, role: 'admin' },
58
+ ],
59
+ });
60
+
61
+ render(
62
+ <WorkspaceSelector
63
+ currentWorkspaceId="ws_abc"
64
+ onWorkspaceChange={vi.fn()}
65
+ />,
66
+ );
67
+
68
+ // Wait for the effect to run and state to update
69
+ await waitFor(() => {
70
+ expect(screen.getByText('Engineering')).toBeInTheDocument();
71
+ });
72
+ });
73
+ });
@@ -0,0 +1,100 @@
1
+ /**
2
+ * bizar-dash/tests/deploy-templates.test.mjs
3
+ *
4
+ * Verify that deploy templates exist and are valid.
5
+ */
6
+ import { describe, it } from 'node:test';
7
+ import assert from 'node:assert';
8
+ import { readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
9
+ import { join } from 'node:path';
10
+ import { fileURLToPath } from 'node:url';
11
+
12
+ const __dirname = fileURLToPath(new URL('.', import.meta.url));
13
+ const PROJECT_ROOT = join(__dirname, '..', '..');
14
+ const TEMPLATES_DIR = join(PROJECT_ROOT, 'templates', 'deploy');
15
+
16
+ // Lazy-import toml/yaml only if needed (they're devDependencies)
17
+ let toml;
18
+ let yaml;
19
+
20
+ function getToml() {
21
+ if (!toml) {
22
+ toml = import('smol-toml').then((m) => m.parse || m.default?.parse || m.default);
23
+ }
24
+ // synchronous wrapper
25
+ return { parse: (s) => { throw new Error('use async version'); } };
26
+ }
27
+
28
+ async function parseToml(content) {
29
+ const mod = await import('smol-toml');
30
+ return (mod.parse || mod.default?.parse || mod.default)(content);
31
+ }
32
+
33
+ async function parseYaml(content) {
34
+ const mod = await import('js-yaml');
35
+ return (mod.load || mod.default?.load || mod.default)(content);
36
+ }
37
+
38
+ describe('Deploy templates', () => {
39
+ const platforms = ['vercel', 'cloudflare', 'fly', 'docker'];
40
+ const requiredFiles = {
41
+ vercel: ['vercel.json.template', 'api-index.template.js', 'README.md'],
42
+ cloudflare: ['wrangler.toml.template', 'functions-index.template.js', 'README.md'],
43
+ fly: ['fly.toml.template', 'README.md'],
44
+ docker: ['docker-compose.template.yml', '.env.template', 'README.md'],
45
+ };
46
+
47
+ for (const platform of platforms) {
48
+ it(platform + ' template directory exists', () => {
49
+ const dir = join(TEMPLATES_DIR, platform);
50
+ assert.ok(existsSync(dir), 'Directory missing: templates/deploy/' + platform);
51
+ });
52
+
53
+ it(platform + ' has at least 2 files', () => {
54
+ const dir = join(TEMPLATES_DIR, platform);
55
+ const entries = readdirSync(dir).filter((f) => {
56
+ return statSync(join(dir, f)).isFile();
57
+ });
58
+ assert.ok(entries.length >= 2, platform + ' has ' + entries.length + ' files (need >=2): ' + entries.join(', '));
59
+ });
60
+
61
+ for (const file of (requiredFiles[platform] || [])) {
62
+ it(platform + '/' + file + ' exists', () => {
63
+ const fpath = join(TEMPLATES_DIR, platform, file);
64
+ assert.ok(existsSync(fpath), 'Missing file: templates/deploy/' + platform + '/' + file);
65
+ });
66
+ }
67
+ }
68
+
69
+ it('vercel.json.template is valid JSON', () => {
70
+ const content = readFileSync(join(TEMPLATES_DIR, 'vercel', 'vercel.json.template'), 'utf8');
71
+ const parsed = JSON.parse(content);
72
+ assert.ok(parsed.name);
73
+ assert.strictEqual(parsed.version, 2);
74
+ assert.ok(Array.isArray(parsed.routes));
75
+ });
76
+
77
+ it('wrangler.toml.template is valid TOML', { timeout: 10000 }, async () => {
78
+ const content = readFileSync(join(TEMPLATES_DIR, 'cloudflare', 'wrangler.toml.template'), 'utf8');
79
+ const parsed = await parseToml(content);
80
+ assert.ok(parsed);
81
+ assert.ok(parsed.name);
82
+ assert.ok(parsed.site);
83
+ });
84
+
85
+ it('fly.toml.template is valid TOML', { timeout: 10000 }, async () => {
86
+ const content = readFileSync(join(TEMPLATES_DIR, 'fly', 'fly.toml.template'), 'utf8');
87
+ const parsed = await parseToml(content);
88
+ assert.ok(parsed);
89
+ assert.ok(parsed.app);
90
+ assert.ok(parsed.http_service);
91
+ });
92
+
93
+ it('docker-compose.template.yml is valid YAML', { timeout: 10000 }, async () => {
94
+ const content = readFileSync(join(TEMPLATES_DIR, 'docker', 'docker-compose.template.yml'), 'utf8');
95
+ const parsed = await parseYaml(content);
96
+ assert.ok(parsed);
97
+ assert.ok(parsed.services);
98
+ assert.ok(parsed.services['bizar-dash']);
99
+ });
100
+ });
@@ -0,0 +1,96 @@
1
+ /**
2
+ * bizar-dash/tests/docker-build.test.mjs
3
+ *
4
+ * v4.9.0 — Validate Docker deployment configuration files.
5
+ *
6
+ * Checks:
7
+ * - Dockerfile exists and has valid syntax (FROM, CMD/ENTRYPOINT, EXPOSE, HEALTHCHECK)
8
+ * - docker-compose.yml exists with bizar-dash service and ports
9
+ * - .dockerignore exists
10
+ */
11
+
12
+ import { test } from 'node:test';
13
+ import assert from 'node:assert/strict';
14
+ import { readFileSync, existsSync } from 'node:fs';
15
+ import { join, dirname } from 'node:path';
16
+ import { fileURLToPath } from 'node:url';
17
+
18
+ const __dirname = dirname(fileURLToPath(import.meta.url));
19
+ const ROOT = join(__dirname, '..', '..');
20
+
21
+ const DOCKERFILE = join(ROOT, 'Dockerfile');
22
+ const COMPOSE_FILE = join(ROOT, 'docker-compose.yml');
23
+ const DOCKERIGNORE = join(ROOT, '.dockerignore');
24
+
25
+ // ── Dockerfile ──────────────────────────────────────────────────────────────
26
+
27
+ test('Dockerfile exists at root', () => {
28
+ assert.ok(existsSync(DOCKERFILE), 'Dockerfile should exist at project root');
29
+ });
30
+
31
+ test('Dockerfile has FROM instruction', () => {
32
+ const content = readFileSync(DOCKERFILE, 'utf8');
33
+ assert.match(content, /^FROM\s+\S+/m, 'Dockerfile must contain a FROM instruction');
34
+ });
35
+
36
+ test('Dockerfile has CMD or ENTRYPOINT', () => {
37
+ const content = readFileSync(DOCKERFILE, 'utf8');
38
+ assert.ok(
39
+ /^(CMD|ENTRYPOINT)\s+/m.test(content),
40
+ 'Dockerfile must have a CMD or ENTRYPOINT instruction',
41
+ );
42
+ });
43
+
44
+ test('Dockerfile exposes at least one port', () => {
45
+ const content = readFileSync(DOCKERFILE, 'utf8');
46
+ assert.match(content, /^EXPOSE\s+\d+/m, 'Dockerfile must expose at least one port');
47
+ });
48
+
49
+ test('Dockerfile has HEALTHCHECK', () => {
50
+ const content = readFileSync(DOCKERFILE, 'utf8');
51
+ assert.match(content, /^HEALTHCHECK\s+/m, 'Dockerfile must have a HEALTHCHECK instruction');
52
+ });
53
+
54
+ test('Dockerfile uses multi-stage build', () => {
55
+ const content = readFileSync(DOCKERFILE, 'utf8');
56
+ const fromCount = content.match(/^FROM\s+\S+/gm);
57
+ assert.ok(fromCount && fromCount.length >= 2, 'Dockerfile must have at least 2 FROM instructions (multi-stage)');
58
+ });
59
+
60
+ // ── docker-compose.yml ──────────────────────────────────────────────────────
61
+
62
+ test('docker-compose.yml exists at root', () => {
63
+ assert.ok(existsSync(COMPOSE_FILE), 'docker-compose.yml should exist at project root');
64
+ });
65
+
66
+ test('docker-compose.yml has bizar-dash service with build and ports', () => {
67
+ const content = readFileSync(COMPOSE_FILE, 'utf8');
68
+ assert.match(content, /bizar-dash:/, 'docker-compose must define a bizar-dash service');
69
+ assert.match(content, /build:\s*\./, 'bizar-dash service must have build: .');
70
+ assert.match(content, /ports:/, 'bizar-dash service must expose ports');
71
+ });
72
+
73
+ test('docker-compose.yml defines required volumes', () => {
74
+ const content = readFileSync(COMPOSE_FILE, 'utf8');
75
+ assert.match(content, /bizar-config:/, 'Must define bizar-config volume');
76
+ assert.match(content, /bizar-memory:/, 'Must define bizar-memory volume');
77
+ assert.match(content, /bizar-usage:/, 'Must define bizar-usage volume');
78
+ assert.match(content, /bizar-backups:/, 'Must define bizar-backups volume');
79
+ });
80
+
81
+ test('docker-compose.yml has healthcheck', () => {
82
+ const content = readFileSync(COMPOSE_FILE, 'utf8');
83
+ assert.match(content, /healthcheck:/, 'bizar-dash service must have a healthcheck');
84
+ });
85
+
86
+ // ── .dockerignore ───────────────────────────────────────────────────────────
87
+
88
+ test('.dockerignore exists at root', () => {
89
+ assert.ok(existsSync(DOCKERIGNORE), '.dockerignore should exist at project root');
90
+ });
91
+
92
+ test('.dockerignore excludes node_modules and .git', () => {
93
+ const content = readFileSync(DOCKERIGNORE, 'utf8');
94
+ assert.match(content, /node_modules/, '.dockerignore must exclude node_modules');
95
+ assert.match(content, /\.git/, '.dockerignore must exclude .git');
96
+ });
@@ -0,0 +1,141 @@
1
+ /**
2
+ * tests/eval/fixtures.test.mjs
3
+ *
4
+ * v5.0.0 — Tests for fixture loading.
5
+ *
6
+ * Verifies:
7
+ * - Example fixtures load without errors
8
+ * - Fixture JSON is valid and has required fields
9
+ * - loadFixtures correctly loads from a directory
10
+ */
11
+
12
+ import { describe, it } from 'node:test';
13
+ import assert from 'node:assert';
14
+ import { join, dirname } from 'node:path';
15
+ import { fileURLToPath } from 'node:url';
16
+
17
+ const __dirname = dirname(fileURLToPath(import.meta.url));
18
+ const EVAL = await import('../../src/server/eval.mjs');
19
+
20
+ // Fixture files to validate
21
+ const EXAMPLE_FIXTURES = [
22
+ '../../../templates/eval-fixtures/code-search-basic.json',
23
+ '../../../templates/eval-fixtures/tool-call-correctness.json',
24
+ '../../../templates/eval-fixtures/response-format.json',
25
+ '../../../templates/eval-fixtures/latency-bounds.json',
26
+ '../../../templates/eval-fixtures/regression-suite.json',
27
+ ];
28
+
29
+ const REQUIRED_FIELDS = ['id', 'name', 'description', 'agent', 'prompt', 'expected'];
30
+ const REQUIRED_EXPECTED = ['contains', 'notContains', 'regex', 'jsonSchema', 'maxTokens', 'maxLatencyMs'];
31
+
32
+ describe('eval fixtures', () => {
33
+
34
+ describe('example fixtures from templates/', () => {
35
+
36
+ for (const fixturePath of EXAMPLE_FIXTURES) {
37
+ const fullPath = join(__dirname, fixturePath);
38
+ it(`loads and validates: ${fixturePath}`, async () => {
39
+ // Dynamically import so we can check it parses
40
+ const { readFileSync } = await import('node:fs');
41
+ const raw = readFileSync(fullPath, 'utf8');
42
+ const fixture = JSON.parse(raw);
43
+
44
+ // Should have id
45
+ assert.ok(fixture.id, 'fixture must have an id');
46
+
47
+ // If it's a suite with fixtures array, validate each
48
+ if (fixture.fixtures && Array.isArray(fixture.fixtures)) {
49
+ assert.ok(fixture.fixtures.length > 0, 'suite must have at least one fixture');
50
+ for (const f of fixture.fixtures) {
51
+ validateFixture(f);
52
+ }
53
+ } else {
54
+ // Single fixture
55
+ validateFixture(fixture);
56
+ }
57
+ });
58
+ }
59
+ });
60
+
61
+ describe('loadFixtures', () => {
62
+ it('loads fixtures from a directory', async () => {
63
+ const fixturesDir = join(__dirname, '../../../templates/eval-fixtures');
64
+ const fixtures = EVAL.loadFixtures(fixturesDir);
65
+ assert.ok(fixtures.length >= 4, `Expected at least 4 fixtures, got ${fixtures.length}`);
66
+ });
67
+
68
+ it('skips non-JSON files', async () => {
69
+ const { mkdirSync, writeFileSync, rmSync } = await import('node:fs');
70
+ const { tmpdir } = await import('node:os');
71
+ const testDir = join(tmpdir(), `eval-load-test-${Date.now()}`);
72
+ mkdirSync(testDir, { recursive: true });
73
+ writeFileSync(join(testDir, 'readme.md'), '# readme');
74
+ writeFileSync(join(testDir, 'valid.json'), JSON.stringify({
75
+ id: 'test',
76
+ name: 'Test',
77
+ description: 'Test',
78
+ agent: 'thor',
79
+ prompt: 'test',
80
+ expected: { contains: [] },
81
+ }));
82
+ const fixtures = EVAL.loadFixtures(testDir);
83
+ rmSync(testDir, { recursive: true, force: true });
84
+ assert.strictEqual(fixtures.length, 1);
85
+ assert.strictEqual(fixtures[0].id, 'test');
86
+ });
87
+
88
+ it('skips invalid JSON files', async () => {
89
+ const { mkdirSync, writeFileSync, rmSync } = await import('node:fs');
90
+ const { tmpdir } = await import('node:os');
91
+ const testDir = join(tmpdir(), `eval-invalid-test-${Date.now()}`);
92
+ mkdirSync(testDir, { recursive: true });
93
+ writeFileSync(join(testDir, 'invalid.json'), 'not valid json {');
94
+ writeFileSync(join(testDir, 'valid.json'), JSON.stringify({
95
+ id: 'test2',
96
+ name: 'Test2',
97
+ description: 'Test2',
98
+ agent: 'thor',
99
+ prompt: 'test',
100
+ expected: { contains: [] },
101
+ }));
102
+ const fixtures = EVAL.loadFixtures(testDir);
103
+ rmSync(testDir, { recursive: true, force: true });
104
+ assert.strictEqual(fixtures.length, 1);
105
+ assert.strictEqual(fixtures[0].id, 'test2');
106
+ });
107
+
108
+ it('returns empty array for non-existent directory', async () => {
109
+ const fixtures = EVAL.loadFixtures('/nonexistent/path/xyz');
110
+ assert.strictEqual(fixtures.length, 0);
111
+ });
112
+ });
113
+ });
114
+
115
+ /**
116
+ * Validate a fixture object has required fields
117
+ * @param {object} fixture
118
+ */
119
+ function validateFixture(fixture) {
120
+ for (const field of REQUIRED_FIELDS) {
121
+ assert.ok(
122
+ fixture[field] !== undefined && fixture[field] !== null,
123
+ `fixture ${fixture.id || 'unknown'} missing required field: ${field}`,
124
+ );
125
+ }
126
+ assert.ok(
127
+ typeof fixture.expected === 'object',
128
+ `fixture ${fixture.id} expected must be an object`,
129
+ );
130
+ // expected should have at least one check type
131
+ const expected = fixture.expected;
132
+ const hasCheckType = [
133
+ expected.contains,
134
+ expected.notContains,
135
+ expected.regex,
136
+ expected.jsonSchema,
137
+ expected.maxTokens,
138
+ expected.maxLatencyMs,
139
+ ].some((v) => v !== undefined && v !== null);
140
+ assert.ok(hasCheckType, `fixture ${fixture.id} expected must have at least one check type`);
141
+ }
@@ -0,0 +1,284 @@
1
+ /**
2
+ * tests/eval/report.test.mjs
3
+ *
4
+ * v5.0.0 — Tests for eval run comparison / diff reporting.
5
+ *
6
+ * Verifies:
7
+ * - compareRuns correctly categorizes fixtures as improved/regressed/unchanged
8
+ * - saveRun and getRun work correctly
9
+ * - listRuns returns runs in order
10
+ */
11
+
12
+ import { describe, it, beforeEach, afterEach } from 'node:test';
13
+ import assert from 'node:assert';
14
+ import { tmpdir } from 'node:os';
15
+ import { join } from 'node:path';
16
+ import { mkdirSync, rmSync } from 'node:fs';
17
+
18
+ const storeHome = join(tmpdir(), `bizar-eval-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
19
+ process.env.BIZAR_STORE_HOME = storeHome;
20
+ mkdirSync(storeHome, { recursive: true });
21
+
22
+ const EVAL_STORE = await import('../../src/server/eval-store.mjs');
23
+
24
+ describe('eval store — run comparison', () => {
25
+
26
+ afterEach(() => {
27
+ EVAL_STORE.__resetStoreForTests();
28
+ });
29
+
30
+ describe('compareRuns', () => {
31
+ it('categorizes improved fixtures (was failing, now passing)', async () => {
32
+ const run1 = {
33
+ id: 'run_1',
34
+ startedAt: new Date().toISOString(),
35
+ finishedAt: new Date().toISOString(),
36
+ suitePath: '/test',
37
+ total: 2,
38
+ passed: 1,
39
+ failed: 1,
40
+ results: [
41
+ { fixtureId: 'f1', ok: false, latencyMs: 100 },
42
+ { fixtureId: 'f2', ok: true, latencyMs: 100 },
43
+ ],
44
+ };
45
+ const run2 = {
46
+ id: 'run_2',
47
+ startedAt: new Date().toISOString(),
48
+ finishedAt: new Date().toISOString(),
49
+ suitePath: '/test',
50
+ total: 2,
51
+ passed: 2,
52
+ failed: 0,
53
+ results: [
54
+ { fixtureId: 'f1', ok: true, latencyMs: 100 },
55
+ { fixtureId: 'f2', ok: true, latencyMs: 100 },
56
+ ],
57
+ };
58
+
59
+ await EVAL_STORE.saveRun(run1);
60
+ await EVAL_STORE.saveRun(run2);
61
+
62
+ const diff = await EVAL_STORE.compareRuns('run_1', 'run_2');
63
+
64
+ assert.strictEqual(diff.improved.length, 1);
65
+ assert.strictEqual(diff.improved[0].fixtureId, 'f1');
66
+ assert.strictEqual(diff.regressed.length, 0);
67
+ assert.strictEqual(diff.unchanged.length, 1);
68
+ assert.strictEqual(diff.unchanged[0].fixtureId, 'f2');
69
+ });
70
+
71
+ it('categorizes regressed fixtures (was passing, now failing)', async () => {
72
+ const run1 = {
73
+ id: 'run_a',
74
+ startedAt: new Date().toISOString(),
75
+ finishedAt: new Date().toISOString(),
76
+ suitePath: '/test',
77
+ total: 2,
78
+ passed: 2,
79
+ failed: 0,
80
+ results: [
81
+ { fixtureId: 'f1', ok: true, latencyMs: 100 },
82
+ { fixtureId: 'f2', ok: true, latencyMs: 100 },
83
+ ],
84
+ };
85
+ const run2 = {
86
+ id: 'run_b',
87
+ startedAt: new Date().toISOString(),
88
+ finishedAt: new Date().toISOString(),
89
+ suitePath: '/test',
90
+ total: 2,
91
+ passed: 1,
92
+ failed: 1,
93
+ results: [
94
+ { fixtureId: 'f1', ok: false, latencyMs: 100 },
95
+ { fixtureId: 'f2', ok: true, latencyMs: 100 },
96
+ ],
97
+ };
98
+
99
+ await EVAL_STORE.saveRun(run1);
100
+ await EVAL_STORE.saveRun(run2);
101
+
102
+ const diff = await EVAL_STORE.compareRuns('run_a', 'run_b');
103
+
104
+ assert.strictEqual(diff.regressed.length, 1);
105
+ assert.strictEqual(diff.regressed[0].fixtureId, 'f1');
106
+ assert.strictEqual(diff.improved.length, 0);
107
+ assert.strictEqual(diff.unchanged.length, 1);
108
+ });
109
+
110
+ it('categorizes unchanged fixtures', async () => {
111
+ const run1 = {
112
+ id: 'run_x',
113
+ startedAt: new Date().toISOString(),
114
+ finishedAt: new Date().toISOString(),
115
+ suitePath: '/test',
116
+ total: 1,
117
+ passed: 1,
118
+ failed: 0,
119
+ results: [{ fixtureId: 'f1', ok: true, latencyMs: 100 }],
120
+ };
121
+ const run2 = {
122
+ id: 'run_y',
123
+ startedAt: new Date().toISOString(),
124
+ finishedAt: new Date().toISOString(),
125
+ suitePath: '/test',
126
+ total: 1,
127
+ passed: 1,
128
+ failed: 0,
129
+ results: [{ fixtureId: 'f1', ok: true, latencyMs: 120 }],
130
+ };
131
+
132
+ await EVAL_STORE.saveRun(run1);
133
+ await EVAL_STORE.saveRun(run2);
134
+
135
+ const diff = await EVAL_STORE.compareRuns('run_x', 'run_y');
136
+
137
+ assert.strictEqual(diff.unchanged.length, 1);
138
+ assert.strictEqual(diff.improved.length, 0);
139
+ assert.strictEqual(diff.regressed.length, 0);
140
+ });
141
+
142
+ it('returns error when run not found', async () => {
143
+ const diff = await EVAL_STORE.compareRuns('nonexistent', 'also-nonexistent');
144
+ assert.ok(diff.error);
145
+ assert.strictEqual(diff.error, 'run not found');
146
+ });
147
+
148
+ it('handles fixtures that only exist in one run', async () => {
149
+ const run1 = {
150
+ id: 'run_new',
151
+ startedAt: new Date().toISOString(),
152
+ finishedAt: new Date().toISOString(),
153
+ suitePath: '/test',
154
+ total: 2,
155
+ passed: 2,
156
+ failed: 0,
157
+ results: [
158
+ { fixtureId: 'f1', ok: true, latencyMs: 100 },
159
+ { fixtureId: 'f2', ok: true, latencyMs: 100 },
160
+ ],
161
+ };
162
+ const run2 = {
163
+ id: 'run_old',
164
+ startedAt: new Date().toISOString(),
165
+ finishedAt: new Date().toISOString(),
166
+ suitePath: '/test',
167
+ total: 1,
168
+ passed: 1,
169
+ failed: 0,
170
+ results: [{ fixtureId: 'f1', ok: true, latencyMs: 100 }],
171
+ };
172
+
173
+ await EVAL_STORE.saveRun(run1);
174
+ await EVAL_STORE.saveRun(run2);
175
+
176
+ const diff = await EVAL_STORE.compareRuns('run_new', 'run_old');
177
+ // f2 only exists in run_new, so it's not in the diff
178
+ assert.strictEqual(diff.unchanged.length, 1);
179
+ });
180
+ });
181
+
182
+ describe('saveRun / getRun', () => {
183
+ it('saves and retrieves a run', async () => {
184
+ const run = {
185
+ id: 'run_save_test',
186
+ startedAt: new Date().toISOString(),
187
+ finishedAt: new Date().toISOString(),
188
+ suitePath: '/test',
189
+ total: 1,
190
+ passed: 1,
191
+ failed: 0,
192
+ results: [{ fixtureId: 'f1', ok: true, latencyMs: 100 }],
193
+ };
194
+
195
+ const saved = await EVAL_STORE.saveRun(run);
196
+ assert.strictEqual(saved.ok, true);
197
+ assert.ok(saved.path.includes('run_save_test'));
198
+
199
+ const retrieved = await EVAL_STORE.getRun('run_save_test');
200
+ assert.strictEqual(retrieved.id, 'run_save_test');
201
+ assert.strictEqual(retrieved.total, 1);
202
+ assert.strictEqual(retrieved.passed, 1);
203
+ });
204
+
205
+ it('returns null for nonexistent run', async () => {
206
+ const result = await EVAL_STORE.getRun('nonexistent_run_id');
207
+ assert.strictEqual(result, null);
208
+ });
209
+ });
210
+
211
+ describe('listRuns', () => {
212
+ it('returns runs in newest-first order', async () => {
213
+ for (let i = 0; i < 5; i++) {
214
+ const run = {
215
+ id: `run_list_${i}`,
216
+ startedAt: new Date().toISOString(),
217
+ finishedAt: new Date().toISOString(),
218
+ suitePath: '/test',
219
+ total: 1,
220
+ passed: 1,
221
+ failed: 0,
222
+ results: [],
223
+ };
224
+ await EVAL_STORE.saveRun(run);
225
+ }
226
+
227
+ const runs = await EVAL_STORE.listRuns({ limit: 3 });
228
+ assert.strictEqual(runs.length, 3);
229
+ // Should be newest first
230
+ assert.strictEqual(runs[0].id, 'run_list_4');
231
+ assert.strictEqual(runs[1].id, 'run_list_3');
232
+ assert.strictEqual(runs[2].id, 'run_list_2');
233
+ });
234
+
235
+ it('respects limit parameter', async () => {
236
+ // Save a few runs first
237
+ for (let i = 0; i < 4; i++) {
238
+ const run = {
239
+ id: `run_limit_${i}`,
240
+ startedAt: new Date().toISOString(),
241
+ finishedAt: new Date().toISOString(),
242
+ suitePath: '/test',
243
+ total: 1,
244
+ passed: 1,
245
+ failed: 0,
246
+ results: [],
247
+ };
248
+ await EVAL_STORE.saveRun(run);
249
+ }
250
+ const runs = await EVAL_STORE.listRuns({ limit: 2 });
251
+ assert.strictEqual(runs.length, 2);
252
+ });
253
+ });
254
+
255
+ describe('deleteRun', () => {
256
+ it('deletes a run', async () => {
257
+ const run = {
258
+ id: 'run_to_delete',
259
+ startedAt: new Date().toISOString(),
260
+ finishedAt: new Date().toISOString(),
261
+ suitePath: '/test',
262
+ total: 1,
263
+ passed: 1,
264
+ failed: 0,
265
+ results: [],
266
+ };
267
+ await EVAL_STORE.saveRun(run);
268
+ const result = await EVAL_STORE.deleteRun('run_to_delete');
269
+ assert.strictEqual(result.ok, true);
270
+ const retrieved = await EVAL_STORE.getRun('run_to_delete');
271
+ assert.strictEqual(retrieved, null);
272
+ });
273
+ });
274
+
275
+ describe('buildRunId', () => {
276
+ it('generates unique IDs', () => {
277
+ const id1 = EVAL_STORE.buildRunId();
278
+ const id2 = EVAL_STORE.buildRunId();
279
+ assert.notStrictEqual(id1, id2);
280
+ assert.ok(id1.startsWith('run_'));
281
+ assert.ok(id2.startsWith('run_'));
282
+ });
283
+ });
284
+ });