@polderlabs/bizar 4.9.0 → 5.0.0

package/bizar-dash/tests/deploy-templates.test.mjs

@@ -0,0 +1,100 @@
+/**
+ * bizar-dash/tests/deploy-templates.test.mjs
+ *
+ * Verify that deploy templates exist and are valid.
+ */
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+import { readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url));
+const PROJECT_ROOT = join(__dirname, '..', '..');
+const TEMPLATES_DIR = join(PROJECT_ROOT, 'templates', 'deploy');
+
+// Lazy-import toml/yaml only if needed (they're devDependencies)
+let toml;
+let yaml;
+
+function getToml() {
+  if (!toml) {
+    toml = import('smol-toml').then((m) => m.parse || m.default?.parse || m.default);
+  }
+  // synchronous wrapper
+  return { parse: (s) => { throw new Error('use async version'); } };
+}
+
+async function parseToml(content) {
+  const mod = await import('smol-toml');
+  return (mod.parse || mod.default?.parse || mod.default)(content);
+}
+
+async function parseYaml(content) {
+  const mod = await import('js-yaml');
+  return (mod.load || mod.default?.load || mod.default)(content);
+}
+
+describe('Deploy templates', () => {
+  const platforms = ['vercel', 'cloudflare', 'fly', 'docker'];
+  const requiredFiles = {
+    vercel: ['vercel.json.template', 'api-index.template.js', 'README.md'],
+    cloudflare: ['wrangler.toml.template', 'functions-index.template.js', 'README.md'],
+    fly: ['fly.toml.template', 'README.md'],
+    docker: ['docker-compose.template.yml', '.env.template', 'README.md'],
+  };
+
+  for (const platform of platforms) {
+    it(platform + ' template directory exists', () => {
+      const dir = join(TEMPLATES_DIR, platform);
+      assert.ok(existsSync(dir), 'Directory missing: templates/deploy/' + platform);
+    });
+
+    it(platform + ' has at least 2 files', () => {
+      const dir = join(TEMPLATES_DIR, platform);
+      const entries = readdirSync(dir).filter((f) => {
+        return statSync(join(dir, f)).isFile();
+      });
+      assert.ok(entries.length >= 2, platform + ' has ' + entries.length + ' files (need >=2): ' + entries.join(', '));
+    });
+
+    for (const file of (requiredFiles[platform] || [])) {
+      it(platform + '/' + file + ' exists', () => {
+        const fpath = join(TEMPLATES_DIR, platform, file);
+        assert.ok(existsSync(fpath), 'Missing file: templates/deploy/' + platform + '/' + file);
+      });
+    }
+  }
+
+  it('vercel.json.template is valid JSON', () => {
+    const content = readFileSync(join(TEMPLATES_DIR, 'vercel', 'vercel.json.template'), 'utf8');
+    const parsed = JSON.parse(content);
+    assert.ok(parsed.name);
+    assert.strictEqual(parsed.version, 2);
+    assert.ok(Array.isArray(parsed.routes));
+  });
+
+  it('wrangler.toml.template is valid TOML', { timeout: 10000 }, async () => {
+    const content = readFileSync(join(TEMPLATES_DIR, 'cloudflare', 'wrangler.toml.template'), 'utf8');
+    const parsed = await parseToml(content);
+    assert.ok(parsed);
+    assert.ok(parsed.name);
+    assert.ok(parsed.site);
+  });
+
+  it('fly.toml.template is valid TOML', { timeout: 10000 }, async () => {
+    const content = readFileSync(join(TEMPLATES_DIR, 'fly', 'fly.toml.template'), 'utf8');
+    const parsed = await parseToml(content);
+    assert.ok(parsed);
+    assert.ok(parsed.app);
+    assert.ok(parsed.http_service);
+  });
+
+  it('docker-compose.template.yml is valid YAML', { timeout: 10000 }, async () => {
+    const content = readFileSync(join(TEMPLATES_DIR, 'docker', 'docker-compose.template.yml'), 'utf8');
+    const parsed = await parseYaml(content);
+    assert.ok(parsed);
+    assert.ok(parsed.services);
+    assert.ok(parsed.services['bizar-dash']);
+  });
+});

package/bizar-dash/tests/eval/fixtures.test.mjs

@@ -0,0 +1,141 @@
+/**
+ * tests/eval/fixtures.test.mjs
+ *
+ * v5.0.0 — Tests for fixture loading.
+ *
+ * Verifies:
+ *   - Example fixtures load without errors
+ *   - Fixture JSON is valid and has required fields
+ *   - loadFixtures correctly loads from a directory
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const EVAL = await import('../../src/server/eval.mjs');
+
+// Fixture files to validate
+const EXAMPLE_FIXTURES = [
+  '../../../templates/eval-fixtures/code-search-basic.json',
+  '../../../templates/eval-fixtures/tool-call-correctness.json',
+  '../../../templates/eval-fixtures/response-format.json',
+  '../../../templates/eval-fixtures/latency-bounds.json',
+  '../../../templates/eval-fixtures/regression-suite.json',
+];
+
+const REQUIRED_FIELDS = ['id', 'name', 'description', 'agent', 'prompt', 'expected'];
+const REQUIRED_EXPECTED = ['contains', 'notContains', 'regex', 'jsonSchema', 'maxTokens', 'maxLatencyMs'];
+
+describe('eval fixtures', () => {
+
+  describe('example fixtures from templates/', () => {
+
+    for (const fixturePath of EXAMPLE_FIXTURES) {
+      const fullPath = join(__dirname, fixturePath);
+      it(`loads and validates: ${fixturePath}`, async () => {
+        // Dynamically import so we can check it parses
+        const { readFileSync } = await import('node:fs');
+        const raw = readFileSync(fullPath, 'utf8');
+        const fixture = JSON.parse(raw);
+
+        // Should have id
+        assert.ok(fixture.id, 'fixture must have an id');
+
+        // If it's a suite with fixtures array, validate each
+        if (fixture.fixtures && Array.isArray(fixture.fixtures)) {
+          assert.ok(fixture.fixtures.length > 0, 'suite must have at least one fixture');
+          for (const f of fixture.fixtures) {
+            validateFixture(f);
+          }
+        } else {
+          // Single fixture
+          validateFixture(fixture);
+        }
+      });
+    }
+  });
+
+  describe('loadFixtures', () => {
+    it('loads fixtures from a directory', async () => {
+      const fixturesDir = join(__dirname, '../../../templates/eval-fixtures');
+      const fixtures = EVAL.loadFixtures(fixturesDir);
+      assert.ok(fixtures.length >= 4, `Expected at least 4 fixtures, got ${fixtures.length}`);
+    });
+
+    it('skips non-JSON files', async () => {
+      const { mkdirSync, writeFileSync, rmSync } = await import('node:fs');
+      const { tmpdir } = await import('node:os');
+      const testDir = join(tmpdir(), `eval-load-test-${Date.now()}`);
+      mkdirSync(testDir, { recursive: true });
+      writeFileSync(join(testDir, 'readme.md'), '# readme');
+      writeFileSync(join(testDir, 'valid.json'), JSON.stringify({
+        id: 'test',
+        name: 'Test',
+        description: 'Test',
+        agent: 'thor',
+        prompt: 'test',
+        expected: { contains: [] },
+      }));
+      const fixtures = EVAL.loadFixtures(testDir);
+      rmSync(testDir, { recursive: true, force: true });
+      assert.strictEqual(fixtures.length, 1);
+      assert.strictEqual(fixtures[0].id, 'test');
+    });
+
+    it('skips invalid JSON files', async () => {
+      const { mkdirSync, writeFileSync, rmSync } = await import('node:fs');
+      const { tmpdir } = await import('node:os');
+      const testDir = join(tmpdir(), `eval-invalid-test-${Date.now()}`);
+      mkdirSync(testDir, { recursive: true });
+      writeFileSync(join(testDir, 'invalid.json'), 'not valid json {');
+      writeFileSync(join(testDir, 'valid.json'), JSON.stringify({
+        id: 'test2',
+        name: 'Test2',
+        description: 'Test2',
+        agent: 'thor',
+        prompt: 'test',
+        expected: { contains: [] },
+      }));
+      const fixtures = EVAL.loadFixtures(testDir);
+      rmSync(testDir, { recursive: true, force: true });
+      assert.strictEqual(fixtures.length, 1);
+      assert.strictEqual(fixtures[0].id, 'test2');
+    });
+
+    it('returns empty array for non-existent directory', async () => {
+      const fixtures = EVAL.loadFixtures('/nonexistent/path/xyz');
+      assert.strictEqual(fixtures.length, 0);
+    });
+  });
+});
+
+/**
+ * Validate a fixture object has required fields
+ * @param {object} fixture
+ */
+function validateFixture(fixture) {
+  for (const field of REQUIRED_FIELDS) {
+    assert.ok(
+      fixture[field] !== undefined && fixture[field] !== null,
+      `fixture ${fixture.id || 'unknown'} missing required field: ${field}`,
+    );
+  }
+  assert.ok(
+    typeof fixture.expected === 'object',
+    `fixture ${fixture.id} expected must be an object`,
+  );
+  // expected should have at least one check type
+  const expected = fixture.expected;
+  const hasCheckType = [
+    expected.contains,
+    expected.notContains,
+    expected.regex,
+    expected.jsonSchema,
+    expected.maxTokens,
+    expected.maxLatencyMs,
+  ].some((v) => v !== undefined && v !== null);
+  assert.ok(hasCheckType, `fixture ${fixture.id} expected must have at least one check type`);
+}

package/bizar-dash/tests/eval/report.test.mjs

@@ -0,0 +1,284 @@
+/**
+ * tests/eval/report.test.mjs
+ *
+ * v5.0.0 — Tests for eval run comparison / diff reporting.
+ *
+ * Verifies:
+ *   - compareRuns correctly categorizes fixtures as improved/regressed/unchanged
+ *   - saveRun and getRun work correctly
+ *   - listRuns returns runs in order
+ */
+
+import { describe, it, beforeEach, afterEach } from 'node:test';
+import assert from 'node:assert';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { mkdirSync, rmSync } from 'node:fs';
+
+const storeHome = join(tmpdir(), `bizar-eval-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+process.env.BIZAR_STORE_HOME = storeHome;
+mkdirSync(storeHome, { recursive: true });
+
+const EVAL_STORE = await import('../../src/server/eval-store.mjs');
+
+describe('eval store — run comparison', () => {
+
+  afterEach(() => {
+    EVAL_STORE.__resetStoreForTests();
+  });
+
+  describe('compareRuns', () => {
+    it('categorizes improved fixtures (was failing, now passing)', async () => {
+      const run1 = {
+        id: 'run_1',
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        suitePath: '/test',
+        total: 2,
+        passed: 1,
+        failed: 1,
+        results: [
+          { fixtureId: 'f1', ok: false, latencyMs: 100 },
+          { fixtureId: 'f2', ok: true, latencyMs: 100 },
+        ],
+      };
+      const run2 = {
+        id: 'run_2',
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        suitePath: '/test',
+        total: 2,
+        passed: 2,
+        failed: 0,
+        results: [
+          { fixtureId: 'f1', ok: true, latencyMs: 100 },
+          { fixtureId: 'f2', ok: true, latencyMs: 100 },
+        ],
+      };
+
+      await EVAL_STORE.saveRun(run1);
+      await EVAL_STORE.saveRun(run2);
+
+      const diff = await EVAL_STORE.compareRuns('run_1', 'run_2');
+
+      assert.strictEqual(diff.improved.length, 1);
+      assert.strictEqual(diff.improved[0].fixtureId, 'f1');
+      assert.strictEqual(diff.regressed.length, 0);
+      assert.strictEqual(diff.unchanged.length, 1);
+      assert.strictEqual(diff.unchanged[0].fixtureId, 'f2');
+    });
+
+    it('categorizes regressed fixtures (was passing, now failing)', async () => {
+      const run1 = {
+        id: 'run_a',
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        suitePath: '/test',
+        total: 2,
+        passed: 2,
+        failed: 0,
+        results: [
+          { fixtureId: 'f1', ok: true, latencyMs: 100 },
+          { fixtureId: 'f2', ok: true, latencyMs: 100 },
+        ],
+      };
+      const run2 = {
+        id: 'run_b',
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        suitePath: '/test',
+        total: 2,
+        passed: 1,
+        failed: 1,
+        results: [
+          { fixtureId: 'f1', ok: false, latencyMs: 100 },
+          { fixtureId: 'f2', ok: true, latencyMs: 100 },
+        ],
+      };
+
+      await EVAL_STORE.saveRun(run1);
+      await EVAL_STORE.saveRun(run2);
+
+      const diff = await EVAL_STORE.compareRuns('run_a', 'run_b');
+
+      assert.strictEqual(diff.regressed.length, 1);
+      assert.strictEqual(diff.regressed[0].fixtureId, 'f1');
+      assert.strictEqual(diff.improved.length, 0);
+      assert.strictEqual(diff.unchanged.length, 1);
+    });
+
+    it('categorizes unchanged fixtures', async () => {
+      const run1 = {
+        id: 'run_x',
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        suitePath: '/test',
+        total: 1,
+        passed: 1,
+        failed: 0,
+        results: [{ fixtureId: 'f1', ok: true, latencyMs: 100 }],
+      };
+      const run2 = {
+        id: 'run_y',
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        suitePath: '/test',
+        total: 1,
+        passed: 1,
+        failed: 0,
+        results: [{ fixtureId: 'f1', ok: true, latencyMs: 120 }],
+      };
+
+      await EVAL_STORE.saveRun(run1);
+      await EVAL_STORE.saveRun(run2);
+
+      const diff = await EVAL_STORE.compareRuns('run_x', 'run_y');
+
+      assert.strictEqual(diff.unchanged.length, 1);
+      assert.strictEqual(diff.improved.length, 0);
+      assert.strictEqual(diff.regressed.length, 0);
+    });
+
+    it('returns error when run not found', async () => {
+      const diff = await EVAL_STORE.compareRuns('nonexistent', 'also-nonexistent');
+      assert.ok(diff.error);
+      assert.strictEqual(diff.error, 'run not found');
+    });
+
+    it('handles fixtures that only exist in one run', async () => {
+      const run1 = {
+        id: 'run_new',
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        suitePath: '/test',
+        total: 2,
+        passed: 2,
+        failed: 0,
+        results: [
+          { fixtureId: 'f1', ok: true, latencyMs: 100 },
+          { fixtureId: 'f2', ok: true, latencyMs: 100 },
+        ],
+      };
+      const run2 = {
+        id: 'run_old',
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        suitePath: '/test',
+        total: 1,
+        passed: 1,
+        failed: 0,
+        results: [{ fixtureId: 'f1', ok: true, latencyMs: 100 }],
+      };
+
+      await EVAL_STORE.saveRun(run1);
+      await EVAL_STORE.saveRun(run2);
+
+      const diff = await EVAL_STORE.compareRuns('run_new', 'run_old');
+      // f2 only exists in run_new, so it's not in the diff
+      assert.strictEqual(diff.unchanged.length, 1);
+    });
+  });
+
+  describe('saveRun / getRun', () => {
+    it('saves and retrieves a run', async () => {
+      const run = {
+        id: 'run_save_test',
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        suitePath: '/test',
+        total: 1,
+        passed: 1,
+        failed: 0,
+        results: [{ fixtureId: 'f1', ok: true, latencyMs: 100 }],
+      };
+
+      const saved = await EVAL_STORE.saveRun(run);
+      assert.strictEqual(saved.ok, true);
+      assert.ok(saved.path.includes('run_save_test'));
+
+      const retrieved = await EVAL_STORE.getRun('run_save_test');
+      assert.strictEqual(retrieved.id, 'run_save_test');
+      assert.strictEqual(retrieved.total, 1);
+      assert.strictEqual(retrieved.passed, 1);
+    });
+
+    it('returns null for nonexistent run', async () => {
+      const result = await EVAL_STORE.getRun('nonexistent_run_id');
+      assert.strictEqual(result, null);
+    });
+  });
+
+  describe('listRuns', () => {
+    it('returns runs in newest-first order', async () => {
+      for (let i = 0; i < 5; i++) {
+        const run = {
+          id: `run_list_${i}`,
+          startedAt: new Date().toISOString(),
+          finishedAt: new Date().toISOString(),
+          suitePath: '/test',
+          total: 1,
+          passed: 1,
+          failed: 0,
+          results: [],
+        };
+        await EVAL_STORE.saveRun(run);
+      }
+
+      const runs = await EVAL_STORE.listRuns({ limit: 3 });
+      assert.strictEqual(runs.length, 3);
+      // Should be newest first
+      assert.strictEqual(runs[0].id, 'run_list_4');
+      assert.strictEqual(runs[1].id, 'run_list_3');
+      assert.strictEqual(runs[2].id, 'run_list_2');
+    });
+
+    it('respects limit parameter', async () => {
+      // Save a few runs first
+      for (let i = 0; i < 4; i++) {
+        const run = {
+          id: `run_limit_${i}`,
+          startedAt: new Date().toISOString(),
+          finishedAt: new Date().toISOString(),
+          suitePath: '/test',
+          total: 1,
+          passed: 1,
+          failed: 0,
+          results: [],
+        };
+        await EVAL_STORE.saveRun(run);
+      }
+      const runs = await EVAL_STORE.listRuns({ limit: 2 });
+      assert.strictEqual(runs.length, 2);
+    });
+  });
+
+  describe('deleteRun', () => {
+    it('deletes a run', async () => {
+      const run = {
+        id: 'run_to_delete',
+        startedAt: new Date().toISOString(),
+        finishedAt: new Date().toISOString(),
+        suitePath: '/test',
+        total: 1,
+        passed: 1,
+        failed: 0,
+        results: [],
+      };
+      await EVAL_STORE.saveRun(run);
+      const result = await EVAL_STORE.deleteRun('run_to_delete');
+      assert.strictEqual(result.ok, true);
+      const retrieved = await EVAL_STORE.getRun('run_to_delete');
+      assert.strictEqual(retrieved, null);
+    });
+  });
+
+  describe('buildRunId', () => {
+    it('generates unique IDs', () => {
+      const id1 = EVAL_STORE.buildRunId();
+      const id2 = EVAL_STORE.buildRunId();
+      assert.notStrictEqual(id1, id2);
+      assert.ok(id1.startsWith('run_'));
+      assert.ok(id2.startsWith('run_'));
+    });
+  });
+});