@polderlabs/bizar 5.0.2 → 5.2.0

package/bizar-dash/tests/components/plugin-permissions.test.tsx

@@ -0,0 +1,95 @@
+// tests/components/plugin-permissions.test.tsx — PluginPermissions and PluginCard unit tests.
+
+import { render, screen } from '@testing-library/react';
+import { describe, it, expect, vi } from 'vitest';
+import { PluginPermissions } from '../../src/web/components/PluginPermissions';
+import { PluginCard } from '../../src/web/components/PluginCard';
+
+describe('PluginPermissions', () => {
+  it('renders each permission as a chip', () => {
+    render(<PluginPermissions permissions={['net', 'fs']} />);
+    expect(screen.getByText('Network access')).toBeInTheDocument();
+    expect(screen.getByText('Filesystem')).toBeInTheDocument();
+  });
+
+  it('shows unknown permission with default label', () => {
+    render(<PluginPermissions permissions={['unknown_perm']} />);
+    expect(screen.getByText('unknown_perm')).toBeInTheDocument();
+  });
+
+  it('handles empty permissions array', () => {
+    const { container } = render(<PluginPermissions permissions={[]} />);
+    expect(container.querySelector('.plugin-permissions')).toBeInTheDocument();
+    expect(container.querySelectorAll('.permission-chip')).toHaveLength(0);
+  });
+});
+
+describe('PluginCard', () => {
+  const defaultPlugin = {
+    id: 'test-plugin',
+    name: 'Test Plugin',
+    version: '1.0.0',
+    description: 'A test plugin',
+    permissions: ['net', 'fs'],
+    invocations: 42,
+    lastInvokedAt: '2026-07-05T10:00:00Z',
+    enabled: true,
+  };
+
+  it('renders name, version, description', () => {
+    render(
+      <PluginCard
+        plugin={defaultPlugin}
+        onToggle={vi.fn()}
+        onUninstall={vi.fn()}
+        onConfigure={vi.fn()}
+      />,
+    );
+    expect(screen.getByText('Test Plugin')).toBeInTheDocument();
+    expect(screen.getByText('v1.0.0 · test-plugin')).toBeInTheDocument();
+    expect(screen.getByText('A test plugin')).toBeInTheDocument();
+  });
+
+  it('calls onToggle when toggle clicked', async () => {
+    const onToggle = vi.fn();
+    render(
+      <PluginCard
+        plugin={defaultPlugin}
+        onToggle={onToggle}
+        onUninstall={vi.fn()}
+        onConfigure={vi.fn()}
+      />,
+    );
+    const toggle = screen.getByRole('switch');
+    toggle.click();
+    expect(onToggle).toHaveBeenCalledWith('test-plugin');
+  });
+
+  it('calls onUninstall when uninstall clicked', () => {
+    const onUninstall = vi.fn();
+    render(
+      <PluginCard
+        plugin={defaultPlugin}
+        onToggle={vi.fn()}
+        onUninstall={onUninstall}
+        onConfigure={vi.fn()}
+      />,
+    );
+    screen.getByText('Uninstall').click();
+    expect(onUninstall).toHaveBeenCalledWith('test-plugin');
+  });
+
+  it('calls onConfigure when configure clicked', () => {
+    const onConfigure = vi.fn();
+    render(
+      <PluginCard
+        plugin={defaultPlugin}
+        onToggle={vi.fn()}
+        onUninstall={vi.fn()}
+        onConfigure={onConfigure}
+      />,
+    );
+    screen.getByText('Configure').click();
+    expect(onConfigure).toHaveBeenCalledWith('test-plugin');
+  });
+});

package/bizar-dash/tests/eval/fixtures-extra.test.mjs

@@ -0,0 +1,285 @@
+/**
+ * tests/eval/fixtures-extra.test.mjs
+ *
+ * v5.0.0 — Tests for expanded fixture set (10 new fixtures).
+ *
+ * Verifies:
+ *   - All 15 fixtures load without errors
+ *   - All fixtures have valid JSON and required fields
+ *   - runFixtureById correctly resolves and runs fixtures
+ *   - New expectation types (toolCallsMin, toolSequence, minItems) work
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const EVAL = await import('../../src/server/eval.mjs');
+
+const FIXTURES_DIR = join(__dirname, '../../../templates/eval-fixtures');
+
+const REQUIRED_FIELDS = ['id', 'name', 'description', 'agent', 'prompt', 'expected', 'tags'];
+const REQUIRED_EXPECTED_FIELDS = ['contains', 'notContains', 'regex', 'jsonSchema', 'maxTokens', 'maxLatencyMs'];
+
+// Expected fixture IDs - the new 10 fixtures
+const NEW_FIXTURE_IDS = [
+  'tool-call-multi-step',
+  'error-recovery',
+  'context-window',
+  'json-output',
+  'code-review',
+  'unicode-handling',
+  'multi-language',
+  'safe-paths',
+  'concise-output',
+  'citation',
+];
+
+describe('eval fixtures (extra 10)', () => {
+
+  describe('loadFixtures loads all 15 fixtures', () => {
+    it('loads at least 15 fixtures from the directory', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      assert.ok(fixtures.length >= 15, `Expected at least 15 fixtures, got ${fixtures.length}`);
+    });
+
+    it('loads all expected fixture IDs', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const ids = fixtures.map((f) => f.id);
+      for (const expectedId of NEW_FIXTURE_IDS) {
+        assert.ok(
+          ids.includes(expectedId),
+          `Expected fixture ID "${expectedId}" not found in: ${ids.join(', ')}`,
+        );
+      }
+    });
+  });
+
+  describe('each new fixture validates correctly', () => {
+    const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+    const newFixtures = fixtures.filter((f) =>
+      NEW_FIXTURE_IDS.includes(f.id)
+    );
+
+    for (const fixture of newFixtures) {
+      it(`fixture "${fixture.id}" has valid JSON structure`, () => {
+        // Required fields
+        for (const field of REQUIRED_FIELDS) {
+          assert.ok(
+            fixture[field] !== undefined && fixture[field] !== null,
+            `fixture "${fixture.id}" missing required field: ${field}`,
+          );
+        }
+
+        // expected must be an object
+        assert.ok(
+          typeof fixture.expected === 'object',
+          `fixture "${fixture.id}".expected must be an object`,
+        );
+
+        // tags must be an array
+        assert.ok(
+          Array.isArray(fixture.tags),
+          `fixture "${fixture.id}".tags must be an array`,
+        );
+      });
+
+      it(`fixture "${fixture.id}" has at least one check type`, () => {
+        const expected = fixture.expected;
+        const hasCheckType = [
+          expected.contains,
+          expected.notContains,
+          expected.regex,
+          expected.jsonSchema,
+          expected.maxTokens,
+          expected.maxLatencyMs,
+          expected.toolCallsMin,
+          expected.toolSequence,
+        ].some((v) => v !== undefined && v !== null);
+        assert.ok(hasCheckType, `fixture "${fixture.id}" expected must have at least one check type`);
+      });
+    }
+  });
+
+  describe('runFixtureById', () => {
+    it('resolves and runs a fixture by id', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'ok',
+        usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
+      });
+      const result = await EVAL.runFixtureById('latency-bounds', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      assert.strictEqual(result.fixtureId, 'latency-bounds');
+      assert.ok(result.checks.length > 0, 'should have run checks');
+    });
+
+    it('throws for unknown fixture id', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({ content: 'ok', usage: { totalTokens: 0 } });
+      await assert.rejects(
+        () => EVAL.runFixtureById('nonexistent-fixture', fixtures, { llmCall: mockLlmCall }),
+        /Fixture not found/,
+      );
+    });
+  });
+
+  describe('toolCallsMin check', () => {
+    it('passes when tool calls meet minimum', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'found it',
+        usage: { totalTokens: 100 },
+      });
+      const result = await EVAL.runFixtureById('tool-call-multi-step', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+        toolCalls: ['search', 'read', 'search'],
+      });
+      // The toolCallsMin check should be in the results
+      const toolCallsMinCheck = result.checks.find((c) => c.kind === 'toolCallsMin');
+      assert.ok(toolCallsMinCheck, 'should have a toolCallsMin check');
+      assert.ok(toolCallsMinCheck.ok, `toolCallsMin check should pass: ${toolCallsMinCheck.message}`);
+    });
+
+    it('fails when tool calls are below minimum', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'found it',
+        usage: { totalTokens: 100 },
+      });
+      const result = await EVAL.runFixtureById('tool-call-multi-step', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+        toolCalls: ['search'], // only 1, requires 3
+      });
+      const toolCallsMinCheck = result.checks.find((c) => c.kind === 'toolCallsMin');
+      assert.ok(toolCallsMinCheck, 'should have a toolCallsMin check');
+      assert.ok(!toolCallsMinCheck.ok, 'toolCallsMin check should fail');
+    });
+  });
+
+  describe('toolSequence check', () => {
+    it('passes when tool sequence matches', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'found it',
+        usage: { totalTokens: 100 },
+      });
+      const result = await EVAL.runFixtureById('tool-call-multi-step', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+        toolCalls: ['search', 'read', 'search'],
+      });
+      const seqCheck = result.checks.find((c) => c.kind === 'toolSequence');
+      assert.ok(seqCheck, 'should have a toolSequence check');
+      assert.ok(seqCheck.ok, `toolSequence check should pass: ${seqCheck.message}`);
+    });
+
+    it('fails when tool sequence does not match', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'found it',
+        usage: { totalTokens: 100 },
+      });
+      const result = await EVAL.runFixtureById('tool-call-multi-step', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+        toolCalls: ['read', 'search', 'search'], // wrong order
+      });
+      const seqCheck = result.checks.find((c) => c.kind === 'toolSequence');
+      assert.ok(seqCheck, 'should have a toolSequence check');
+      assert.ok(!seqCheck.ok, 'toolSequence check should fail');
+    });
+  });
+
+  describe('jsonSchema minItems check', () => {
+    it('passes when array has enough items', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: '["red", "green", "blue"]',
+        usage: { totalTokens: 50 },
+      });
+      const result = await EVAL.runFixtureById('json-output', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      const jsonCheck = result.checks.find((c) => c.kind === 'jsonSchema');
+      assert.ok(jsonCheck, 'should have a jsonSchema check');
+      assert.ok(jsonCheck.ok, `jsonSchema check should pass: ${jsonCheck.message}`);
+    });
+
+    it('fails when array has too few items', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: '["red"]',
+        usage: { totalTokens: 50 },
+      });
+      const result = await EVAL.runFixtureById('json-output', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      const jsonCheck = result.checks.find((c) => c.kind === 'jsonSchema');
+      assert.ok(jsonCheck, 'should have a jsonSchema check');
+      assert.ok(!jsonCheck.ok, 'jsonSchema check should fail for short array');
+    });
+  });
+
+  describe('notContains check', () => {
+    it('passes when forbidden content is absent', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'The file was not found',
+        usage: { totalTokens: 50 },
+      });
+      const result = await EVAL.runFixtureById('error-recovery', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      // Should not contain crash or panic
+      const notContainsChecks = result.checks.filter((c) => c.kind === 'notContains');
+      assert.ok(notContainsChecks.length > 0, 'should have notContains checks');
+      for (const check of notContainsChecks) {
+        assert.ok(check.ok, `notContains "${check.message}" should pass`);
+      }
+    });
+
+    it('fails when forbidden content is present', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'The system crashed and panicked',
+        usage: { totalTokens: 50 },
+      });
+      const result = await EVAL.runFixtureById('error-recovery', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      const notContainsChecks = result.checks.filter((c) => c.kind === 'notContains');
+      assert.ok(notContainsChecks.length > 0, 'should have notContains checks');
+      assert.ok(!result.ok, 'overall result should fail');
+    });
+  });
+
+  describe('regex check', () => {
+    it('passes when regex matches', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'The function is at src/utils/add.ts:42',
+        usage: { totalTokens: 50 },
+      });
+      const result = await EVAL.runFixtureById('citation', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      const regexChecks = result.checks.filter((c) => c.kind === 'regex');
+      assert.ok(regexChecks.length > 0, 'should have regex checks');
+      for (const check of regexChecks) {
+        assert.ok(check.ok, `regex "${check.message}" should pass`);
+      }
+    });
+  });
+});

package/bizar-dash/tests/eval-web-ui.test.tsx

@@ -0,0 +1,220 @@
+// tests/eval-web-ui.test.tsx — v5.2.0
+//
+// Tests for the eval framework web UI components:
+//   - EvalRunCard: pass/total rendering, status pill (pass/warn/fail)
+//   - EvalDiff: categorize fixtures as improved/regressed/unchanged
+//
+// We render with @testing-library/react in jsdom and assert on the
+// status pill text + class list.
+
+import { render, screen } from '@testing-library/react';
+import { describe, it, expect } from 'vitest';
+import { EvalRunCard, type EvalRunSummary } from '../src/web/components/EvalRunCard';
+import { EvalDiff, computeEvalDiff, type EvalRunWithResults } from '../src/web/components/EvalDiff';
+
+const BASE_RUN: EvalRunSummary = {
+  id: 'run_2026-07-05_a1b2',
+  startedAt: '2026-07-05T10:00:00.000Z',
+  finishedAt: '2026-07-05T10:01:00.000Z',
+  suitePath: '/tmp/fixtures',
+  total: 10,
+  passed: 10,
+  failed: 0,
+};
+
+describe('EvalRunCard', () => {
+  it('renders the run id and suite path', () => {
+    render(<EvalRunCard run={BASE_RUN} />);
+    expect(screen.getByText(BASE_RUN.id)).toBeInTheDocument();
+    // The card meta includes the suite path. The format helper prepends
+    // a formatted timestamp, so we look for the suite path fragment.
+    expect(screen.getByText(/fixtures/)).toBeInTheDocument();
+  });
+
+  it('renders the pass count and total in the status pill', () => {
+    render(<EvalRunCard run={BASE_RUN} />);
+    expect(screen.getByText('10/10 (100.0%)')).toBeInTheDocument();
+  });
+
+  it('uses the "pass" status when failed === 0', () => {
+    const { container } = render(<EvalRunCard run={BASE_RUN} />);
+    const pill = container.querySelector('.eval-status');
+    expect(pill).toBeInTheDocument();
+    expect(pill).toHaveClass('is-pass');
+    expect(pill).not.toHaveClass('is-warn');
+    expect(pill).not.toHaveClass('is-fail');
+  });
+
+  it('uses the "warn" status when pass rate is between 0.8 and 1.0', () => {
+    const run: EvalRunSummary = { ...BASE_RUN, total: 10, passed: 9, failed: 1 };
+    const { container } = render(<EvalRunCard run={run} />);
+    const pill = container.querySelector('.eval-status');
+    expect(pill).toHaveClass('is-warn');
+    expect(screen.getByText('9/10 (90.0%)')).toBeInTheDocument();
+  });
+
+  it('uses the "fail" status when pass rate is at or below 0.8', () => {
+    const run: EvalRunSummary = { ...BASE_RUN, total: 10, passed: 7, failed: 3 };
+    const { container } = render(<EvalRunCard run={run} />);
+    const pill = container.querySelector('.eval-status');
+    expect(pill).toHaveClass('is-fail');
+    expect(screen.getByText('7/10 (70.0%)')).toBeInTheDocument();
+  });
+
+  it('uses the "fail" status when there is any failure at all (10/10a vs 8/10b)', () => {
+    // pass rate exactly 0.8 → still "fail" (the brief says strictly > 0.8)
+    const run: EvalRunSummary = { ...BASE_RUN, total: 10, passed: 8, failed: 2 };
+    const { container } = render(<EvalRunCard run={run} />);
+    expect(container.querySelector('.eval-status')).toHaveClass('is-fail');
+  });
+
+  it('renders 0/0 (0.0%) without crashing when total is zero', () => {
+    const run: EvalRunSummary = { ...BASE_RUN, total: 0, passed: 0, failed: 0 };
+    render(<EvalRunCard run={run} />);
+    expect(screen.getByText('0/0 (0.0%)')).toBeInTheDocument();
+  });
+
+  it('marks the card as selected when selected=true', () => {
+    const { container } = render(<EvalRunCard run={BASE_RUN} selected />);
+    const card = container.querySelector('.eval-run-card');
+    expect(card).toHaveClass('eval-run-card-selected');
+  });
+
+  it('calls onClick when the card is clicked', () => {
+    let clicked = false;
+    const { container } = render(<EvalRunCard run={BASE_RUN} onClick={() => { clicked = true; }} />);
+    const card = container.querySelector('.eval-run-card') as HTMLElement;
+    card.click();
+    expect(clicked).toBe(true);
+  });
+});
+
+describe('EvalDiff — computeEvalDiff', () => {
+  const runA: EvalRunWithResults = {
+    id: 'run_a',
+    startedAt: '2026-07-01T00:00:00Z',
+    suitePath: '/tmp/fixtures',
+    total: 3,
+    passed: 2,
+    failed: 1,
+    results: [
+      { fixtureId: 'f1', ok: false, latencyMs: 100 },
+      { fixtureId: 'f2', ok: true, latencyMs: 120 },
+      { fixtureId: 'f3', ok: true, latencyMs: 140 },
+    ],
+  };
+
+  const runB: EvalRunWithResults = {
+    id: 'run_b',
+    startedAt: '2026-07-05T00:00:00Z',
+    suitePath: '/tmp/fixtures',
+    total: 3,
+    passed: 3,
+    failed: 0,
+    results: [
+      { fixtureId: 'f1', ok: true, latencyMs: 110 },
+      { fixtureId: 'f2', ok: true, latencyMs: 115 },
+      { fixtureId: 'f3', ok: true, latencyMs: 150 },
+    ],
+  };
+
+  const runC: EvalRunWithResults = {
+    ...runB,
+    id: 'run_c',
+    results: [
+      { fixtureId: 'f1', ok: false, latencyMs: 110 },
+      { fixtureId: 'f2', ok: true, latencyMs: 115 },
+      { fixtureId: 'f3', ok: true, latencyMs: 150 },
+    ],
+  };
+
+  it('identifies improvements (was failing, now passing)', () => {
+    const diff = computeEvalDiff(runA, runB);
+    const improved = diff.filter((d) => d.change === 'improved');
+    expect(improved).toHaveLength(1);
+    expect(improved[0].fixtureId).toBe('f1');
+    expect(improved[0].before).toBe(false);
+    expect(improved[0].after).toBe(true);
+  });
+
+  it('identifies regressions (was passing, now failing)', () => {
+    const diff = computeEvalDiff(runB, runC);
+    const regressed = diff.filter((d) => d.change === 'regressed');
+    expect(regressed).toHaveLength(1);
+    expect(regressed[0].fixtureId).toBe('f1');
+    expect(regressed[0].before).toBe(true);
+    expect(regressed[0].after).toBe(false);
+  });
+
+  it('lists unchanged fixtures separately', () => {
+    const diff = computeEvalDiff(runA, runB);
+    const unchanged = diff.filter((d) => d.change === 'same');
+    expect(unchanged.map((d) => d.fixtureId).sort()).toEqual(['f2', 'f3']);
+  });
+
+  it('returns empty arrays for identical runs', () => {
+    const diff = computeEvalDiff(runA, runA);
+    const changed = diff.filter((d) => d.change !== 'same');
+    expect(changed).toHaveLength(0);
+  });
+});
+
+describe('EvalDiff — rendering', () => {
+  const runA: EvalRunWithResults = {
+    id: 'run_a',
+    startedAt: '2026-07-01T00:00:00Z',
+    suitePath: '/tmp/fixtures',
+    total: 3,
+    passed: 2,
+    failed: 1,
+    results: [
+      { fixtureId: 'f1', ok: false, latencyMs: 100 },
+      { fixtureId: 'f2', ok: true, latencyMs: 120 },
+      { fixtureId: 'f3', ok: true, latencyMs: 140 },
+    ],
+  };
+
+  const runB: EvalRunWithResults = {
+    id: 'run_b',
+    startedAt: '2026-07-05T00:00:00Z',
+    suitePath: '/tmp/fixtures',
+    total: 3,
+    passed: 3,
+    failed: 0,
+    results: [
+      { fixtureId: 'f1', ok: true, latencyMs: 110 },
+      { fixtureId: 'f2', ok: true, latencyMs: 115 },
+      { fixtureId: 'f4', ok: true, latencyMs: 200 },
+    ],
+  };
+
+  it('renders the diff container with a heading', () => {
+    render(<EvalDiff runA={runA} runB={runB} />);
+    expect(screen.getByTestId('eval-diff')).toBeInTheDocument();
+    expect(screen.getByRole('heading', { level: 3 })).toHaveTextContent('Regression Analysis');
+  });
+
+  it('renders the improved/regressed/unchanged counters', () => {
+    render(<EvalDiff runA={runA} runB={runB} />);
+    // f1: false → true = improved
+    // f2: true → true = unchanged
+    // f3: true → true (only in A) → removed
+    // f4: true (only in B) → added
+    expect(screen.getByText(/1 improved/)).toBeInTheDocument();
+    expect(screen.getByText(/0 regressed/)).toBeInTheDocument();
+    expect(screen.getByText(/1 unchanged/)).toBeInTheDocument();
+  });
+
+  it('renders improved and regressed rows with their fixture ids', () => {
+    render(<EvalDiff runA={runA} runB={runB} />);
+    const improvedRow = screen.getByText('f1').closest('.eval-diff-row');
+    expect(improvedRow).not.toBeNull();
+    expect(improvedRow).toHaveClass('is-improved');
+  });
+
+  it('shows the empty state when no fixtures changed', () => {
+    // Pass the same run for both A and B — no changes.
+    render(<EvalDiff runA={runA} runB={runA} />);
+    expect(screen.getByText(/No fixture-level changes/)).toBeInTheDocument();
+  });
+});