@polderlabs/bizar 5.0.2 → 5.1.0

package/bizar-dash/tests/eval/fixtures-extra.test.mjs

@@ -0,0 +1,285 @@
+/**
+ * tests/eval/fixtures-extra.test.mjs
+ *
+ * v5.0.0 — Tests for expanded fixture set (10 new fixtures).
+ *
+ * Verifies:
+ *   - All 15 fixtures load without errors
+ *   - All fixtures have valid JSON and required fields
+ *   - runFixtureById correctly resolves and runs fixtures
+ *   - New expectation types (toolCallsMin, toolSequence, minItems) work
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const EVAL = await import('../../src/server/eval.mjs');
+
+const FIXTURES_DIR = join(__dirname, '../../../templates/eval-fixtures');
+
+const REQUIRED_FIELDS = ['id', 'name', 'description', 'agent', 'prompt', 'expected', 'tags'];
+const REQUIRED_EXPECTED_FIELDS = ['contains', 'notContains', 'regex', 'jsonSchema', 'maxTokens', 'maxLatencyMs'];
+
+// Expected fixture IDs - the new 10 fixtures
+const NEW_FIXTURE_IDS = [
+  'tool-call-multi-step',
+  'error-recovery',
+  'context-window',
+  'json-output',
+  'code-review',
+  'unicode-handling',
+  'multi-language',
+  'safe-paths',
+  'concise-output',
+  'citation',
+];
+
+describe('eval fixtures (extra 10)', () => {
+
+  describe('loadFixtures loads all 15 fixtures', () => {
+    it('loads at least 15 fixtures from the directory', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      assert.ok(fixtures.length >= 15, `Expected at least 15 fixtures, got ${fixtures.length}`);
+    });
+
+    it('loads all expected fixture IDs', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const ids = fixtures.map((f) => f.id);
+      for (const expectedId of NEW_FIXTURE_IDS) {
+        assert.ok(
+          ids.includes(expectedId),
+          `Expected fixture ID "${expectedId}" not found in: ${ids.join(', ')}`,
+        );
+      }
+    });
+  });
+
+  describe('each new fixture validates correctly', () => {
+    const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+    const newFixtures = fixtures.filter((f) =>
+      NEW_FIXTURE_IDS.includes(f.id)
+    );
+
+    for (const fixture of newFixtures) {
+      it(`fixture "${fixture.id}" has valid JSON structure`, () => {
+        // Required fields
+        for (const field of REQUIRED_FIELDS) {
+          assert.ok(
+            fixture[field] !== undefined && fixture[field] !== null,
+            `fixture "${fixture.id}" missing required field: ${field}`,
+          );
+        }
+
+        // expected must be an object
+        assert.ok(
+          typeof fixture.expected === 'object',
+          `fixture "${fixture.id}".expected must be an object`,
+        );
+
+        // tags must be an array
+        assert.ok(
+          Array.isArray(fixture.tags),
+          `fixture "${fixture.id}".tags must be an array`,
+        );
+      });
+
+      it(`fixture "${fixture.id}" has at least one check type`, () => {
+        const expected = fixture.expected;
+        const hasCheckType = [
+          expected.contains,
+          expected.notContains,
+          expected.regex,
+          expected.jsonSchema,
+          expected.maxTokens,
+          expected.maxLatencyMs,
+          expected.toolCallsMin,
+          expected.toolSequence,
+        ].some((v) => v !== undefined && v !== null);
+        assert.ok(hasCheckType, `fixture "${fixture.id}" expected must have at least one check type`);
+      });
+    }
+  });
+
+  describe('runFixtureById', () => {
+    it('resolves and runs a fixture by id', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'ok',
+        usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
+      });
+      const result = await EVAL.runFixtureById('latency-bounds', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      assert.strictEqual(result.fixtureId, 'latency-bounds');
+      assert.ok(result.checks.length > 0, 'should have run checks');
+    });
+
+    it('throws for unknown fixture id', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({ content: 'ok', usage: { totalTokens: 0 } });
+      await assert.rejects(
+        () => EVAL.runFixtureById('nonexistent-fixture', fixtures, { llmCall: mockLlmCall }),
+        /Fixture not found/,
+      );
+    });
+  });
+
+  describe('toolCallsMin check', () => {
+    it('passes when tool calls meet minimum', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'found it',
+        usage: { totalTokens: 100 },
+      });
+      const result = await EVAL.runFixtureById('tool-call-multi-step', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+        toolCalls: ['search', 'read', 'search'],
+      });
+      // The toolCallsMin check should be in the results
+      const toolCallsMinCheck = result.checks.find((c) => c.kind === 'toolCallsMin');
+      assert.ok(toolCallsMinCheck, 'should have a toolCallsMin check');
+      assert.ok(toolCallsMinCheck.ok, `toolCallsMin check should pass: ${toolCallsMinCheck.message}`);
+    });
+
+    it('fails when tool calls are below minimum', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'found it',
+        usage: { totalTokens: 100 },
+      });
+      const result = await EVAL.runFixtureById('tool-call-multi-step', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+        toolCalls: ['search'], // only 1, requires 3
+      });
+      const toolCallsMinCheck = result.checks.find((c) => c.kind === 'toolCallsMin');
+      assert.ok(toolCallsMinCheck, 'should have a toolCallsMin check');
+      assert.ok(!toolCallsMinCheck.ok, 'toolCallsMin check should fail');
+    });
+  });
+
+  describe('toolSequence check', () => {
+    it('passes when tool sequence matches', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'found it',
+        usage: { totalTokens: 100 },
+      });
+      const result = await EVAL.runFixtureById('tool-call-multi-step', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+        toolCalls: ['search', 'read', 'search'],
+      });
+      const seqCheck = result.checks.find((c) => c.kind === 'toolSequence');
+      assert.ok(seqCheck, 'should have a toolSequence check');
+      assert.ok(seqCheck.ok, `toolSequence check should pass: ${seqCheck.message}`);
+    });
+
+    it('fails when tool sequence does not match', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'found it',
+        usage: { totalTokens: 100 },
+      });
+      const result = await EVAL.runFixtureById('tool-call-multi-step', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+        toolCalls: ['read', 'search', 'search'], // wrong order
+      });
+      const seqCheck = result.checks.find((c) => c.kind === 'toolSequence');
+      assert.ok(seqCheck, 'should have a toolSequence check');
+      assert.ok(!seqCheck.ok, 'toolSequence check should fail');
+    });
+  });
+
+  describe('jsonSchema minItems check', () => {
+    it('passes when array has enough items', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: '["red", "green", "blue"]',
+        usage: { totalTokens: 50 },
+      });
+      const result = await EVAL.runFixtureById('json-output', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      const jsonCheck = result.checks.find((c) => c.kind === 'jsonSchema');
+      assert.ok(jsonCheck, 'should have a jsonSchema check');
+      assert.ok(jsonCheck.ok, `jsonSchema check should pass: ${jsonCheck.message}`);
+    });
+
+    it('fails when array has too few items', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: '["red"]',
+        usage: { totalTokens: 50 },
+      });
+      const result = await EVAL.runFixtureById('json-output', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      const jsonCheck = result.checks.find((c) => c.kind === 'jsonSchema');
+      assert.ok(jsonCheck, 'should have a jsonSchema check');
+      assert.ok(!jsonCheck.ok, 'jsonSchema check should fail for short array');
+    });
+  });
+
+  describe('notContains check', () => {
+    it('passes when forbidden content is absent', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'The file was not found',
+        usage: { totalTokens: 50 },
+      });
+      const result = await EVAL.runFixtureById('error-recovery', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      // Should not contain crash or panic
+      const notContainsChecks = result.checks.filter((c) => c.kind === 'notContains');
+      assert.ok(notContainsChecks.length > 0, 'should have notContains checks');
+      for (const check of notContainsChecks) {
+        assert.ok(check.ok, `notContains "${check.message}" should pass`);
+      }
+    });
+
+    it('fails when forbidden content is present', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'The system crashed and panicked',
+        usage: { totalTokens: 50 },
+      });
+      const result = await EVAL.runFixtureById('error-recovery', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      const notContainsChecks = result.checks.filter((c) => c.kind === 'notContains');
+      assert.ok(notContainsChecks.length > 0, 'should have notContains checks');
+      assert.ok(!result.ok, 'overall result should fail');
+    });
+  });
+
+  describe('regex check', () => {
+    it('passes when regex matches', async () => {
+      const fixtures = EVAL.loadFixtures(FIXTURES_DIR);
+      const mockLlmCall = async () => ({
+        content: 'The function is at src/utils/add.ts:42',
+        usage: { totalTokens: 50 },
+      });
+      const result = await EVAL.runFixtureById('citation', fixtures, {
+        llmCall: mockLlmCall,
+        timeoutMs: 5000,
+      });
+      const regexChecks = result.checks.filter((c) => c.kind === 'regex');
+      assert.ok(regexChecks.length > 0, 'should have regex checks');
+      for (const check of regexChecks) {
+        assert.ok(check.ok, `regex "${check.message}" should pass`);
+      }
+    });
+  });
+});

package/bizar-dash/tests/otel-spans.test.mjs

@@ -0,0 +1,253 @@
+/**
+ * tests/otel-spans.test.mjs
+ *
+ * v5.1.0 — Tests for the OpenTelemetry span helpers added alongside
+ * the v4.9 → v5.1 OTLP trace expansion.
+ *
+ * Coverage:
+ *   - `withSpan(name, fn, attrs)` — success path returns the callback
+ *     value and does not throw; failure path records the exception
+ *     and rethrows the original error (so `wrap()` can still emit a
+ *     JSON envelope).
+ *   - `setCommonAttributes(span, attrs)` — sets the four documented
+ *     attribute keys when given, and silently skips falsy values so
+ *     unauthenticated routes don't pollute spans with empty ids.
+ *   - `getResourceAttributes()` — produces a non-empty map with the
+ *     seven semantic-convention keys documented at the top of
+ *     otel.mjs.
+ *   - `recordTrace(name, attrs)` — increments the per-key counter and
+ *     matches the brief's key shape (`name:jsonString`).
+ *
+ * The tests use the no-op tracer (no `initOtel` call) so the span
+ * methods are guaranteed to be no-ops and the round-trip semantics
+ * are exactly what route handlers will see when OTEL is disabled.
+ */
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+const OTEL_URL = '../src/server/otel.mjs';
+const METRICS_URL = '../src/server/metrics.mjs';
+
+async function loadOtel() {
+  // Fresh module each call so module-scoped SDK state is clean.
+  const url = `${OTEL_URL}?v=${Date.now()}-${Math.random()}`;
+  return import(url);
+}
+
+async function loadMetrics() {
+  const url = `${METRICS_URL}?v=${Date.now()}-${Math.random()}`;
+  return import(url);
+}
+
+test('withSpan returns the callback value on success', async () => {
+  const otel = await loadOtel();
+  try {
+    // `withSpan(name, fn)` returns an Express-shaped handler that
+    // takes (req, res, …rest) and returns a Promise. Callers `wrap`
+    // it; here we invoke the returned function directly with a
+    // synthetic arg list so we can assert the await semantics.
+    const handler = otel.withSpan('test.success', async (span) => {
+      span.setAttribute('test.key', 'value');
+      return 42;
+    });
+    const result = await handler();
+    assert.equal(result, 42, 'withSpan handler must propagate the callback return value');
+  } finally {
+    await otel.shutdownOtel();
+  }
+});
+
+test('withSpan re-throws when the callback throws (matches wrap() contract)', async () => {
+  const otel = await loadOtel();
+  try {
+    const handler = otel.withSpan('test.error', async () => {
+      throw new Error('boom');
+    });
+    await assert.rejects(handler(), /boom/);
+  } finally {
+    await otel.shutdownOtel();
+  }
+});
+
+test('withSpan re-throws non-Error throwables (e.g. strings)', async () => {
+  const otel = await loadOtel();
+  try {
+    const handler = otel.withSpan('test.stringthrow', async () => {
+      // eslint-disable-next-line no-throw-literal
+      throw 'plain-string-error';
+    });
+    await assert.rejects(handler(), /plain-string-error/);
+  } finally {
+    await otel.shutdownOtel();
+  }
+});
+
+test('withSpan works under no-op tracer when SDK not initialised', async () => {
+  const otel = await loadOtel();
+  try {
+    assert.equal(otel.isOtelEnabled(), false, 'must start with no SDK');
+    const result = await otel.withSpan('test.noop', async () => 'ok')();
+    assert.equal(result, 'ok', 'no-op tracer must not eat the return value');
+    // Calling again must still work — guards against a span leak
+    // where the no-op tracer throws on a second invocation.
+    const result2 = await otel.withSpan('test.noop2', async () => 'still-ok')();
+    assert.equal(result2, 'still-ok');
+  } finally {
+    await otel.shutdownOtel();
+  }
+});
+
+test('withSpan forwards positional args to the handler (req, res shape)', async () => {
+  const otel = await loadOtel();
+  try {
+    const seen = [];
+    const handler = otel.withSpan('test.forward', async (span, req, res) => {
+      seen.push({ method: req?.method, url: req?.url, statusSet: !!res });
+      return 'ok';
+    });
+    await handler({ method: 'GET', url: '/api/foo' }, { writeHead() {} });
+    assert.equal(seen.length, 1);
+    assert.equal(seen[0].method, 'GET');
+    assert.equal(seen[0].url, '/api/foo');
+    assert.equal(seen[0].statusSet, true);
+  } finally {
+    await otel.shutdownOtel();
+  }
+});
+
+test('setCommonAttributes sets all four keys when given', async () => {
+  const otel = await loadOtel();
+  try {
+    const calls = [];
+    const fakeSpan = {
+      setAttribute(key, value) {
+        calls.push({ key, value });
+      },
+    };
+    otel.setCommonAttributes(fakeSpan, {
+      userId: 'u_123',
+      workspaceId: 'w_456',
+      ip: '10.0.0.1',
+      userAgent: 'bizar-test/1.0',
+    });
+    const byKey = Object.fromEntries(calls.map((c) => [c.key, c.value]));
+    assert.equal(byKey['bizar.user.id'], 'u_123');
+    assert.equal(byKey['bizar.workspace.id'], 'w_456');
+    assert.equal(byKey['http.client_ip'], '10.0.0.1');
+    assert.equal(byKey['http.user_agent'], 'bizar-test/1.0');
+  } finally {
+    await otel.shutdownOtel();
+  }
+});
+
+test('setCommonAttributes silently drops empty / undefined values', async () => {
+  const otel = await loadOtel();
+  try {
+    const calls = [];
+    const fakeSpan = {
+      setAttribute(key, value) {
+        calls.push({ key, value });
+      },
+    };
+    otel.setCommonAttributes(fakeSpan, {
+      userId: undefined,
+      workspaceId: null,
+      ip: '',
+      userAgent: '',
+    });
+    assert.equal(
+      calls.length,
+      0,
+      'all-empty inputs must skip every setAttribute call',
+    );
+    otel.setCommonAttributes(fakeSpan, {
+      userId: 'only-user',
+    });
+    assert.deepEqual(calls, [{ key: 'bizar.user.id', value: 'only-user' }]);
+  } finally {
+    await otel.shutdownOtel();
+  }
+});
+
+test('setCommonAttributes tolerates a missing span (defensive)', async () => {
+  const otel = await loadOtel();
+  try {
+    // No throw, no error — route handlers down the call stack
+    // rely on this not throwing so a stray import won't crash.
+    otel.setCommonAttributes(null, { userId: 'x' });
+    otel.setCommonAttributes(undefined, { ip: '1.2.3.4' });
+  } finally {
+    await otel.shutdownOtel();
+  }
+});
+
+test('getResourceAttributes includes the seven OTel semantic keys', async () => {
+  const otel = await loadOtel();
+  try {
+    const attrs = otel.getResourceAttributes();
+    const keys = Object.keys(attrs);
+    // We don't pin the literal attribute strings (semconv 1.41 uses
+    // "service.name", "deployment.environment" etc.) — instead assert
+    // the dashboard-specific payload has the right number of entries
+    // and that service.name is the canonical bizar-dash string.
+    assert.equal(keys.length, 7, 'should populate exactly 7 resource attributes');
+    const values = Object.values(attrs);
+    assert.ok(values.includes('bizar-dash'), 'service.name must be bizar-dash');
+    assert.ok(values.some((v) => v === process.pid), 'process.pid attribute must match process.pid');
+    assert.ok(values.some((v) => typeof v === 'string' && v.length > 0 && v !== 'bizar-dash'), 'must have at least one host/os string attribute');
+  } finally {
+    await otel.shutdownOtel();
+  }
+});
+
+test('recordTrace increments a counter keyed by name + sorted JSON attribute set', async () => {
+  const m = await loadMetrics();
+  try {
+    m.resetTraceCounts();
+    assert.equal(m.recordTrace('alpha', { outcome: 'ok', code: '200' }), 1);
+    assert.equal(m.recordTrace('alpha', { outcome: 'ok', code: '200' }), 2);
+    assert.equal(m.recordTrace('alpha', { code: '200', outcome: 'ok' }), 3,
+      'same attributes in a different insertion order must collide on the same key');
+    assert.equal(m.recordTrace('alpha', { outcome: 'error', code: '500' }), 1,
+      'different attribute values must produce a different key');
+    // Empty attribute object should not collide with the 'undefined' key.
+    assert.equal(m.recordTrace('beta'), 1);
+    assert.equal(m.recordTrace('beta', {}), 2);
+  } finally {
+    m.resetTraceCounts();
+  }
+});
+
+test('recordTrace drops undefined / null / empty attribute values', async () => {
+  const m = await loadMetrics();
+  try {
+    m.resetTraceCounts();
+    const r1 = m.recordTrace('dropper', { outcome: 'ok', code: undefined, ip: null, ua: '' });
+    const r2 = m.recordTrace('dropper', { outcome: 'ok' });
+    assert.equal(r1, 1, 'first call returns 1');
+    assert.equal(r2, 2, 'second call increments to 2 on the same key');
+    // Both calls must produce one shared entry, not two — that proves
+    // the undefined/null/empty values were dropped from the key.
+    const entryKeys = [...m.traceCountByName.keys()];
+    assert.equal(entryKeys.length, 1, 'should have exactly one composite key after dedup');
+    assert.match(entryKeys[0], /^dropper:\{"outcome":"ok"\}$/, 'key payload is just name + non-empty attribute set');
+  } finally {
+    m.resetTraceCounts();
+  }
+});
+
+test('module exports the documented API surface', async () => {
+  const otel = await loadOtel();
+  try {
+    assert.equal(typeof otel.tracer, 'object');
+    assert.equal(typeof otel.initOtel, 'function');
+    assert.equal(typeof otel.shutdownOtel, 'function');
+    assert.equal(typeof otel.isOtelEnabled, 'function');
+    assert.equal(typeof otel.withSpan, 'function');
+    assert.equal(typeof otel.setCommonAttributes, 'function');
+    assert.equal(typeof otel.getResourceAttributes, 'function');
+  } finally {
+    await otel.shutdownOtel();
+  }
+});