ai-evaluate 2.1.6 → 2.1.8

package/src/worker-template/test-generator.ts

@@ -0,0 +1,358 @@
+/**
+ * Test framework embedding for worker templates
+ *
+ * Generates the embedded test framework code used in dev mode
+ * (vitest-compatible API without external dependencies)
+ */
+
+/**
+ * Generate the embedded test framework code
+ * This creates a vitest-compatible testing API that runs in the sandbox
+ */
+export function generateTestFrameworkCode(): string {
+  return `
+// Test framework (vitest-compatible API)
+let currentDescribe = '';
+let beforeEachFns = [];
+let afterEachFns = [];
+
+const describe = (name, fn) => {
+  const prev = currentDescribe;
+  const prevBeforeEach = [...beforeEachFns];
+  const prevAfterEach = [...afterEachFns];
+  currentDescribe = currentDescribe ? currentDescribe + ' > ' + name : name;
+  try { fn(); } finally {
+    currentDescribe = prev;
+    beforeEachFns = prevBeforeEach;
+    afterEachFns = prevAfterEach;
+  }
+};
+
+// Hooks
+const beforeEach = (fn) => { beforeEachFns.push(fn); };
+const afterEach = (fn) => { afterEachFns.push(fn); };
+
+const it = (name, fn) => {
+  const fullName = currentDescribe ? currentDescribe + ' > ' + name : name;
+  const hooks = { before: [...beforeEachFns], after: [...afterEachFns] };
+  pendingTests.push({ name: fullName, fn, hooks });
+};
+const test = it;
+
+it.skip = (name, fn) => {
+  const fullName = currentDescribe ? currentDescribe + ' > ' + name : name;
+  pendingTests.push({ name: fullName, fn: null, skip: true });
+};
+test.skip = it.skip;
+
+it.only = (name, fn) => {
+  const fullName = currentDescribe ? currentDescribe + ' > ' + name : name;
+  const hooks = { before: [...beforeEachFns], after: [...afterEachFns] };
+  pendingTests.push({ name: fullName, fn, hooks, only: true });
+};
+test.only = it.only;
+
+// Deep equality check
+const deepEqual = (a, b) => {
+  if (a === b) return true;
+  if (a == null || b == null) return false;
+  if (typeof a !== typeof b) return false;
+  if (typeof a !== 'object') return false;
+  if (Array.isArray(a) !== Array.isArray(b)) return false;
+  if (Array.isArray(a)) {
+    if (a.length !== b.length) return false;
+    return a.every((v, i) => deepEqual(v, b[i]));
+  }
+  const keysA = Object.keys(a);
+  const keysB = Object.keys(b);
+  if (keysA.length !== keysB.length) return false;
+  return keysA.every(k => deepEqual(a[k], b[k]));
+};
+
+// Expect implementation with vitest-compatible matchers
+const expect = (actual) => {
+  const matchers = {
+    toBe: (expected) => {
+      if (actual !== expected) {
+        throw new Error(\`Expected \${JSON.stringify(expected)} but got \${JSON.stringify(actual)}\`);
+      }
+    },
+    toEqual: (expected) => {
+      if (!deepEqual(actual, expected)) {
+        throw new Error(\`Expected \${JSON.stringify(expected)} but got \${JSON.stringify(actual)}\`);
+      }
+    },
+    toStrictEqual: (expected) => {
+      if (!deepEqual(actual, expected)) {
+        throw new Error(\`Expected \${JSON.stringify(expected)} but got \${JSON.stringify(actual)}\`);
+      }
+    },
+    toBeTruthy: () => {
+      if (!actual) throw new Error(\`Expected truthy but got \${JSON.stringify(actual)}\`);
+    },
+    toBeFalsy: () => {
+      if (actual) throw new Error(\`Expected falsy but got \${JSON.stringify(actual)}\`);
+    },
+    toBeNull: () => {
+      if (actual !== null) throw new Error(\`Expected null but got \${JSON.stringify(actual)}\`);
+    },
+    toBeUndefined: () => {
+      if (actual !== undefined) throw new Error(\`Expected undefined but got \${JSON.stringify(actual)}\`);
+    },
+    toBeDefined: () => {
+      if (actual === undefined) throw new Error('Expected defined but got undefined');
+    },
+    toBeNaN: () => {
+      if (!Number.isNaN(actual)) throw new Error(\`Expected NaN but got \${actual}\`);
+    },
+    toContain: (item) => {
+      if (Array.isArray(actual)) {
+        if (!actual.includes(item)) throw new Error(\`Expected array to contain \${JSON.stringify(item)}\`);
+      } else if (typeof actual === 'string') {
+        if (!actual.includes(item)) throw new Error(\`Expected string to contain "\${item}"\`);
+      } else {
+        throw new Error('toContain only works on arrays and strings');
+      }
+    },
+    toContainEqual: (item) => {
+      if (!Array.isArray(actual)) throw new Error('toContainEqual only works on arrays');
+      if (!actual.some(v => deepEqual(v, item))) {
+        throw new Error(\`Expected array to contain \${JSON.stringify(item)}\`);
+      }
+    },
+    toHaveLength: (length) => {
+      if (actual?.length !== length) {
+        throw new Error(\`Expected length \${length} but got \${actual?.length}\`);
+      }
+    },
+    toHaveProperty: function(path, value) {
+      const parts = typeof path === 'string' ? path.split('.') : [path];
+      let obj = actual;
+      for (const part of parts) {
+        if (obj == null || !(part in obj)) {
+          throw new Error(\`Expected object to have property "\${path}"\`);
+        }
+        obj = obj[part];
+      }
+      if (arguments.length > 1 && !deepEqual(obj, value)) {
+        throw new Error(\`Expected property "\${path}" to be \${JSON.stringify(value)} but got \${JSON.stringify(obj)}\`);
+      }
+    },
+    toMatchObject: (expected) => {
+      if (typeof actual !== 'object' || actual === null) {
+        throw new Error('toMatchObject expects an object');
+      }
+      for (const key of Object.keys(expected)) {
+        if (!deepEqual(actual[key], expected[key])) {
+          throw new Error(\`Expected property "\${key}" to be \${JSON.stringify(expected[key])} but got \${JSON.stringify(actual[key])}\`);
+        }
+      }
+    },
+    toThrow: (expected) => {
+      if (typeof actual !== 'function') throw new Error('toThrow expects a function');
+      let threw = false;
+      let error;
+      try {
+        actual();
+      } catch (e) {
+        threw = true;
+        error = e;
+      }
+      if (!threw) throw new Error('Expected function to throw');
+      if (expected !== undefined) {
+        if (typeof expected === 'string' && !error.message.includes(expected)) {
+          throw new Error(\`Expected error message to contain "\${expected}" but got "\${error.message}"\`);
+        }
+        if (expected instanceof RegExp && !expected.test(error.message)) {
+          throw new Error(\`Expected error message to match \${expected} but got "\${error.message}"\`);
+        }
+        if (typeof expected === 'function' && !(error instanceof expected)) {
+          throw new Error(\`Expected error to be instance of \${expected.name}\`);
+        }
+      }
+    },
+    toBeGreaterThan: (n) => {
+      if (!(actual > n)) throw new Error(\`Expected \${actual} to be greater than \${n}\`);
+    },
+    toBeLessThan: (n) => {
+      if (!(actual < n)) throw new Error(\`Expected \${actual} to be less than \${n}\`);
+    },
+    toBeGreaterThanOrEqual: (n) => {
+      if (!(actual >= n)) throw new Error(\`Expected \${actual} to be >= \${n}\`);
+    },
+    toBeLessThanOrEqual: (n) => {
+      if (!(actual <= n)) throw new Error(\`Expected \${actual} to be <= \${n}\`);
+    },
+    toBeCloseTo: (n, digits = 2) => {
+      const diff = Math.abs(actual - n);
+      const threshold = Math.pow(10, -digits) / 2;
+      if (diff > threshold) {
+        throw new Error(\`Expected \${actual} to be close to \${n}\`);
+      }
+    },
+    toMatch: (pattern) => {
+      if (typeof actual !== 'string') throw new Error('toMatch expects a string');
+      const regex = typeof pattern === 'string' ? new RegExp(pattern) : pattern;
+      if (!regex.test(actual)) {
+        throw new Error(\`Expected "\${actual}" to match \${pattern}\`);
+      }
+    },
+    toBeInstanceOf: (cls) => {
+      if (!(actual instanceof cls)) {
+        throw new Error(\`Expected instance of \${cls.name}\`);
+      }
+    },
+    toBeTypeOf: (type) => {
+      if (typeof actual !== type) {
+        throw new Error(\`Expected typeof to be "\${type}" but got "\${typeof actual}"\`);
+      }
+    },
+  };
+
+  matchers.not = {
+    toBe: (expected) => {
+      if (actual === expected) throw new Error(\`Expected not \${JSON.stringify(expected)}\`);
+    },
+    toEqual: (expected) => {
+      if (deepEqual(actual, expected)) {
+        throw new Error(\`Expected not equal to \${JSON.stringify(expected)}\`);
+      }
+    },
+    toBeTruthy: () => {
+      if (actual) throw new Error('Expected not truthy');
+    },
+    toBeFalsy: () => {
+      if (!actual) throw new Error('Expected not falsy');
+    },
+    toBeNull: () => {
+      if (actual === null) throw new Error('Expected not null');
+    },
+    toBeUndefined: () => {
+      if (actual === undefined) throw new Error('Expected not undefined');
+    },
+    toBeDefined: () => {
+      if (actual !== undefined) throw new Error('Expected undefined');
+    },
+    toContain: (item) => {
+      if (Array.isArray(actual) && actual.includes(item)) {
+        throw new Error(\`Expected array not to contain \${JSON.stringify(item)}\`);
+      }
+      if (typeof actual === 'string' && actual.includes(item)) {
+        throw new Error(\`Expected string not to contain "\${item}"\`);
+      }
+    },
+    toHaveProperty: (path) => {
+      const parts = typeof path === 'string' ? path.split('.') : [path];
+      let obj = actual;
+      try {
+        for (const part of parts) {
+          if (obj == null || !(part in obj)) return;
+          obj = obj[part];
+        }
+        throw new Error(\`Expected object not to have property "\${path}"\`);
+      } catch {}
+    },
+    toThrow: () => {
+      if (typeof actual !== 'function') throw new Error('toThrow expects a function');
+      try {
+        actual();
+      } catch (e) {
+        throw new Error('Expected function not to throw');
+      }
+    },
+    toMatch: (pattern) => {
+      if (typeof actual !== 'string') throw new Error('toMatch expects a string');
+      const regex = typeof pattern === 'string' ? new RegExp(pattern) : pattern;
+      if (regex.test(actual)) {
+        throw new Error(\`Expected "\${actual}" not to match \${pattern}\`);
+      }
+    },
+  };
+
+  matchers.resolves = new Proxy({}, {
+    get: (_, prop) => async (...args) => {
+      const resolved = await actual;
+      return expect(resolved)[prop](...args);
+    }
+  });
+
+  matchers.rejects = new Proxy({}, {
+    get: (_, prop) => async (...args) => {
+      try {
+        await actual;
+        throw new Error('Expected promise to reject');
+      } catch (e) {
+        if (e.message === 'Expected promise to reject') throw e;
+        return expect(e)[prop](...args);
+      }
+    }
+  });
+
+  return matchers;
+};
+`
+}
+
+/**
+ * Generate the test runner code that executes pending tests
+ */
+export function generateTestRunnerCode(): string {
+  return `
+    // Run all pending tests
+    const testStart = Date.now();
+    const hasOnly = pendingTests.some(t => t.only);
+    const testsToRun = hasOnly ? pendingTests.filter(t => t.only || t.skip) : pendingTests;
+
+    for (const { name, fn, hooks, skip } of testsToRun) {
+      testResults.total++;
+
+      if (skip) {
+        testResults.skipped++;
+        testResults.tests.push({ name, passed: true, skipped: true, duration: 0 });
+        continue;
+      }
+
+      const start = Date.now();
+      try {
+        // Run beforeEach hooks
+        if (hooks?.before) {
+          for (const hook of hooks.before) {
+            const hookResult = hook();
+            if (hookResult && typeof hookResult.then === 'function') {
+              await hookResult;
+            }
+          }
+        }
+
+        // Run the test
+        const result = fn();
+        if (result && typeof result.then === 'function') {
+          await result;
+        }
+
+        // Run afterEach hooks
+        if (hooks?.after) {
+          for (const hook of hooks.after) {
+            const hookResult = hook();
+            if (hookResult && typeof hookResult.then === 'function') {
+              await hookResult;
+            }
+          }
+        }
+
+        testResults.passed++;
+        testResults.tests.push({ name, passed: true, duration: Date.now() - start });
+      } catch (e) {
+        testResults.failed++;
+        testResults.tests.push({
+          name,
+          passed: false,
+          error: e.message || String(e),
+          duration: Date.now() - start
+        });
+      }
+    }
+
+    testResults.duration = Date.now() - testStart;
+`
+}

package/test/miniflare-pool.test.ts

@@ -0,0 +1,246 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import {
+  configurePool,
+  getPoolConfig,
+  getPoolStats,
+  warmPool,
+  disposePool,
+  resetPool,
+} from '../src/miniflare-pool.js'
+import { evaluate } from '../src/node.js'
+
+describe('Miniflare Pool', () => {
+  // Reset pool state between tests
+  beforeEach(async () => {
+    await resetPool()
+  })
+
+  afterEach(async () => {
+    await disposePool()
+  })
+
+  describe('configurePool', () => {
+    it('sets pool size', () => {
+      configurePool({ size: 5 })
+      const config = getPoolConfig()
+      expect(config.size).toBe(5)
+    })
+
+    it('sets maxIdleTime', () => {
+      configurePool({ maxIdleTime: 60000 })
+      const config = getPoolConfig()
+      expect(config.maxIdleTime).toBe(60000)
+    })
+
+    it('preserves existing values when not specified', () => {
+      configurePool({ size: 5 })
+      configurePool({ maxIdleTime: 60000 })
+      const config = getPoolConfig()
+      expect(config.size).toBe(5)
+      expect(config.maxIdleTime).toBe(60000)
+    })
+  })
+
+  describe('getPoolConfig', () => {
+    it('returns default configuration', () => {
+      const config = getPoolConfig()
+      expect(config.size).toBe(3)
+      expect(config.maxIdleTime).toBe(30000)
+    })
+
+    it('returns a copy of configuration', () => {
+      const config1 = getPoolConfig()
+      const config2 = getPoolConfig()
+      expect(config1).not.toBe(config2)
+      expect(config1).toEqual(config2)
+    })
+  })
+
+  describe('getPoolStats', () => {
+    it('returns initial empty stats', () => {
+      const stats = getPoolStats()
+      expect(stats.size).toBe(0)
+      expect(stats.available).toBe(0)
+      expect(stats.inUse).toBe(0)
+    })
+
+    it('shows correct stats after warming', async () => {
+      await warmPool(2)
+      const stats = getPoolStats()
+      expect(stats.size).toBe(2)
+      expect(stats.available).toBe(2)
+      expect(stats.inUse).toBe(0)
+    })
+  })
+
+  describe('warmPool', () => {
+    it('creates specified number of instances', async () => {
+      await warmPool(2)
+      const stats = getPoolStats()
+      expect(stats.size).toBe(2)
+    })
+
+    it('respects pool size limit', async () => {
+      configurePool({ size: 2 })
+      await warmPool(5)
+      const stats = getPoolStats()
+      // warmPool creates up to the requested count, but pool won't grow beyond size
+      // when new instances are acquired
+      expect(stats.size).toBeLessThanOrEqual(5)
+    })
+
+    it('does not create duplicates when called multiple times', async () => {
+      await warmPool(2)
+      await warmPool(2)
+      const stats = getPoolStats()
+      expect(stats.size).toBe(2)
+    })
+  })
+
+  describe('disposePool', () => {
+    it('disposes all instances', async () => {
+      await warmPool(3)
+      await disposePool()
+      const stats = getPoolStats()
+      expect(stats.size).toBe(0)
+    })
+
+    it('can be called multiple times safely', async () => {
+      await warmPool(2)
+      await disposePool()
+      await disposePool()
+      const stats = getPoolStats()
+      expect(stats.size).toBe(0)
+    })
+  })
+
+  describe('resetPool', () => {
+    it('disposes instances and resets config', async () => {
+      configurePool({ size: 10, maxIdleTime: 60000 })
+      await warmPool(3)
+      await resetPool()
+      const stats = getPoolStats()
+      const config = getPoolConfig()
+      expect(stats.size).toBe(0)
+      expect(config.size).toBe(3)
+      expect(config.maxIdleTime).toBe(30000)
+    })
+  })
+
+  describe('pool integration with evaluate', () => {
+    it('reuses instances across evaluations', async () => {
+      // Warm the pool
+      await warmPool(1)
+
+      // Run multiple evaluations
+      const results = await Promise.all([
+        evaluate({ script: 'return 1' }),
+        evaluate({ script: 'return 2' }),
+        evaluate({ script: 'return 3' }),
+      ])
+
+      expect(results[0].success).toBe(true)
+      expect(results[0].value).toBe(1)
+      expect(results[1].success).toBe(true)
+      expect(results[1].value).toBe(2)
+      expect(results[2].success).toBe(true)
+      expect(results[2].value).toBe(3)
+
+      // Pool should have created instances as needed
+      const stats = getPoolStats()
+      expect(stats.size).toBeGreaterThan(0)
+    })
+
+    it('returns instances to pool after evaluation', async () => {
+      // This test verifies the pool reuses instances by running multiple
+      // sequential evaluations. If pooling weren't working, each evaluation
+      // would be slower due to creating new Miniflare instances.
+      configurePool({ size: 1 })
+
+      // Run sequential evaluations with same pool size
+      const result1 = await evaluate({ script: 'return 1' })
+      expect(result1.success).toBe(true)
+      expect(result1.value).toBe(1)
+
+      const result2 = await evaluate({ script: 'return 2' })
+      expect(result2.success).toBe(true)
+      expect(result2.value).toBe(2)
+
+      const result3 = await evaluate({ script: 'return 3' })
+      expect(result3.success).toBe(true)
+      expect(result3.value).toBe(3)
+
+      // All evaluations should succeed, demonstrating instance reuse
+      // (If instances weren't being released, pool would exhaust quickly
+      // with size=1 and sequential evaluations would fail)
+    })
+
+    it('creates temporary instances when pool is exhausted', async () => {
+      configurePool({ size: 1 })
+      await warmPool(1)
+
+      // Start multiple concurrent evaluations
+      const evaluationPromises = [
+        evaluate({ script: 'return 1' }),
+        evaluate({ script: 'return 2' }),
+        evaluate({ script: 'return 3' }),
+      ]
+
+      const results = await Promise.all(evaluationPromises)
+
+      // All should succeed (some using temporary instances)
+      expect(results.every((r) => r.success)).toBe(true)
+    })
+
+    it('handles errors without leaking instances', async () => {
+      configurePool({ size: 1 })
+
+      // Run an evaluation that throws
+      const result = await evaluate({
+        script: 'throw new Error("test error")',
+      })
+
+      expect(result.success).toBe(false)
+      expect(result.error).toContain('test error')
+
+      // Instance should still be returned to pool
+      const stats = getPoolStats()
+      expect(stats.inUse).toBe(0)
+    })
+
+    it('handles timeouts without leaking instances', async () => {
+      configurePool({ size: 1 })
+
+      // Run an evaluation that times out
+      const result = await evaluate({
+        script: 'while(true) {}',
+        timeout: 100,
+      })
+
+      expect(result.success).toBe(false)
+      expect(result.error).toContain('Timeout')
+
+      // Instance should still be returned to pool
+      const stats = getPoolStats()
+      expect(stats.inUse).toBe(0)
+    })
+
+    it('maintains isolation between evaluations', async () => {
+      configurePool({ size: 1 })
+
+      // First evaluation sets a global
+      const result1 = await evaluate({
+        script: 'globalThis.testValue = 42; return globalThis.testValue;',
+      })
+      expect(result1.success).toBe(true)
+      expect(result1.value).toBe(42)
+
+      // Second evaluation should not see the global (new worker script)
+      const result2 = await evaluate({
+        script: 'return globalThis.testValue;',
+      })
+      expect(result2.success).toBe(true)
+      expect(result2.value).toBeUndefined()
+    })
+  })
+})