ai-evaluate 2.1.7 → 2.1.8

package/src/evaluate.ts

@@ -10,13 +10,9 @@
  */
 
 import type { EvaluateOptions, EvaluateResult, WorkerLoader, SandboxEnv } from './types.js'
-import { generateWorkerCode } from './worker-template.js'
+import { generateWorkerCode } from './worker-template/index.js'
 import { CAPNWEB_SOURCE } from './capnweb-bundle.js'
-
-/**
- * Compatibility date for dynamic workers (2026)
- */
-const COMPATIBILITY_DATE = '2026-01-01'
+import { COMPATIBILITY_DATE, normalizeImport, extractPackageName } from './shared.js'
 
 /**
  * Generate a minimal worker for simple script execution
@@ -37,10 +33,8 @@ function generateSimpleWorkerCode(options: {
 
   // Make imports available as globals
   const importGlobals = imports
-    .map((url, i) => {
-      // Extract package name from URL for variable naming
-      const match = url.match(/esm\.sh\/([^@/]+)/)
-      const pkgName = match ? match[1].replace(/-/g, '_') : `pkg${i}`
+    .map((specifier, i) => {
+      const pkgName = extractPackageName(specifier, i)
       const varName = pkgName === 'lodash' ? '_' : pkgName
       return `globalThis.${varName} = __import${i}__.default || __import${i}__;
 globalThis.pkg = __import${i}__.default || __import${i}__;`
@@ -157,17 +151,24 @@ export async function evaluate(
 }
 
 /**
- * Pre-fetch external modules from URLs (e.g., esm.sh)
+ * Pre-fetch external modules from URLs or package names
  * Returns a map of module name to source code
  *
+ * Supports:
+ * - Full URLs: https://esm.sh/lodash@4.17.21
+ * - Bare package names: lodash, lodash@4.17.21, @scope/pkg
+ *
  * Handles esm.sh's redirect-style modules by following the internal import paths.
  */
 async function prefetchModules(imports: string[]): Promise<Record<string, string>> {
   const modules: Record<string, string> = {}
 
   await Promise.all(
-    imports.map(async (url, i) => {
+    imports.map(async (specifier, i) => {
       try {
+        // Normalize bare package names to esm.sh URLs
+        const url = normalizeImport(specifier)
+
         // For esm.sh URLs, try to get the bundled version directly
         let fetchUrl = url
         if (url.includes('esm.sh/') && !url.includes('.mjs') && !url.includes('.js')) {
@@ -197,7 +198,9 @@ async function prefetchModules(imports: string[]): Promise<Record<string, string
         modules[moduleName] = source
       } catch (error) {
         throw new Error(
-          `Failed to fetch import ${url}: ${error instanceof Error ? error.message : String(error)}`
+          `Failed to fetch import ${specifier}: ${
+            error instanceof Error ? error.message : String(error)
+          }`
         )
       }
     })
@@ -244,8 +247,8 @@ async function evaluateSimple(
       ...externalModules,
     },
     compatibilityDate: COMPATIBILITY_DATE,
-    // Block network access only if fetch: null
-    globalOutbound: options.fetch === null ? null : undefined,
+    // Block network if fetch is false or null
+    globalOutbound: options.fetch === false || options.fetch === null ? null : undefined,
   }))
 
   // Get the entrypoint and call fetch
@@ -274,6 +277,7 @@ async function evaluateWithWorkerLoader(
     script: options.script,
     sdk: options.sdk,
     imports: options.imports,
+    fetch: options.fetch,
   })
   const id = `sandbox-${Date.now()}-${Math.random().toString(36).slice(2)}`
 
@@ -285,8 +289,8 @@ async function evaluateWithWorkerLoader(
       'capnweb.js': CAPNWEB_SOURCE,
     },
     compatibilityDate: COMPATIBILITY_DATE,
-    // Block network access only if fetch: null
-    globalOutbound: options.fetch === null ? null : undefined,
+    // Block network if fetch is false or null
+    globalOutbound: options.fetch === false || options.fetch === null ? null : undefined,
     bindings: {
       TEST: testService,
     },

package/src/index.ts

@@ -9,6 +9,7 @@
  */
 
 export { evaluate, createEvaluator } from './evaluate.js'
+export { normalizeImport, normalizeImports } from './shared.js'
 
 export type {
   EvaluateOptions,
@@ -17,5 +18,6 @@ export type {
   TestResults,
   TestResult,
   SandboxEnv,
-  SDKConfig
+  SDKConfig,
+  FetchConfig,
 } from './types.js'

package/src/miniflare-pool.ts

@@ -0,0 +1,395 @@
+/**
+ * Miniflare instance pool for improved performance
+ *
+ * Reuses Miniflare instances between evaluations instead of creating/disposing
+ * for each evaluation, providing 4-5x performance improvement.
+ *
+ * Uses Miniflare's setOptions() to update the worker script between uses,
+ * avoiding the expensive instance creation/teardown cycle.
+ */
+
+import type {
+  Miniflare as MiniflareType,
+  MiniflareOptions as MiniflareOptionsType,
+} from 'miniflare'
+
+/**
+ * Pool configuration options
+ */
+export interface PoolConfig {
+  /** Number of instances to maintain in the pool (default: 3) */
+  size?: number
+  /** Milliseconds before disposing idle instance (default: 30000) */
+  maxIdleTime?: number
+}
+
+/**
+ * Outbound service handler type for network control
+ * - () => never: Block all network access (throw error)
+ * - (request: Request) => Response | Promise<Response>: Custom handler (allowlist, proxy, etc.)
+ */
+export type OutboundServiceHandler =
+  | (() => never)
+  | ((request: Request) => Response | Promise<Response>)
+
+/**
+ * Options for updating a pooled instance's worker
+ */
+export interface WorkerOptions {
+  script: string
+  compatibilityDate?: string
+  outboundService?: OutboundServiceHandler
+}
+
+/**
+ * A pooled Miniflare instance with metadata
+ */
+interface PooledInstance {
+  instance: MiniflareType
+  inUse: boolean
+  lastUsed: number
+  createdAt: number
+}
+
+// Type for the Miniflare constructor
+type MiniflareConstructor = new (config: MiniflareOptionsType) => MiniflareType
+
+/**
+ * Global pool state (singleton per process)
+ */
+let pool: PooledInstance[] = []
+let poolConfig: Required<PoolConfig> = {
+  size: 3,
+  maxIdleTime: 30000,
+}
+let idleCleanupInterval: NodeJS.Timeout | null = null
+let MiniflareClass: MiniflareConstructor | null = null
+let isShuttingDown = false
+
+// Default worker script for warm instances
+const WARM_WORKER_SCRIPT = `
+export default {
+  async fetch(request, env) {
+    return new Response('ready', { status: 200 });
+  }
+};
+`
+
+/**
+ * Configure the Miniflare pool
+ *
+ * @example
+ * ```ts
+ * import { configurePool } from 'ai-evaluate/node'
+ *
+ * configurePool({
+ *   size: 5,        // Keep 5 warm instances
+ *   maxIdleTime: 60000  // Dispose after 60s idle
+ * })
+ * ```
+ */
+export function configurePool(config: PoolConfig): void {
+  poolConfig = {
+    size: config.size ?? poolConfig.size,
+    maxIdleTime: config.maxIdleTime ?? poolConfig.maxIdleTime,
+  }
+  startIdleCleanup()
+}
+
+/**
+ * Get the current pool configuration
+ */
+export function getPoolConfig(): Required<PoolConfig> {
+  return { ...poolConfig }
+}
+
+/**
+ * Get pool statistics for monitoring
+ */
+export function getPoolStats(): {
+  size: number
+  available: number
+  inUse: number
+  config: Required<PoolConfig>
+} {
+  const available = pool.filter((p) => !p.inUse).length
+  return {
+    size: pool.length,
+    available,
+    inUse: pool.length - available,
+    config: { ...poolConfig },
+  }
+}
+
+/**
+ * Initialize the Miniflare class (lazy load)
+ */
+async function getMiniflareClass(): Promise<MiniflareConstructor> {
+  if (!MiniflareClass) {
+    const { Miniflare } = await import('miniflare')
+    MiniflareClass = Miniflare as MiniflareConstructor
+  }
+  return MiniflareClass
+}
+
+/**
+ * Create a new Miniflare instance with a warm worker
+ */
+async function createInstance(): Promise<MiniflareType> {
+  const Miniflare = await getMiniflareClass()
+  return new Miniflare({
+    modules: true,
+    script: WARM_WORKER_SCRIPT,
+    compatibilityDate: '2026-01-01',
+  })
+}
+
+/**
+ * Start the idle cleanup interval
+ */
+function startIdleCleanup(): void {
+  if (idleCleanupInterval) {
+    clearInterval(idleCleanupInterval)
+  }
+
+  idleCleanupInterval = setInterval(async () => {
+    if (isShuttingDown) return
+
+    const now = Date.now()
+    const toDispose: PooledInstance[] = []
+
+    // Find idle instances beyond the idle timeout
+    for (let i = pool.length - 1; i >= 0; i--) {
+      const item = pool[i]
+      if (!item.inUse && now - item.lastUsed > poolConfig.maxIdleTime) {
+        // Keep at least one warm instance
+        if (pool.filter((p) => !p.inUse && !toDispose.includes(p)).length > 1) {
+          toDispose.push(item)
+          pool.splice(i, 1)
+        }
+      }
+    }
+
+    // Dispose old instances
+    for (const item of toDispose) {
+      try {
+        await item.instance.dispose()
+      } catch {
+        // Ignore disposal errors
+      }
+    }
+  }, 5000) // Check every 5 seconds
+
+  // Don't keep the process alive just for cleanup
+  if (idleCleanupInterval.unref) {
+    idleCleanupInterval.unref()
+  }
+}
+
+/**
+ * Acquire a Miniflare instance from the pool and configure it with a worker
+ *
+ * If a free instance is available, it will be reconfigured and returned.
+ * Otherwise, a new instance will be created (up to pool size limit).
+ * If pool is exhausted, creates a temporary instance.
+ *
+ * @param workerOptions - Configuration for the worker to run
+ * @returns Object with the configured instance and a release function
+ */
+export async function acquireInstance(workerOptions: WorkerOptions): Promise<{
+  instance: MiniflareType
+  release: () => Promise<void>
+  isPooled: boolean
+}> {
+  if (isShuttingDown) {
+    throw new Error('Pool is shutting down')
+  }
+
+  // Start idle cleanup if not started
+  if (!idleCleanupInterval) {
+    startIdleCleanup()
+  }
+
+  const { script, compatibilityDate = '2026-01-01', outboundService } = workerOptions
+
+  // Build the options for setOptions
+  const updateOptions: MiniflareOptionsType = {
+    modules: true,
+    script,
+    compatibilityDate,
+  }
+
+  // Only add outboundService if it's defined (for blocking network)
+  if (outboundService !== undefined) {
+    updateOptions.outboundService = outboundService
+  }
+
+  // Try to find an available instance
+  const available = pool.find((p) => !p.inUse)
+  if (available) {
+    available.inUse = true
+    // Reconfigure the instance with the new worker script
+    await available.instance.setOptions(updateOptions)
+    return {
+      instance: available.instance,
+      release: async () => {
+        available.inUse = false
+        available.lastUsed = Date.now()
+      },
+      isPooled: true,
+    }
+  }
+
+  // Create new instance if pool not full
+  if (pool.length < poolConfig.size) {
+    const Miniflare = await getMiniflareClass()
+    const instance = new Miniflare(updateOptions)
+    const pooled: PooledInstance = {
+      instance,
+      inUse: true,
+      lastUsed: Date.now(),
+      createdAt: Date.now(),
+    }
+    pool.push(pooled)
+    return {
+      instance,
+      release: async () => {
+        pooled.inUse = false
+        pooled.lastUsed = Date.now()
+      },
+      isPooled: true,
+    }
+  }
+
+  // Pool exhausted - create temporary instance
+  const Miniflare = await getMiniflareClass()
+  const tempInstance = new Miniflare(updateOptions)
+  return {
+    instance: tempInstance,
+    release: async () => {
+      // Dispose temporary instance immediately
+      await tempInstance.dispose()
+    },
+    isPooled: false,
+  }
+}
+
+/**
+ * Pre-warm the pool with instances
+ *
+ * Call this at application startup to avoid cold start latency.
+ *
+ * @example
+ * ```ts
+ * import { warmPool } from 'ai-evaluate/node'
+ *
+ * // Pre-warm 3 instances at startup
+ * await warmPool(3)
+ * ```
+ */
+export async function warmPool(count?: number): Promise<void> {
+  const targetCount = count ?? poolConfig.size
+  const toCreate = Math.max(0, targetCount - pool.length)
+
+  const promises: Promise<void>[] = []
+  for (let i = 0; i < toCreate; i++) {
+    promises.push(
+      (async () => {
+        const instance = await createInstance()
+        pool.push({
+          instance,
+          inUse: false,
+          lastUsed: Date.now(),
+          createdAt: Date.now(),
+        })
+      })()
+    )
+  }
+
+  await Promise.all(promises)
+
+  // Start idle cleanup if not already started
+  if (!idleCleanupInterval) {
+    startIdleCleanup()
+  }
+}
+
+/**
+ * Dispose all instances and clean up the pool
+ *
+ * Call this before process exit to ensure clean shutdown.
+ *
+ * @example
+ * ```ts
+ * import { disposePool } from 'ai-evaluate/node'
+ *
+ * process.on('beforeExit', async () => {
+ *   await disposePool()
+ * })
+ * ```
+ */
+export async function disposePool(): Promise<void> {
+  isShuttingDown = true
+
+  if (idleCleanupInterval) {
+    clearInterval(idleCleanupInterval)
+    idleCleanupInterval = null
+  }
+
+  const instances = [...pool]
+  pool = []
+
+  await Promise.all(
+    instances.map(async (item) => {
+      try {
+        await item.instance.dispose()
+      } catch {
+        // Ignore disposal errors
+      }
+    })
+  )
+
+  isShuttingDown = false
+}
+
+/**
+ * Reset the pool (for testing purposes)
+ */
+export async function resetPool(): Promise<void> {
+  await disposePool()
+  poolConfig = {
+    size: 3,
+    maxIdleTime: 30000,
+  }
+  MiniflareClass = null
+}
+
+// Register cleanup on process exit
+if (typeof process !== 'undefined') {
+  const cleanup = () => {
+    isShuttingDown = true
+    if (idleCleanupInterval) {
+      clearInterval(idleCleanupInterval)
+    }
+    // Synchronous disposal attempt - best effort
+    for (const item of pool) {
+      try {
+        // Fire and forget - we're exiting anyway
+        item.instance.dispose().catch(() => {})
+      } catch {
+        // Ignore errors during shutdown
+      }
+    }
+    pool = []
+  }
+
+  process.on('exit', cleanup)
+  process.on('SIGINT', () => {
+    cleanup()
+    process.exit(0)
+  })
+  process.on('SIGTERM', () => {
+    cleanup()
+    process.exit(0)
+  })
+}

package/src/node.ts

@@ -5,8 +5,15 @@
  * For Workers-only builds, import from 'ai-evaluate' instead.
  */
 
-import type { EvaluateOptions, EvaluateResult, WorkerLoader, SandboxEnv } from './types.js'
-import { generateWorkerCode, generateDevWorkerCode } from './worker-template.js'
+import type {
+  EvaluateOptions,
+  EvaluateResult,
+  WorkerLoader,
+  SandboxEnv,
+  FetchConfig,
+} from './types.js'
+import { generateWorkerCode, generateDevWorkerCode } from './worker-template/index.js'
+import { isDomainAllowed, normalizeImports } from './shared.js'
 
 /**
  * Check if code contains JSX syntax that needs transformation
@@ -60,6 +67,7 @@ export async function evaluate(
       module: transformedModule,
       tests: transformedTests,
       script: transformedScript,
+      imports: normalizeImports(options.imports),
     }
 
     // Use worker_loaders if available (Cloudflare Workers)
@@ -122,6 +130,25 @@ async function evaluateWithWorkerLoader(
   }
 }
 
+/**
+ * Determine if network access should be blocked based on fetch options
+ * fetch: false | null -> block
+ */
+function shouldBlockNetwork(options: EvaluateOptions): boolean {
+  return options.fetch === false || options.fetch === null
+}
+
+/**
+ * Get allowlist domains if fetch is an array
+ * fetch: string[] -> allowlist
+ */
+function getAllowlistDomains(options: EvaluateOptions): string[] | null {
+  if (Array.isArray(options.fetch)) {
+    return options.fetch
+  }
+  return null
+}
+
 /**
  * Evaluate using Miniflare (for Node.js/development)
  */
@@ -140,20 +167,38 @@ async function evaluateWithMiniflare(
     fetch: options.fetch, // Pass fetch option to worker template
   })
 
-  // Block outbound network requests at Miniflare level when fetch: null
-  // This complements the globalThis.fetch override in the worker template
-  const blockNetwork = options.fetch === null
+  // Determine outbound service configuration based on fetch option
+  const blockNetwork = shouldBlockNetwork(options)
+  const allowlistDomains = getAllowlistDomains(options)
+
+  // Build outboundService based on mode:
+  // - block: throw error for all requests
+  // - allowlist: check domain against allowlist
+  // - allow (default): no outboundService (allow all)
+  type OutboundServiceFn = (() => never) | ((request: Request) => Response | Promise<Response>)
+  let outboundService: OutboundServiceFn | undefined
+  if (blockNetwork) {
+    outboundService = () => {
+      throw new Error('Network access blocked: fetch is disabled in this sandbox')
+    }
+  } else if (allowlistDomains) {
+    outboundService = (request: Request) => {
+      const url = request.url
+      if (!isDomainAllowed(url, allowlistDomains)) {
+        const hostname = new URL(url).hostname
+        throw new Error(`Network access blocked: domain not in allowlist. Attempted: ${hostname}`)
+      }
+      // Allow the request by returning a fetched response
+      return fetch(request)
+    }
+  }
 
   const mf = new Miniflare({
     modules: true,
     script: workerCode,
     compatibilityDate: '2026-01-01',
-    // Block all outbound fetch/connect when network is disabled
-    ...(blockNetwork && {
-      outboundService: () => {
-        throw new Error('Network access blocked: fetch is disabled in this sandbox')
-      },
-    }),
+    // Configure outbound service based on fetch mode
+    ...(outboundService && { outboundService }),
   })
 
   try {