ai-evaluate 0.1.0

package/.turbo/turbo-build.log

@@ -0,0 +1,5 @@
+
+
+> ai-sandbox@0.1.0 build /Users/nathanclevenger/projects/mdx.org.ai/primitives/packages/ai-sandbox
+> tsc -p tsconfig.json
+

package/.turbo/turbo-test.log

@@ -0,0 +1,51 @@
+
+> ai-sandbox@0.0.1 test /Users/nathanclevenger/projects/mdx.org.ai/primitives/packages/ai-sandbox
+> vitest
+
+
+ DEV  v2.1.9 /Users/nathanclevenger/projects/mdx.org.ai/primitives/packages/ai-sandbox
+
+ ✓ test/worker-template.test.ts (65 tests) 8ms
+stdout | test/evaluate.test.ts > evaluate > script execution > captures console output
+hello
+
+stderr | test/evaluate.test.ts > evaluate > script execution > captures console output
+warning
+error
+
+stdout | test/index.test.ts > types > LogEntry has correct shape
+test
+
+stderr | test/evaluate.test.ts > evaluate > script execution > handles script errors
+Script error: test error
+
+ ✓ test/index.test.ts (8 tests) 421ms
+stderr | test/evaluate-extended.test.ts > evaluate - extended tests > module errors > captures module runtime errors
+Module error: module error
+
+stdout | test/evaluate-extended.test.ts > evaluate - extended tests > console methods > captures console.info
+info message
+
+stdout | test/evaluate-extended.test.ts > evaluate - extended tests > console methods > captures console.debug
+debug message
+
+stdout | test/evaluate-extended.test.ts > evaluate - extended tests > console methods > stringifies objects in console output
+{ a: 1 }
+
+stdout | test/evaluate-extended.test.ts > evaluate - extended tests > console methods > joins multiple console arguments
+a b c
+
+stderr | test/evaluate-extended.test.ts > evaluate - extended tests > async scripts > handles Promise rejection in script
+Script error: async error
+
+ ✓ test/evaluate.test.ts (18 tests) 765ms
+ ✓ test/evaluate-extended.test.ts (40 tests) 1352ms
+
+ Test Files  4 passed (4)
+      Tests  131 passed (131)
+   Start at  14:14:37
+   Duration  1.87s (transform 186ms, setup 0ms, collect 405ms, tests 2.55s, environment 0ms, prepare 339ms)
+
+ PASS  Waiting for file changes...
+       press h to show help, press q to quit
+ ELIFECYCLE  Test failed. See above for more details.

package/README.md

@@ -0,0 +1,420 @@
+# ai-evaluate
+
+Secure code execution in sandboxed environments. Run untrusted code safely using Cloudflare Workers or Miniflare.
+
+## Installation
+
+```bash
+pnpm add ai-evaluate
+```
+
+## Quick Start
+
+```typescript
+import { evaluate } from 'ai-evaluate'
+
+// Run a simple script
+const result = await evaluate({
+  script: '1 + 1'
+})
+// { success: true, value: 2, logs: [], duration: 5 }
+
+// With a module and tests
+const result = await evaluate({
+  module: `
+    export const add = (a, b) => a + b
+    export const multiply = (a, b) => a * b
+  `,
+  tests: `
+    describe('math', () => {
+      it('adds numbers', () => {
+        expect(add(2, 3)).toBe(5);
+      })
+      it('multiplies numbers', () => {
+        expect(multiply(2, 3)).toBe(6);
+      })
+    })
+  `,
+  script: 'add(10, 20)'
+})
+```
+
+## Features
+
+- **Secure isolation** - Code runs in a sandboxed V8 isolate
+- **Vitest-compatible tests** - `describe`, `it`, `expect` in global scope
+- **Module exports** - Define modules and use exports in scripts/tests
+- **Cloudflare Workers** - Uses worker_loaders in production
+- **Miniflare** - Uses Miniflare for local development and Node.js
+- **Network isolation** - External network access blocked by default
+
+## API
+
+### evaluate(options)
+
+Execute code in a sandboxed environment.
+
+```typescript
+interface EvaluateOptions {
+  /** Module code with exports */
+  module?: string
+  /** Test code using vitest-style API */
+  tests?: string
+  /** Script code to run (module exports in scope) */
+  script?: string
+  /** Timeout in milliseconds (default: 5000) */
+  timeout?: number
+  /** Environment variables */
+  env?: Record<string, string>
+}
+
+interface EvaluateResult {
+  /** Whether execution succeeded */
+  success: boolean
+  /** Return value from script */
+  value?: unknown
+  /** Console output */
+  logs: LogEntry[]
+  /** Test results (if tests provided) */
+  testResults?: TestResults
+  /** Error message if failed */
+  error?: string
+  /** Execution time in ms */
+  duration: number
+}
+```
+
+### createEvaluator(env)
+
+Create an evaluate function bound to a specific environment. Useful for Cloudflare Workers.
+
+```typescript
+import { createEvaluator } from 'ai-evaluate'
+
+export default {
+  async fetch(request, env) {
+    const sandbox = createEvaluator(env)
+    const result = await sandbox({
+      script: '1 + 1'
+    })
+    return Response.json(result)
+  }
+}
+```
+
+## Usage Patterns
+
+### Simple Script Execution
+
+```typescript
+const result = await evaluate({
+  script: `
+    const x = 10;
+    const y = 20;
+    return x + y;
+  `
+})
+// result.value === 30
+```
+
+### Module with Exports
+
+```typescript
+const result = await evaluate({
+  module: `
+    exports.greet = (name) => \`Hello, \${name}!\`;
+    exports.sum = (...nums) => nums.reduce((a, b) => a + b, 0);
+  `,
+  script: `
+    console.log(greet('World'));
+    return sum(1, 2, 3, 4, 5);
+  `
+})
+// result.value === 15
+// result.logs[0].message === 'Hello, World!'
+```
+
+### Running Tests
+
+```typescript
+const result = await evaluate({
+  module: `
+    exports.isPrime = (n) => {
+      if (n < 2) return false;
+      for (let i = 2; i <= Math.sqrt(n); i++) {
+        if (n % i === 0) return false;
+      }
+      return true;
+    };
+  `,
+  tests: `
+    describe('isPrime', () => {
+      it('returns false for numbers less than 2', () => {
+        expect(isPrime(0)).toBe(false);
+        expect(isPrime(1)).toBe(false);
+      });
+
+      it('returns true for prime numbers', () => {
+        expect(isPrime(2)).toBe(true);
+        expect(isPrime(3)).toBe(true);
+        expect(isPrime(17)).toBe(true);
+      });
+
+      it('returns false for composite numbers', () => {
+        expect(isPrime(4)).toBe(false);
+        expect(isPrime(9)).toBe(false);
+        expect(isPrime(100)).toBe(false);
+      });
+    });
+  `
+})
+
+console.log(result.testResults)
+// {
+//   total: 3,
+//   passed: 3,
+//   failed: 0,
+//   skipped: 0,
+//   tests: [...]
+// }
+```
+
+## Test Framework
+
+The sandbox provides a vitest-compatible test API with async support.
+
+### describe / it / test
+
+```typescript
+describe('group name', () => {
+  it('test name', () => {
+    // test code
+  });
+
+  test('another test', () => {
+    // test code
+  });
+
+  it.skip('skipped test', () => {
+    // won't run
+  });
+
+  it.only('only this test', () => {
+    // when .only is used, only these tests run
+  });
+});
+```
+
+### Async Tests
+
+```typescript
+describe('async operations', () => {
+  it('supports async/await', async () => {
+    const result = await someAsyncFunction();
+    expect(result).toBe('expected');
+  });
+
+  it('supports promises', () => {
+    return fetchData().then(data => {
+      expect(data).toBeDefined();
+    });
+  });
+});
+```
+
+### Hooks
+
+```typescript
+describe('with setup', () => {
+  let data;
+
+  beforeEach(() => {
+    data = { count: 0 };
+  });
+
+  afterEach(() => {
+    data = null;
+  });
+
+  it('uses setup data', () => {
+    data.count++;
+    expect(data.count).toBe(1);
+  });
+});
+```
+
+### expect matchers
+
+```typescript
+// Equality
+expect(value).toBe(expected)           // Strict equality (===)
+expect(value).toEqual(expected)        // Deep equality
+expect(value).toStrictEqual(expected)  // Strict deep equality
+
+// Truthiness
+expect(value).toBeTruthy()             // Truthy check
+expect(value).toBeFalsy()              // Falsy check
+expect(value).toBeNull()               // null check
+expect(value).toBeUndefined()          // undefined check
+expect(value).toBeDefined()            // not undefined
+expect(value).toBeNaN()                // NaN check
+
+// Numbers
+expect(value).toBeGreaterThan(n)       // > comparison
+expect(value).toBeLessThan(n)          // < comparison
+expect(value).toBeGreaterThanOrEqual(n)// >= comparison
+expect(value).toBeLessThanOrEqual(n)   // <= comparison
+expect(value).toBeCloseTo(n, digits)   // Floating point comparison
+
+// Strings
+expect(value).toMatch(/pattern/)       // Regex match
+expect(value).toMatch('substring')     // Contains substring
+
+// Arrays & Strings
+expect(value).toContain(item)          // Array/string contains
+expect(value).toContainEqual(item)     // Array contains (deep equality)
+expect(value).toHaveLength(n)          // Length check
+
+// Objects
+expect(value).toHaveProperty('path')   // Has property
+expect(value).toHaveProperty('path', v)// Has property with value
+expect(value).toMatchObject(partial)   // Partial object match
+
+// Types
+expect(value).toBeInstanceOf(Class)    // instanceof check
+expect(value).toBeTypeOf('string')     // typeof check
+
+// Errors
+expect(fn).toThrow()                   // Throws any error
+expect(fn).toThrow('message')          // Throws with message
+expect(fn).toThrow(/pattern/)          // Throws matching pattern
+expect(fn).toThrow(ErrorClass)         // Throws specific error type
+
+// Negated matchers
+expect(value).not.toBe(expected)
+expect(value).not.toEqual(expected)
+expect(value).not.toContain(item)
+expect(fn).not.toThrow()
+
+// Promise matchers
+await expect(promise).resolves.toBe(value)
+await expect(promise).rejects.toThrow('error')
+```
+
+## Cloudflare Workers Setup
+
+To use in Cloudflare Workers with worker_loaders:
+
+### wrangler.toml
+
+```toml
+name = "my-worker"
+main = "src/index.ts"
+
+[[worker_loaders]]
+binding = "LOADER"
+```
+
+### Worker Code
+
+```typescript
+import { createEvaluator } from 'ai-evaluate'
+
+export interface Env {
+  LOADER: unknown
+}
+
+export default {
+  async fetch(request: Request, env: Env): Promise<Response> {
+    const sandbox = createEvaluator(env)
+
+    const { code, tests } = await request.json()
+
+    const result = await sandbox({
+      module: code,
+      tests: tests
+    })
+
+    return Response.json(result)
+  }
+}
+```
+
+## Node.js / Development
+
+In Node.js or during development, the evaluate function automatically uses Miniflare:
+
+```typescript
+import { evaluate } from 'ai-evaluate'
+
+// Miniflare is used automatically when LOADER binding is not present
+const result = await evaluate({
+  script: 'return "Hello from Node!"'
+})
+```
+
+Make sure `miniflare` is installed:
+
+```bash
+pnpm add miniflare
+```
+
+## Security
+
+The sandbox provides several security features:
+
+1. **V8 Isolate** - Code runs in an isolated V8 context
+2. **No Network** - External network access is blocked (`globalOutbound: null`)
+3. **No File System** - No access to the file system
+4. **Memory Limits** - Standard Worker memory limits apply
+5. **CPU Limits** - Execution time is limited
+
+## Example: Code Evaluation API
+
+```typescript
+import { evaluate } from 'ai-evaluate'
+import { Hono } from 'hono'
+
+const app = new Hono()
+
+app.post('/evaluate', async (c) => {
+  const { module, tests, script } = await c.req.json()
+
+  const result = await evaluate({
+    module,
+    tests,
+    script,
+    timeout: 5000
+  })
+
+  return c.json(result)
+})
+
+export default app
+```
+
+## Types
+
+```typescript
+interface LogEntry {
+  level: 'log' | 'warn' | 'error' | 'info' | 'debug'
+  message: string
+  timestamp: number
+}
+
+interface TestResults {
+  total: number
+  passed: number
+  failed: number
+  skipped: number
+  tests: TestResult[]
+  duration: number
+}
+
+interface TestResult {
+  name: string
+  passed: boolean
+  error?: string
+  duration: number
+}
+```

package/coverage/base.css

@@ -0,0 +1,224 @@
+body, html {
+  margin:0; padding: 0;
+  height: 100%;
+}
+body {
+    font-family: Helvetica Neue, Helvetica, Arial;
+    font-size: 14px;
+    color:#333;
+}
+.small { font-size: 12px; }
+*, *:after, *:before {
+  -webkit-box-sizing:border-box;
+     -moz-box-sizing:border-box;
+          box-sizing:border-box;
+  }
+h1 { font-size: 20px; margin: 0;}
+h2 { font-size: 14px; }
+pre {
+    font: 12px/1.4 Consolas, "Liberation Mono", Menlo, Courier, monospace;
+    margin: 0;
+    padding: 0;
+    -moz-tab-size: 2;
+    -o-tab-size:  2;
+    tab-size: 2;
+}
+a { color:#0074D9; text-decoration:none; }
+a:hover { text-decoration:underline; }
+.strong { font-weight: bold; }
+.space-top1 { padding: 10px 0 0 0; }
+.pad2y { padding: 20px 0; }
+.pad1y { padding: 10px 0; }
+.pad2x { padding: 0 20px; }
+.pad2 { padding: 20px; }
+.pad1 { padding: 10px; }
+.space-left2 { padding-left:55px; }
+.space-right2 { padding-right:20px; }
+.center { text-align:center; }
+.clearfix { display:block; }
+.clearfix:after {
+  content:'';
+  display:block;
+  height:0;
+  clear:both;
+  visibility:hidden;
+  }
+.fl { float: left; }
+@media only screen and (max-width:640px) {
+  .col3 { width:100%; max-width:100%; }
+  .hide-mobile { display:none!important; }
+}
+
+.quiet {
+  color: #7f7f7f;
+  color: rgba(0,0,0,0.5);
+}
+.quiet a { opacity: 0.7; }
+
+.fraction {
+  font-family: Consolas, 'Liberation Mono', Menlo, Courier, monospace;
+  font-size: 10px;
+  color: #555;
+  background: #E8E8E8;
+  padding: 4px 5px;
+  border-radius: 3px;
+  vertical-align: middle;
+}
+
+div.path a:link, div.path a:visited { color: #333; }
+table.coverage {
+  border-collapse: collapse;
+  margin: 10px 0 0 0;
+  padding: 0;
+}
+
+table.coverage td {
+  margin: 0;
+  padding: 0;
+  vertical-align: top;
+}
+table.coverage td.line-count {
+    text-align: right;
+    padding: 0 5px 0 20px;
+}
+table.coverage td.line-coverage {
+    text-align: right;
+    padding-right: 10px;
+    min-width:20px;
+}
+
+table.coverage td span.cline-any {
+    display: inline-block;
+    padding: 0 5px;
+    width: 100%;
+}
+.missing-if-branch {
+    display: inline-block;
+    margin-right: 5px;
+    border-radius: 3px;
+    position: relative;
+    padding: 0 4px;
+    background: #333;
+    color: yellow;
+}
+
+.skip-if-branch {
+    display: none;
+    margin-right: 10px;
+    position: relative;
+    padding: 0 4px;
+    background: #ccc;
+    color: white;
+}
+.missing-if-branch .typ, .skip-if-branch .typ {
+    color: inherit !important;
+}
+.coverage-summary {
+  border-collapse: collapse;
+  width: 100%;
+}
+.coverage-summary tr { border-bottom: 1px solid #bbb; }
+.keyline-all { border: 1px solid #ddd; }
+.coverage-summary td, .coverage-summary th { padding: 10px; }
+.coverage-summary tbody { border: 1px solid #bbb; }
+.coverage-summary td { border-right: 1px solid #bbb; }
+.coverage-summary td:last-child { border-right: none; }
+.coverage-summary th {
+  text-align: left;
+  font-weight: normal;
+  white-space: nowrap;
+}
+.coverage-summary th.file { border-right: none !important; }
+.coverage-summary th.pct { }
+.coverage-summary th.pic,
+.coverage-summary th.abs,
+.coverage-summary td.pct,
+.coverage-summary td.abs { text-align: right; }
+.coverage-summary td.file { white-space: nowrap;  }
+.coverage-summary td.pic { min-width: 120px !important;  }
+.coverage-summary tfoot td { }
+
+.coverage-summary .sorter {
+    height: 10px;
+    width: 7px;
+    display: inline-block;
+    margin-left: 0.5em;
+    background: url(sort-arrow-sprite.png) no-repeat scroll 0 0 transparent;
+}
+.coverage-summary .sorted .sorter {
+    background-position: 0 -20px;
+}
+.coverage-summary .sorted-desc .sorter {
+    background-position: 0 -10px;
+}
+.status-line {  height: 10px; }
+/* yellow */
+.cbranch-no { background: yellow !important; color: #111; }
+/* dark red */
+.red.solid, .status-line.low, .low .cover-fill { background:#C21F39 }
+.low .chart { border:1px solid #C21F39 }
+.highlighted,
+.highlighted .cstat-no, .highlighted .fstat-no, .highlighted .cbranch-no{
+  background: #C21F39 !important;
+}
+/* medium red */
+.cstat-no, .fstat-no, .cbranch-no, .cbranch-no { background:#F6C6CE }
+/* light red */
+.low, .cline-no { background:#FCE1E5 }
+/* light green */
+.high, .cline-yes { background:rgb(230,245,208) }
+/* medium green */
+.cstat-yes { background:rgb(161,215,106) }
+/* dark green */
+.status-line.high, .high .cover-fill { background:rgb(77,146,33) }
+.high .chart { border:1px solid rgb(77,146,33) }
+/* dark yellow (gold) */
+.status-line.medium, .medium .cover-fill { background: #f9cd0b; }
+.medium .chart { border:1px solid #f9cd0b; }
+/* light yellow */
+.medium { background: #fff4c2; }
+
+.cstat-skip { background: #ddd; color: #111; }
+.fstat-skip { background: #ddd; color: #111 !important; }
+.cbranch-skip { background: #ddd !important; color: #111; }
+
+span.cline-neutral { background: #eaeaea; }
+
+.coverage-summary td.empty {
+    opacity: .5;
+    padding-top: 4px;
+    padding-bottom: 4px;
+    line-height: 1;
+    color: #888;
+}
+
+.cover-fill, .cover-empty {
+  display:inline-block;
+  height: 12px;
+}
+.chart {
+  line-height: 0;
+}
+.cover-empty {
+    background: white;
+}
+.cover-full {
+    border-right: none !important;
+}
+pre.prettyprint {
+    border: none !important;
+    padding: 0 !important;
+    margin: 0 !important;
+}
+.com { color: #999 !important; }
+.ignore-none { color: #999; font-weight: normal; }
+
+.wrapper {
+  min-height: 100%;
+  height: auto !important;
+  height: 100%;
+  margin: 0 auto -48px;
+}
+.footer, .push {
+  height: 48px;
+}