@amodalai/amodal 0.3.35 → 0.3.37

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@amodalai/amodal",
-  "version": "0.3.35",
+  "version": "0.3.37",
   "description": "Amodal CLI",
   "license": "MIT",
   "repository": {
@@ -26,12 +26,12 @@
     "react": "^19.2.4",
     "yargs": "^17.7.2",
     "zod": "^4.3.6",
-    "@amodalai/types": "0.3.35",
-    "@amodalai/core": "0.3.35",
-    "@amodalai/db": "0.3.35",
-    "@amodalai/runtime": "0.3.35",
-    "@amodalai/studio": "0.3.35",
-    "@amodalai/runtime-app": "0.3.35"
+    "@amodalai/types": "0.3.37",
+    "@amodalai/core": "0.3.37",
+    "@amodalai/db": "0.3.37",
+    "@amodalai/runtime": "0.3.37",
+    "@amodalai/studio": "0.3.37",
+    "@amodalai/runtime-app": "0.3.37"
   },
   "devDependencies": {
     "@types/node": "^20.11.24",

package/src/e2e-subprocess.test.ts

@@ -87,6 +87,21 @@ describe.skipIf(!!skipReason)('subprocess smoke tests', () => {
     mkdirSync(knowledgeDir, {recursive: true});
     writeFileSync(resolve(knowledgeDir, 'test-doc.md'), '# Test\n\nSENTINEL_FILE_TOOLS_9923\n');
 
+    // Create a test eval for eval/arena tests
+    const evalsDir = resolve(agentDir, 'evals');
+    mkdirSync(evalsDir, {recursive: true});
+    writeFileSync(resolve(evalsDir, 'math-check.md'), [
+      '# Eval: Math Check',
+      '',
+      '## Query',
+      'What is 2 + 2? Reply with just the number.',
+      '',
+      '## Assertions',
+      '- Should contain the number 4',
+      '- contains: 4',
+      '- Should NOT contain the word elephant',
+    ].join('\n'));
+
     const cliEntry = resolve(__dir, '../dist/src/main.js');
     if (!existsSync(cliEntry)) {
       throw new Error(`CLI not built — run pnpm --filter @amodalai/amodal run build first`);
@@ -188,4 +203,51 @@ describe.skipIf(!!skipReason)('subprocess smoke tests', () => {
     expect(text).toContain('tool_call_start');
     expect(text).toContain('SENTINEL_FILE_TOOLS_9923');
   }, 45_000);
+
+  it('runtime runs eval and returns results with assertions', async () => {
+    const res = await fetch(`http://localhost:${RUNTIME_PORT}/api/evals/run`, {
+      method: 'POST',
+      headers: {'Content-Type': 'application/json'},
+      body: JSON.stringify({evalNames: ['math-check']}),
+      signal: AbortSignal.timeout(30_000),
+    });
+    expect(res.status).toBe(200);
+    const text = await res.text();
+    // Parse the eval_complete event
+    const evalLine = text.split('\n').find((l) => l.includes('eval_complete'));
+    expect(evalLine).toBeDefined();
+    const event = JSON.parse(evalLine!.replace('data: ', '')) as Record<string, unknown>;
+    expect(event['evalName']).toBe('math-check');
+    expect(typeof event['passed']).toBe('boolean');
+    const result = event['result'] as Record<string, unknown>;
+    expect(result['response']).toBeDefined();
+    expect(Array.isArray(result['assertions'])).toBe(true);
+    // Verify deterministic assertions were evaluated (3 total: 1 LLM-judged + 1 deterministic + 1 negated)
+    const assertions = result['assertions'] as Array<Record<string, unknown>>;
+    expect(assertions.length).toBe(3);
+    // The deterministic "contains: 4" assertion should have a reason mentioning "contains"
+    const containsAssertion = assertions.find((a) => a['text'] === 'contains: 4');
+    expect(containsAssertion).toBeDefined();
+    expect(containsAssertion!['reason']).toBeDefined();
+    expect(result['durationMs']).toBeDefined();
+  }, 45_000);
+
+  it('runtime runs arena eval with specified model', async () => {
+    const res = await fetch(`http://localhost:${RUNTIME_PORT}/api/evals/run`, {
+      method: 'POST',
+      headers: {'Content-Type': 'application/json'},
+      body: JSON.stringify({
+        evalNames: ['math-check'],
+        model: {provider: 'google', model: 'gemini-2.0-flash'},
+      }),
+      signal: AbortSignal.timeout(30_000),
+    });
+    expect(res.status).toBe(200);
+    const text = await res.text();
+    const evalLine = text.split('\n').find((l) => l.includes('eval_complete'));
+    expect(evalLine).toBeDefined();
+    const event = JSON.parse(evalLine!.replace('data: ', '')) as Record<string, unknown>;
+    expect(event['evalName']).toBe('math-check');
+    expect(typeof event['passed']).toBe('boolean');
+  }, 45_000);
 });