@amodalai/runtime 0.1.26 → 0.2.0

package/dist/src/__fixtures__/README.md

@@ -0,0 +1,84 @@
+# Smoke Tests
+
+End-to-end integration tests that start a real `amodal dev` server with mock REST and MCP backends, then run assertions against the chat API using a live LLM.
+
+## Setup
+
+Create `.env.test` at the **repo root** with your API key:
+
+```
+ANTHROPIC_API_KEY=sk-ant-...
+```
+
+This file is gitignored. Tests skip automatically if the key isn't set.
+
+## Running
+
+```bash
+# All smoke tests (~90s, ~23 LLM calls)
+pnpm --filter @amodalai/runtime run test:smoke
+
+# Single test by name (~5s, 1 LLM call)
+pnpm --filter @amodalai/runtime run test:smoke -t "dispatch_task tool is available"
+
+# Multiple tests by pattern
+pnpm --filter @amodalai/runtime run test:smoke -t "dispatch_task|persists data"
+
+# Regex patterns work too
+pnpm --filter @amodalai/runtime run test:smoke -t "store.*persist"
+```
+
+> **No `--` before `-t`.** pnpm passes trailing args directly to vitest. Adding `--` breaks the flag parsing.
+
+The `-t` flag filters which `it()` blocks execute. The server still starts once (shared `beforeAll`), but only matched tests make LLM calls — so filtering saves both time and money.
+
+## What's tested
+
+| #   | Test               | What it verifies                                       |
+| --- | ------------------ | ------------------------------------------------------ |
+| 1   | health endpoint    | Server starts and responds                             |
+| 2   | config endpoint    | Agent config loads from repo                           |
+| 3   | system prompt (G9) | Prompt includes connections, skills, knowledge, stores |
+| 4   | chat streaming     | Init, text_delta, done events with usage               |
+| 5   | session resume     | Multi-turn context preserved across messages           |
+| 6   | store tool call    | Model calls query_store                                |
+| 7   | connection request | Request tool hits mock-api                             |
+| 8   | tool error status  | Failed tool calls report status: error                 |
+| 9   | eval run           | Eval suite executes and scores                         |
+| 10  | admin chat         | Admin agent reads repo files                           |
+| 11  | write intent (G8)  | POST with intent "read" rejected                       |
+| 12  | store persistence  | Write + query in separate sessions                     |
+| 13  | session isolation  | Concurrent sessions don't share context                |
+| 14  | automation API     | Automation endpoint responds                           |
+| 15  | multi-turn tools   | Tool call → reasoning about result                     |
+| 16  | evals list         | Eval suites listed from repo                           |
+| 17  | inspect endpoint   | Connection status visible                              |
+| 18  | MCP tool call      | MCP tool executes via stdio transport                  |
+| 19  | custom tool        | echo_tool with ctx.request + ctx.store                 |
+| 20  | stop_execution     | Tool is available to model                             |
+| 21  | done usage (G2)    | Done event always has token counts                     |
+| 22  | dispatch_task      | Child agent runs with tool subset                      |
+| 23  | dispatch available | dispatch_task tool in model's tool list                |
+
+## Architecture
+
+```
+smoke.test.ts          — test suite (vitest)
+smoke-agent/           — self-contained agent repo (.amodal/ config)
+  amodal.json          — agent config (name, model)
+  connections/         — mock-api (REST) + mock-mcp (MCP stdio)
+  stores/              — test-items store schema
+  skills/              — test-skill
+  knowledge/           — reference doc with tool list
+  tools/               — echo-tool custom handler
+  evals/               — basic-eval suite
+  automations/         — test-auto definition
+smoke-rest-server.mjs  — Express mock returning test data on :9901
+smoke-mcp-server.mjs   — MCP stdio server with search/lookup/count tools
+```
+
+The test `beforeAll` starts the mock REST server and `createLocalServer()` programmatically on port 9900. Tests call `POST /chat` and parse SSE events from the response.
+
+## LLM non-determinism
+
+Some tests depend on the model calling specific tools. When the model chooses not to (despite explicit instruction), the test skips gracefully with a console warning rather than failing. This prevents flaky CI from LLM variability while still catching real code bugs.

package/dist/src/__fixtures__/smoke-agent/amodal.json

@@ -0,0 +1,11 @@
+{
+  "name": "smoke-test-agent",
+  "version": "1.0.0",
+  "description": "Self-contained agent for smoke testing the runtime",
+  "models": {
+    "main": {
+      "provider": "anthropic",
+      "model": "claude-sonnet-4-20250514"
+    }
+  }
+}

package/dist/src/__fixtures__/smoke-agent/automations/test-auto.md

@@ -0,0 +1,5 @@
+---
+name: test-automation
+trigger: manual
+prompt: "Say 'automation ran successfully' and nothing else."
+---

package/dist/src/__fixtures__/smoke-agent/connections/mock-api/access.json

@@ -0,0 +1,11 @@
+{
+  "endpoints": {
+    "GET /items": {
+      "returns": ["id", "name", "status"]
+    },
+    "POST /items": {
+      "returns": ["id"],
+      "confirm": true
+    }
+  }
+}

package/dist/src/__fixtures__/smoke-agent/connections/mock-api/spec.json

@@ -0,0 +1,4 @@
+{
+  "protocol": "rest",
+  "baseUrl": "http://localhost:9901"
+}

package/dist/src/__fixtures__/smoke-agent/connections/mock-api/surface.md

@@ -0,0 +1,9 @@
+## Included
+
+### GET /items
+
+List all items. Returns `[{id, name, status}]`.
+
+### POST /items
+
+Create a new item. Body: `{name, status}`. Returns `{id, name, status}`.

package/dist/src/__fixtures__/smoke-agent/connections/mock-mcp/access.json

@@ -0,0 +1,3 @@
+{
+  "endpoints": {}
+}

package/dist/src/__fixtures__/smoke-agent/connections/mock-mcp/spec.json

@@ -0,0 +1,8 @@
+{
+  "protocol": "mcp",
+  "transport": "stdio",
+  "command": "node",
+  "args": [
+    "/Users/bob/code/amodal-smoke/packages/runtime/src/__fixtures__/smoke-mcp-server.mjs"
+  ]
+}

package/dist/src/__fixtures__/smoke-agent/evals/basic-eval.md

@@ -0,0 +1,12 @@
+# Eval: Basic Smoke
+
+Test that the agent can respond to a simple question.
+
+## Query
+
+"What is 2 + 2?"
+
+## Assertions
+
+- Should contain the number 4
+- Should NOT contain the word "maybe"

package/dist/src/__fixtures__/smoke-agent/knowledge/test-knowledge.md

@@ -0,0 +1,3 @@
+# Smoke Test Reference
+
+This is reference knowledge for the smoke test agent. It verifies that knowledge documents are included in the system prompt.

package/dist/src/__fixtures__/smoke-agent/skills/test-skill/SKILL.md

@@ -0,0 +1,11 @@
+# Skill: test-skill
+
+Trigger: When asked to run the smoke test
+
+## Test Skill
+
+Query the mock-api connection for items and store the results.
+
+1. Use the request tool to GET /items from mock-api
+2. Store each item in the test-items store
+3. Report what you found

package/dist/src/__fixtures__/smoke-agent/stores/test-items.json

@@ -0,0 +1,11 @@
+{
+  "entity": {
+    "name": "TestItem",
+    "key": "{item_id}",
+    "schema": {
+      "item_id": { "type": "string" },
+      "name": { "type": "string" },
+      "status": { "type": "enum", "values": ["active", "archived"] }
+    }
+  }
+}

package/dist/src/__fixtures__/smoke-agent/tools/echo_tool/handler.d.ts

@@ -0,0 +1,18 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+export default function (params: {
+    message: string;
+}, ctx: {
+    request: (conn: string, endpoint: string, params?: unknown) => Promise<unknown>;
+    store: (name: string, payload: Record<string, unknown>) => Promise<{
+        key: string;
+    }>;
+    log: (msg: string) => void;
+}): Promise<{
+    echoed: string;
+    itemCount: number;
+    storedKey: string;
+}>;

package/dist/src/__fixtures__/smoke-agent/tools/echo_tool/handler.js

@@ -0,0 +1,22 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+export default async function (params, ctx) {
+    ctx.log(`echo_tool called with: ${params.message}`);
+    // Test ctx.request() — fetch from mock-api
+    const items = await ctx.request('mock-api', '/items');
+    // Test ctx.store() — write to test-items store
+    const storeResult = await ctx.store('test-items', {
+        item_id: 'echo-test',
+        name: params.message,
+        status: 'active',
+    });
+    return {
+        echoed: params.message,
+        itemCount: Array.isArray(items) ? items.length : 0,
+        storedKey: storeResult.key,
+    };
+}
+//# sourceMappingURL=handler.js.map

package/dist/src/__fixtures__/smoke-agent/tools/echo_tool/handler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"handler.js","sourceRoot":"","sources":["../../../../../../src/__fixtures__/smoke-agent/tools/echo_tool/handler.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,CAAC,OAAO,CAAC,KAAK,WAClB,MAAyB,EACzB,GAIC;IAED,GAAG,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAEpD,2CAA2C;IAC3C,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEtD,+CAA+C;IAC/C,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE;QAChD,OAAO,EAAE,WAAW;QACpB,IAAI,EAAE,MAAM,CAAC,OAAO;QACpB,MAAM,EAAE,QAAQ;KACjB,CAAC,CAAC;IAEH,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,OAAO;QACtB,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAClD,SAAS,EAAE,WAAW,CAAC,GAAG;KAC3B,CAAC;AACJ,CAAC"}

package/dist/src/__fixtures__/smoke-agent/tools/echo_tool/tool.json

@@ -0,0 +1,17 @@
+{
+  "name": "echo_tool",
+  "description": "Echo a message, fetch from mock API, and write to store. Used for smoke testing custom tool wiring.",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "message": {
+        "type": "string",
+        "description": "Message to echo back"
+      }
+    },
+    "required": ["message"]
+  },
+  "confirm": true,
+  "timeout": 10000,
+  "env": []
+}