@amodalai/runtime 0.1.26 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/__fixtures__/README.md +84 -0
- package/dist/src/__fixtures__/smoke-agent/amodal.json +11 -0
- package/dist/src/__fixtures__/smoke-agent/automations/test-auto.md +5 -0
- package/dist/src/__fixtures__/smoke-agent/connections/mock-api/access.json +11 -0
- package/dist/src/__fixtures__/smoke-agent/connections/mock-api/spec.json +4 -0
- package/dist/src/__fixtures__/smoke-agent/connections/mock-api/surface.md +9 -0
- package/dist/src/__fixtures__/smoke-agent/connections/mock-mcp/access.json +3 -0
- package/dist/src/__fixtures__/smoke-agent/connections/mock-mcp/spec.json +8 -0
- package/dist/src/__fixtures__/smoke-agent/evals/basic-eval.md +12 -0
- package/dist/src/__fixtures__/smoke-agent/knowledge/test-knowledge.md +3 -0
- package/dist/src/__fixtures__/smoke-agent/skills/test-skill/SKILL.md +11 -0
- package/dist/src/__fixtures__/smoke-agent/stores/test-items.json +11 -0
- package/dist/src/__fixtures__/smoke-agent/tools/echo_tool/handler.d.ts +18 -0
- package/dist/src/__fixtures__/smoke-agent/tools/echo_tool/handler.js +22 -0
- package/dist/src/__fixtures__/smoke-agent/tools/echo_tool/handler.js.map +1 -0
- package/dist/src/__fixtures__/smoke-agent/tools/echo_tool/tool.json +17 -0
- package/dist/src/__fixtures__/smoke.test.js +718 -0
- package/dist/src/__fixtures__/smoke.test.js.map +1 -0
- package/dist/src/__tests__/test-providers.d.ts +40 -0
- package/dist/src/__tests__/test-providers.js +61 -0
- package/dist/src/__tests__/test-providers.js.map +1 -0
- package/dist/src/agent/local-server.d.ts +3 -3
- package/dist/src/agent/local-server.js +213 -122
- package/dist/src/agent/local-server.js.map +1 -1
- package/dist/src/agent/loop-types.d.ts +175 -0
- package/dist/src/agent/loop-types.js +20 -0
- package/dist/src/agent/loop-types.js.map +1 -0
- package/dist/src/agent/loop.d.ts +31 -0
- package/dist/src/agent/loop.js +139 -0
- package/dist/src/agent/loop.js.map +1 -0
- package/dist/src/agent/loop.test.js +1030 -0
- package/dist/src/agent/loop.test.js.map +1 -0
- package/dist/src/agent/mcp-config.d.ts +28 -0
- package/dist/src/agent/mcp-config.js +57 -0
- package/dist/src/agent/mcp-config.js.map +1 -0
- package/dist/src/agent/page-builder.js +6 -1
- package/dist/src/agent/page-builder.js.map +1 -1
- package/dist/src/agent/proactive/proactive-runner.d.ts +24 -8
- package/dist/src/agent/proactive/proactive-runner.js +30 -32
- package/dist/src/agent/proactive/proactive-runner.js.map +1 -1
- package/dist/src/agent/proactive/proactive-runner.test.d.ts +1 -1
- package/dist/src/agent/proactive/proactive-runner.test.js +75 -87
- package/dist/src/agent/proactive/proactive-runner.test.js.map +1 -1
- package/dist/src/agent/routes/admin-chat.d.ts +15 -3
- package/dist/src/agent/routes/admin-chat.js +63 -18
- package/dist/src/agent/routes/admin-chat.js.map +1 -1
- package/dist/src/agent/routes/automations.js +5 -6
- package/dist/src/agent/routes/automations.js.map +1 -1
- package/dist/src/agent/routes/evals.d.ts +3 -2
- package/dist/src/agent/routes/evals.js +25 -12
- package/dist/src/agent/routes/evals.js.map +1 -1
- package/dist/src/agent/routes/files.js +7 -9
- package/dist/src/agent/routes/files.js.map +1 -1
- package/dist/src/agent/routes/inspect.d.ts +6 -2
- package/dist/src/agent/routes/inspect.js +31 -17
- package/dist/src/agent/routes/inspect.js.map +1 -1
- package/dist/src/agent/routes/inspect.test.js +18 -42
- package/dist/src/agent/routes/inspect.test.js.map +1 -1
- package/dist/src/agent/routes/stores.js +9 -12
- package/dist/src/agent/routes/stores.js.map +1 -1
- package/dist/src/agent/routes/task.d.ts +15 -3
- package/dist/src/agent/routes/task.js +16 -7
- package/dist/src/agent/routes/task.js.map +1 -1
- package/dist/src/agent/routes/task.test.d.ts +1 -1
- package/dist/src/agent/routes/task.test.js +70 -53
- package/dist/src/agent/routes/task.test.js.map +1 -1
- package/dist/src/agent/routes/webhooks.js +12 -3
- package/dist/src/agent/routes/webhooks.js.map +1 -1
- package/dist/src/agent/session-store.d.ts +11 -2
- package/dist/src/agent/session-store.js +1 -1
- package/dist/src/agent/session-store.js.map +1 -1
- package/dist/src/agent/snapshot-server.d.ts +2 -22
- package/dist/src/agent/snapshot-server.js +50 -27
- package/dist/src/agent/snapshot-server.js.map +1 -1
- package/dist/src/agent/states/compacting.d.ts +14 -0
- package/dist/src/agent/states/compacting.js +258 -0
- package/dist/src/agent/states/compacting.js.map +1 -0
- package/dist/src/agent/states/confirming.d.ts +10 -0
- package/dist/src/agent/states/confirming.js +76 -0
- package/dist/src/agent/states/confirming.js.map +1 -0
- package/dist/src/agent/states/dispatching.d.ts +18 -0
- package/dist/src/agent/states/dispatching.js +241 -0
- package/dist/src/agent/states/dispatching.js.map +1 -0
- package/dist/src/agent/states/executing.d.ts +21 -0
- package/dist/src/agent/states/executing.js +308 -0
- package/dist/src/agent/states/executing.js.map +1 -0
- package/dist/src/agent/states/streaming.d.ts +10 -0
- package/dist/src/agent/states/streaming.js +155 -0
- package/dist/src/agent/states/streaming.js.map +1 -0
- package/dist/src/agent/states/thinking.d.ts +13 -0
- package/dist/src/agent/states/thinking.js +233 -0
- package/dist/src/agent/states/thinking.js.map +1 -0
- package/dist/src/agent/token-estimate.d.ts +17 -0
- package/dist/src/agent/token-estimate.js +13 -0
- package/dist/src/agent/token-estimate.js.map +1 -0
- package/dist/src/agent/tool-executor-local.js +9 -18
- package/dist/src/agent/tool-executor-local.js.map +1 -1
- package/dist/src/agent/tool-executor-local.test.js +3 -5
- package/dist/src/agent/tool-executor-local.test.js.map +1 -1
- package/dist/src/api/create-agent.d.ts +15 -0
- package/dist/src/api/create-agent.js +137 -0
- package/dist/src/api/create-agent.js.map +1 -0
- package/dist/src/api/types.d.ts +68 -0
- package/dist/src/api/types.js +7 -0
- package/dist/src/api/types.js.map +1 -0
- package/dist/src/context/compiler.d.ts +13 -0
- package/dist/src/context/compiler.js +358 -0
- package/dist/src/context/compiler.js.map +1 -0
- package/dist/src/context/compiler.test.js +532 -0
- package/dist/src/context/compiler.test.js.map +1 -0
- package/dist/src/context/types.d.ts +110 -0
- package/dist/src/context/types.js +7 -0
- package/dist/src/context/types.js.map +1 -0
- package/dist/src/index.d.ts +33 -6
- package/dist/src/index.js +35 -21
- package/dist/src/index.js.map +1 -1
- package/dist/src/providers/create-provider.d.ts +23 -0
- package/dist/src/providers/create-provider.js +185 -0
- package/dist/src/providers/create-provider.js.map +1 -0
- package/dist/src/{agent/stores-e2e.test.d.ts → providers/create-provider.test.d.ts} +1 -1
- package/dist/src/providers/create-provider.test.js +95 -0
- package/dist/src/providers/create-provider.test.js.map +1 -0
- package/dist/src/providers/failover.d.ts +38 -0
- package/dist/src/providers/failover.js +147 -0
- package/dist/src/providers/failover.js.map +1 -0
- package/dist/src/providers/failover.test.d.ts +6 -0
- package/dist/src/providers/failover.test.js +169 -0
- package/dist/src/providers/failover.test.js.map +1 -0
- package/dist/src/providers/types.d.ts +110 -0
- package/dist/src/providers/types.js +7 -0
- package/dist/src/providers/types.js.map +1 -0
- package/dist/src/routes/ai-stream.d.ts +13 -10
- package/dist/src/routes/ai-stream.js +76 -41
- package/dist/src/routes/ai-stream.js.map +1 -1
- package/dist/src/routes/chat-new.test.d.ts +6 -0
- package/dist/src/routes/chat-new.test.js +107 -0
- package/dist/src/routes/chat-new.test.js.map +1 -0
- package/dist/src/routes/chat-stream-new.test.d.ts +6 -0
- package/dist/src/routes/chat-stream-new.test.js +135 -0
- package/dist/src/routes/chat-stream-new.test.js.map +1 -0
- package/dist/src/routes/chat-stream.d.ts +14 -4
- package/dist/src/routes/chat-stream.js +47 -29
- package/dist/src/routes/chat-stream.js.map +1 -1
- package/dist/src/routes/chat.d.ts +13 -4
- package/dist/src/routes/chat.js +60 -23
- package/dist/src/routes/chat.js.map +1 -1
- package/dist/src/routes/health.d.ts +3 -2
- package/dist/src/routes/health.js.map +1 -1
- package/dist/src/routes/route-helpers.d.ts +50 -0
- package/dist/src/routes/route-helpers.js +80 -0
- package/dist/src/routes/route-helpers.js.map +1 -0
- package/dist/src/routes/session-resolver.d.ts +72 -0
- package/dist/src/routes/session-resolver.js +123 -0
- package/dist/src/routes/session-resolver.js.map +1 -0
- package/dist/src/routes/session-resolver.test.d.ts +6 -0
- package/dist/src/routes/session-resolver.test.js +206 -0
- package/dist/src/routes/session-resolver.test.js.map +1 -0
- package/dist/src/routes/webhooks.d.ts +3 -1
- package/dist/src/routes/webhooks.js +12 -4
- package/dist/src/routes/webhooks.js.map +1 -1
- package/dist/src/security/permission-checker.d.ts +80 -0
- package/dist/src/security/permission-checker.js +75 -0
- package/dist/src/security/permission-checker.js.map +1 -0
- package/dist/src/security/permission-checker.test.d.ts +6 -0
- package/dist/src/security/permission-checker.test.js +208 -0
- package/dist/src/security/permission-checker.test.js.map +1 -0
- package/dist/src/server.d.ts +12 -11
- package/dist/src/server.js +44 -46
- package/dist/src/server.js.map +1 -1
- package/dist/src/server.test.d.ts +1 -1
- package/dist/src/server.test.js +6 -144
- package/dist/src/server.test.js.map +1 -1
- package/dist/src/session/manager.d.ts +98 -0
- package/dist/src/session/manager.js +364 -0
- package/dist/src/session/manager.js.map +1 -0
- package/dist/src/session/manager.test.d.ts +6 -0
- package/dist/src/session/manager.test.js +315 -0
- package/dist/src/session/manager.test.js.map +1 -0
- package/dist/src/session/session-builder.d.ts +71 -0
- package/dist/src/session/session-builder.js +364 -0
- package/dist/src/session/session-builder.js.map +1 -0
- package/dist/src/session/session-builder.test.d.ts +6 -0
- package/dist/src/session/session-builder.test.js +352 -0
- package/dist/src/session/session-builder.test.js.map +1 -0
- package/dist/src/session/store.d.ts +57 -0
- package/dist/src/session/store.js +167 -0
- package/dist/src/session/store.js.map +1 -0
- package/dist/src/session/store.test.d.ts +6 -0
- package/dist/src/session/store.test.js +145 -0
- package/dist/src/session/store.test.js.map +1 -0
- package/dist/src/session/stream-hooks.d.ts +39 -0
- package/dist/src/session/stream-hooks.js +7 -0
- package/dist/src/session/stream-hooks.js.map +1 -0
- package/dist/src/session/tool-context-factory.d.ts +60 -0
- package/dist/src/session/tool-context-factory.js +190 -0
- package/dist/src/session/tool-context-factory.js.map +1 -0
- package/dist/src/session/tool-context-factory.test.d.ts +6 -0
- package/dist/src/session/tool-context-factory.test.js +287 -0
- package/dist/src/session/tool-context-factory.test.js.map +1 -0
- package/dist/src/session/types.d.ts +188 -0
- package/dist/src/session/types.js +7 -0
- package/dist/src/session/types.js.map +1 -0
- package/dist/src/stores/drizzle-store-backend.d.ts +49 -0
- package/dist/src/stores/drizzle-store-backend.js +306 -0
- package/dist/src/stores/drizzle-store-backend.js.map +1 -0
- package/dist/src/stores/drizzle-store-backend.test.d.ts +6 -0
- package/dist/src/stores/drizzle-store-backend.test.js +215 -0
- package/dist/src/stores/drizzle-store-backend.test.js.map +1 -0
- package/dist/src/stores/index.d.ts +4 -0
- package/dist/src/stores/index.js +2 -0
- package/dist/src/stores/index.js.map +1 -1
- package/dist/src/stores/pglite-store-backend.d.ts +16 -19
- package/dist/src/stores/pglite-store-backend.js +85 -239
- package/dist/src/stores/pglite-store-backend.js.map +1 -1
- package/dist/src/stores/postgres-store-backend.d.ts +30 -0
- package/dist/src/stores/postgres-store-backend.js +100 -0
- package/dist/src/stores/postgres-store-backend.js.map +1 -0
- package/dist/src/stores/schema.d.ts +491 -0
- package/dist/src/stores/schema.js +57 -0
- package/dist/src/stores/schema.js.map +1 -0
- package/dist/src/tools/admin-file-tools.d.ts +13 -0
- package/dist/src/tools/admin-file-tools.js +200 -0
- package/dist/src/tools/admin-file-tools.js.map +1 -0
- package/dist/src/tools/admin-file-tools.test.d.ts +6 -0
- package/dist/src/tools/admin-file-tools.test.js +152 -0
- package/dist/src/tools/admin-file-tools.test.js.map +1 -0
- package/dist/src/tools/custom-tool-adapter.d.ts +41 -0
- package/dist/src/tools/custom-tool-adapter.js +190 -0
- package/dist/src/tools/custom-tool-adapter.js.map +1 -0
- package/dist/src/tools/custom-tool-adapter.test.d.ts +6 -0
- package/dist/src/tools/custom-tool-adapter.test.js +244 -0
- package/dist/src/tools/custom-tool-adapter.test.js.map +1 -0
- package/dist/src/tools/dispatch-tool.d.ts +52 -0
- package/dist/src/tools/dispatch-tool.js +71 -0
- package/dist/src/tools/dispatch-tool.js.map +1 -0
- package/dist/src/tools/dispatch-tool.test.d.ts +6 -0
- package/dist/src/tools/dispatch-tool.test.js +75 -0
- package/dist/src/tools/dispatch-tool.test.js.map +1 -0
- package/dist/src/tools/mcp-tool-adapter.d.ts +18 -0
- package/dist/src/tools/mcp-tool-adapter.js +135 -0
- package/dist/src/tools/mcp-tool-adapter.js.map +1 -0
- package/dist/src/tools/mcp-tool-adapter.test.d.ts +6 -0
- package/dist/src/tools/mcp-tool-adapter.test.js +227 -0
- package/dist/src/tools/mcp-tool-adapter.test.js.map +1 -0
- package/dist/src/tools/registry.d.ts +25 -0
- package/dist/src/tools/registry.js +72 -0
- package/dist/src/tools/registry.js.map +1 -0
- package/dist/src/tools/registry.test.d.ts +6 -0
- package/dist/src/tools/registry.test.js +121 -0
- package/dist/src/tools/registry.test.js.map +1 -0
- package/dist/src/tools/request-tool.d.ts +42 -0
- package/dist/src/tools/request-tool.js +190 -0
- package/dist/src/tools/request-tool.js.map +1 -0
- package/dist/src/tools/request-tool.test.d.ts +6 -0
- package/dist/src/tools/request-tool.test.js +254 -0
- package/dist/src/tools/request-tool.test.js.map +1 -0
- package/dist/src/tools/store-tools.d.ts +29 -0
- package/dist/src/tools/store-tools.js +224 -0
- package/dist/src/tools/store-tools.js.map +1 -0
- package/dist/src/tools/store-tools.test.d.ts +6 -0
- package/dist/src/tools/store-tools.test.js +216 -0
- package/dist/src/tools/store-tools.test.js.map +1 -0
- package/dist/src/tools/types.d.ts +111 -0
- package/dist/src/tools/types.js +7 -0
- package/dist/src/tools/types.js.map +1 -0
- package/dist/src/types.d.ts +20 -12
- package/dist/src/types.js +3 -2
- package/dist/src/types.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +13 -4
- package/dist/src/__tests__/sse-contract.test.js +0 -464
- package/dist/src/__tests__/sse-contract.test.js.map +0 -1
- package/dist/src/__tests__/tools.test.js +0 -583
- package/dist/src/__tests__/tools.test.js.map +0 -1
- package/dist/src/agent/agent-runner.d.ts +0 -33
- package/dist/src/agent/agent-runner.js +0 -1040
- package/dist/src/agent/agent-runner.js.map +0 -1
- package/dist/src/agent/custom-tools-e2e.test.d.ts +0 -6
- package/dist/src/agent/custom-tools-e2e.test.js +0 -566
- package/dist/src/agent/custom-tools-e2e.test.js.map +0 -1
- package/dist/src/agent/request-helper.d.ts +0 -16
- package/dist/src/agent/request-helper.js +0 -96
- package/dist/src/agent/request-helper.js.map +0 -1
- package/dist/src/agent/stores-e2e.test.js +0 -433
- package/dist/src/agent/stores-e2e.test.js.map +0 -1
- package/dist/src/agent/tool-context-builder.d.ts +0 -11
- package/dist/src/agent/tool-context-builder.js +0 -102
- package/dist/src/agent/tool-context-builder.js.map +0 -1
- package/dist/src/agent/tool-context-builder.test.d.ts +0 -6
- package/dist/src/agent/tool-context-builder.test.js +0 -152
- package/dist/src/agent/tool-context-builder.test.js.map +0 -1
- package/dist/src/agent/write-repo-file.test.js +0 -270
- package/dist/src/agent/write-repo-file.test.js.map +0 -1
- package/dist/src/cron/heartbeat-runner.d.ts +0 -21
- package/dist/src/cron/heartbeat-runner.js +0 -79
- package/dist/src/cron/heartbeat-runner.js.map +0 -1
- package/dist/src/cron/heartbeat-runner.test.d.ts +0 -6
- package/dist/src/cron/heartbeat-runner.test.js +0 -120
- package/dist/src/cron/heartbeat-runner.test.js.map +0 -1
- package/dist/src/cron/heartbeat-scheduler.d.ts +0 -26
- package/dist/src/cron/heartbeat-scheduler.js +0 -55
- package/dist/src/cron/heartbeat-scheduler.js.map +0 -1
- package/dist/src/cron/heartbeat-scheduler.test.d.ts +0 -6
- package/dist/src/cron/heartbeat-scheduler.test.js +0 -61
- package/dist/src/cron/heartbeat-scheduler.test.js.map +0 -1
- package/dist/src/routes/ai-stream.test.d.ts +0 -6
- package/dist/src/routes/ai-stream.test.js +0 -586
- package/dist/src/routes/ai-stream.test.js.map +0 -1
- package/dist/src/routes/ask-user-response.d.ts +0 -30
- package/dist/src/routes/ask-user-response.js +0 -61
- package/dist/src/routes/ask-user-response.js.map +0 -1
- package/dist/src/routes/ask-user-response.test.d.ts +0 -6
- package/dist/src/routes/ask-user-response.test.js +0 -88
- package/dist/src/routes/ask-user-response.test.js.map +0 -1
- package/dist/src/routes/chat-stream.test.d.ts +0 -6
- package/dist/src/routes/chat-stream.test.js +0 -155
- package/dist/src/routes/chat-stream.test.js.map +0 -1
- package/dist/src/routes/chat.test.d.ts +0 -6
- package/dist/src/routes/chat.test.js +0 -99
- package/dist/src/routes/chat.test.js.map +0 -1
- package/dist/src/routes/widget-actions.d.ts +0 -49
- package/dist/src/routes/widget-actions.js +0 -78
- package/dist/src/routes/widget-actions.js.map +0 -1
- package/dist/src/session/custom-tool-adapter.d.ts +0 -74
- package/dist/src/session/custom-tool-adapter.js +0 -180
- package/dist/src/session/custom-tool-adapter.js.map +0 -1
- package/dist/src/session/history-converter.d.ts +0 -21
- package/dist/src/session/history-converter.js +0 -59
- package/dist/src/session/history-converter.js.map +0 -1
- package/dist/src/session/history-converter.test.d.ts +0 -6
- package/dist/src/session/history-converter.test.js +0 -130
- package/dist/src/session/history-converter.test.js.map +0 -1
- package/dist/src/session/session-manager.d.ts +0 -219
- package/dist/src/session/session-manager.js +0 -915
- package/dist/src/session/session-manager.js.map +0 -1
- package/dist/src/session/session-manager.test.d.ts +0 -6
- package/dist/src/session/session-manager.test.js +0 -455
- package/dist/src/session/session-manager.test.js.map +0 -1
- package/dist/src/session/session-runner.d.ts +0 -45
- package/dist/src/session/session-runner.js +0 -719
- package/dist/src/session/session-runner.js.map +0 -1
- package/dist/src/session/session-runner.test.d.ts +0 -6
- package/dist/src/session/session-runner.test.js +0 -834
- package/dist/src/session/session-runner.test.js.map +0 -1
- /package/dist/src/{__tests__/sse-contract.test.d.ts → __fixtures__/smoke.test.d.ts} +0 -0
- /package/dist/src/{__tests__/tools.test.d.ts → agent/loop.test.d.ts} +0 -0
- /package/dist/src/{agent/write-repo-file.test.d.ts → context/compiler.test.d.ts} +0 -0
|
@@ -0,0 +1,718 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Amodal Labs, Inc.
|
|
4
|
+
* SPDX-License-Identifier: MIT
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Smoke tests — end-to-end integration tests against a self-contained
|
|
8
|
+
* test agent with mock REST and MCP servers.
|
|
9
|
+
*
|
|
10
|
+
* Requires ANTHROPIC_API_KEY in the environment (skips otherwise).
|
|
11
|
+
* Starts amodal dev programmatically, runs assertions, tears down.
|
|
12
|
+
*/
|
|
13
|
+
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
|
14
|
+
import { fork } from 'node:child_process';
|
|
15
|
+
import { resolve } from 'node:path';
|
|
16
|
+
import { readFileSync, writeFileSync, rmSync } from 'node:fs';
|
|
17
|
+
// Load API keys from repo root .env.test if not already set.
|
|
18
|
+
// To run smoke tests: create .env.test at the repo root with ANTHROPIC_API_KEY=sk-ant-...
|
|
19
|
+
// This file is gitignored — never commit API keys.
|
|
20
|
+
if (!process.env['ANTHROPIC_API_KEY']) {
|
|
21
|
+
try {
|
|
22
|
+
const envPath = resolve(__dirname, '../../../../.env.test');
|
|
23
|
+
const envContent = readFileSync(envPath, 'utf-8');
|
|
24
|
+
for (const line of envContent.split('\n')) {
|
|
25
|
+
const match = line.match(/^([^#=]+)=(.*)$/);
|
|
26
|
+
if (match) {
|
|
27
|
+
const [, key, value] = match;
|
|
28
|
+
if (key && value && !process.env[key.trim()]) {
|
|
29
|
+
process.env[key.trim()] = value.trim();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
catch { /* no .env.test — tests will skip */ }
|
|
35
|
+
}
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
// Config
|
|
38
|
+
// ---------------------------------------------------------------------------
|
|
39
|
+
const AGENT_PORT = 9900;
|
|
40
|
+
const REST_PORT = 9901;
|
|
41
|
+
const AGENT_DIR = resolve(__dirname, 'smoke-agent');
|
|
42
|
+
const REST_SERVER = resolve(__dirname, 'smoke-rest-server.mjs');
|
|
43
|
+
const MCP_SERVER = resolve(__dirname, 'smoke-mcp-server.mjs');
|
|
44
|
+
const TIMEOUT = 45_000; // per-test timeout for LLM calls
|
|
45
|
+
// ---------------------------------------------------------------------------
|
|
46
|
+
// Helpers
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
async function waitForServer(port, maxMs = 15_000) {
|
|
49
|
+
const deadline = Date.now() + maxMs;
|
|
50
|
+
while (Date.now() < deadline) {
|
|
51
|
+
try {
|
|
52
|
+
const res = await fetch(`http://localhost:${port}/health`, { signal: AbortSignal.timeout(1000) });
|
|
53
|
+
if (res.ok)
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
catch { /* not ready yet */ }
|
|
57
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
58
|
+
}
|
|
59
|
+
throw new Error(`Server on port ${port} did not start within ${maxMs}ms`);
|
|
60
|
+
}
|
|
61
|
+
async function chat(message, sessionId) {
|
|
62
|
+
const body = { message };
|
|
63
|
+
if (sessionId)
|
|
64
|
+
body['session_id'] = sessionId;
|
|
65
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/chat`, {
|
|
66
|
+
method: 'POST',
|
|
67
|
+
headers: { 'Content-Type': 'application/json' },
|
|
68
|
+
body: JSON.stringify(body),
|
|
69
|
+
signal: AbortSignal.timeout(TIMEOUT),
|
|
70
|
+
});
|
|
71
|
+
const text = await res.text();
|
|
72
|
+
const events = [];
|
|
73
|
+
let sid = '';
|
|
74
|
+
for (const line of text.split('\n')) {
|
|
75
|
+
if (!line.startsWith('data: '))
|
|
76
|
+
continue;
|
|
77
|
+
try {
|
|
78
|
+
const event = JSON.parse(line.slice(6));
|
|
79
|
+
events.push(event);
|
|
80
|
+
if (event['type'] === 'init' && typeof event['session_id'] === 'string') {
|
|
81
|
+
sid = event['session_id'];
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch { /* skip */ }
|
|
85
|
+
}
|
|
86
|
+
return { events, sessionId: sid };
|
|
87
|
+
}
|
|
88
|
+
function findEvent(events, type) {
|
|
89
|
+
return events.find((e) => e['type'] === type);
|
|
90
|
+
}
|
|
91
|
+
function findEvents(events, type) {
|
|
92
|
+
return events.filter((e) => e['type'] === type);
|
|
93
|
+
}
|
|
94
|
+
function allText(events) {
|
|
95
|
+
return events
|
|
96
|
+
.filter((e) => e['type'] === 'text_delta')
|
|
97
|
+
.map((e) => String(e['content'] ?? ''))
|
|
98
|
+
.join('');
|
|
99
|
+
}
|
|
100
|
+
// ---------------------------------------------------------------------------
|
|
101
|
+
// Setup / Teardown
|
|
102
|
+
// ---------------------------------------------------------------------------
|
|
103
|
+
let restServer = null;
|
|
104
|
+
let agentServer = null;
|
|
105
|
+
const skipReason = process.env['ANTHROPIC_API_KEY'] ? '' : 'ANTHROPIC_API_KEY not set';
|
|
106
|
+
describe.skipIf(!!skipReason)('smoke tests', () => {
|
|
107
|
+
beforeAll(async () => {
|
|
108
|
+
// 0. Nuke prior state — clean slate for every run
|
|
109
|
+
rmSync(resolve(AGENT_DIR, '.amodal/store-data'), { recursive: true, force: true });
|
|
110
|
+
rmSync(resolve(AGENT_DIR, '.amodal/sessions'), { recursive: true, force: true });
|
|
111
|
+
// 2. Write MCP server spec with absolute path (loadRepo reads this as-is)
|
|
112
|
+
writeFileSync(resolve(AGENT_DIR, 'connections/mock-mcp/spec.json'), JSON.stringify({ protocol: 'mcp', transport: 'stdio', command: 'node', args: [MCP_SERVER] }, null, 2));
|
|
113
|
+
// 3. Start mock REST server
|
|
114
|
+
restServer = fork(REST_SERVER, [], {
|
|
115
|
+
env: { ...process.env, SMOKE_REST_PORT: String(REST_PORT) },
|
|
116
|
+
stdio: 'pipe',
|
|
117
|
+
});
|
|
118
|
+
await new Promise((r) => setTimeout(r, 1000));
|
|
119
|
+
// 4. Start amodal dev programmatically
|
|
120
|
+
const { createLocalServer } = await import('../agent/local-server.js');
|
|
121
|
+
agentServer = await createLocalServer({
|
|
122
|
+
repoPath: AGENT_DIR,
|
|
123
|
+
port: AGENT_PORT,
|
|
124
|
+
hotReload: false,
|
|
125
|
+
});
|
|
126
|
+
await agentServer.start();
|
|
127
|
+
await waitForServer(AGENT_PORT);
|
|
128
|
+
}, 30_000);
|
|
129
|
+
afterAll(async () => {
|
|
130
|
+
if (agentServer) {
|
|
131
|
+
await agentServer.stop();
|
|
132
|
+
}
|
|
133
|
+
if (restServer) {
|
|
134
|
+
restServer.kill('SIGTERM');
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
// -------------------------------------------------------------------------
|
|
138
|
+
// 1. Server lifecycle
|
|
139
|
+
// -------------------------------------------------------------------------
|
|
140
|
+
it('health endpoint returns ok', async () => {
|
|
141
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/health`);
|
|
142
|
+
const body = await res.json();
|
|
143
|
+
expect(res.status).toBe(200);
|
|
144
|
+
expect(body['status']).toBe('ok');
|
|
145
|
+
});
|
|
146
|
+
it('config endpoint returns agent info', async () => {
|
|
147
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/config`);
|
|
148
|
+
const body = await res.json();
|
|
149
|
+
expect(res.status).toBe(200);
|
|
150
|
+
expect(body['name']).toBe('smoke-test-agent');
|
|
151
|
+
});
|
|
152
|
+
// -------------------------------------------------------------------------
|
|
153
|
+
// 2. System prompt (G9)
|
|
154
|
+
// -------------------------------------------------------------------------
|
|
155
|
+
it('system prompt includes all context sections', async () => {
|
|
156
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/inspect/context`);
|
|
157
|
+
const body = await res.json();
|
|
158
|
+
const prompt = String(body['system_prompt'] ?? '');
|
|
159
|
+
expect(prompt.length).toBeGreaterThan(500);
|
|
160
|
+
expect(prompt).toContain('mock-api'); // connection
|
|
161
|
+
expect(prompt).toContain('test-skill'); // skill
|
|
162
|
+
expect(prompt).toContain('Smoke Test Reference'); // knowledge
|
|
163
|
+
expect(prompt).toContain('test-items'); // store
|
|
164
|
+
});
|
|
165
|
+
// -------------------------------------------------------------------------
|
|
166
|
+
// 3. Chat streaming
|
|
167
|
+
// -------------------------------------------------------------------------
|
|
168
|
+
it('streams chat with init, text, and done events', async () => {
|
|
169
|
+
const { events } = await chat('Say hello in exactly 3 words.');
|
|
170
|
+
const init = findEvent(events, 'init');
|
|
171
|
+
const done = findEvent(events, 'done');
|
|
172
|
+
const textDeltas = findEvents(events, 'text_delta');
|
|
173
|
+
expect(init).toBeDefined();
|
|
174
|
+
expect(done).toBeDefined();
|
|
175
|
+
expect(textDeltas.length).toBeGreaterThan(0);
|
|
176
|
+
// Done event should have usage
|
|
177
|
+
const usage = done?.['usage'];
|
|
178
|
+
expect(usage?.['input_tokens']).toBeGreaterThan(0);
|
|
179
|
+
expect(usage?.['output_tokens']).toBeGreaterThan(0);
|
|
180
|
+
}, TIMEOUT);
|
|
181
|
+
// -------------------------------------------------------------------------
|
|
182
|
+
// 4. Session resume
|
|
183
|
+
// -------------------------------------------------------------------------
|
|
184
|
+
it('resumes session with prior context', async () => {
|
|
185
|
+
const first = await chat('Remember this code: SMOKE7742. Just confirm you noted it.');
|
|
186
|
+
expect(first.sessionId).toBeTruthy();
|
|
187
|
+
const second = await chat('What was the code I asked you to remember? Reply with just the code.', first.sessionId);
|
|
188
|
+
const responseText = allText(second.events);
|
|
189
|
+
expect(responseText).toContain('SMOKE7742');
|
|
190
|
+
}, TIMEOUT * 2);
|
|
191
|
+
// -------------------------------------------------------------------------
|
|
192
|
+
// 5. Tool call — store
|
|
193
|
+
// -------------------------------------------------------------------------
|
|
194
|
+
it('makes at least one tool call across chat interactions', async () => {
|
|
195
|
+
// Use a prompt that strongly implies tool use — query existing data
|
|
196
|
+
const { events } = await chat('Query the test-items store for all items. Use the query_store tool with store="test-items".');
|
|
197
|
+
// The model should call query_store. If no tool calls at all, the test
|
|
198
|
+
// is still valid — it means the model chose not to call tools, which is
|
|
199
|
+
// an LLM non-determinism issue, not a code bug. We mark it as a soft check.
|
|
200
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
201
|
+
if (toolResults.length === 0) {
|
|
202
|
+
// Soft fail — log but don't block CI
|
|
203
|
+
// eslint-disable-next-line no-console -- intentional test diagnostic
|
|
204
|
+
console.warn('[smoke] Model did not call any tools — LLM non-determinism, not a code bug');
|
|
205
|
+
}
|
|
206
|
+
else {
|
|
207
|
+
// If tools were called, verify they have proper status
|
|
208
|
+
for (const result of toolResults) {
|
|
209
|
+
expect(result['status']).toMatch(/^(success|error)$/);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}, TIMEOUT);
|
|
213
|
+
// -------------------------------------------------------------------------
|
|
214
|
+
// 6. Tool call — connection request
|
|
215
|
+
// -------------------------------------------------------------------------
|
|
216
|
+
it('calls request tool against mock-api', async () => {
|
|
217
|
+
const { events } = await chat('Use the request tool to GET /items from the mock-api connection with intent "read".');
|
|
218
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
219
|
+
const success = toolResults.find((e) => e['status'] === 'success');
|
|
220
|
+
expect(success).toBeDefined();
|
|
221
|
+
const responseText = allText(events);
|
|
222
|
+
expect(responseText).toContain('Widget');
|
|
223
|
+
}, TIMEOUT);
|
|
224
|
+
// -------------------------------------------------------------------------
|
|
225
|
+
// 7. Tool error status
|
|
226
|
+
// -------------------------------------------------------------------------
|
|
227
|
+
it('reports tool errors with status error, not success', async () => {
|
|
228
|
+
const { events } = await chat('Use the request tool to call GET /items on a connection called "nonexistent-connection" with intent "read".');
|
|
229
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
230
|
+
const errorResult = toolResults.find((e) => e['status'] === 'error');
|
|
231
|
+
expect(errorResult).toBeDefined();
|
|
232
|
+
}, TIMEOUT);
|
|
233
|
+
// -------------------------------------------------------------------------
|
|
234
|
+
// 8. Eval run
|
|
235
|
+
// -------------------------------------------------------------------------
|
|
236
|
+
it('runs eval and returns results', async () => {
|
|
237
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/evals/run`, {
|
|
238
|
+
method: 'POST',
|
|
239
|
+
headers: { 'Content-Type': 'application/json' },
|
|
240
|
+
body: JSON.stringify({ evalNames: ['basic-eval'] }),
|
|
241
|
+
signal: AbortSignal.timeout(60_000),
|
|
242
|
+
});
|
|
243
|
+
const text = await res.text();
|
|
244
|
+
const events = [];
|
|
245
|
+
for (const line of text.split('\n')) {
|
|
246
|
+
if (!line.startsWith('data: '))
|
|
247
|
+
continue;
|
|
248
|
+
try {
|
|
249
|
+
events.push(JSON.parse(line.slice(6)));
|
|
250
|
+
}
|
|
251
|
+
catch { /* skip */ }
|
|
252
|
+
}
|
|
253
|
+
const complete = findEvent(events, 'eval_complete');
|
|
254
|
+
expect(complete).toBeDefined();
|
|
255
|
+
expect(complete?.['passed']).toBe(true);
|
|
256
|
+
}, 60_000);
|
|
257
|
+
// -------------------------------------------------------------------------
|
|
258
|
+
// 9. Admin chat — reads repo files
|
|
259
|
+
// -------------------------------------------------------------------------
|
|
260
|
+
it('admin agent can read skill files', async () => {
|
|
261
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/config/chat`, {
|
|
262
|
+
method: 'POST',
|
|
263
|
+
headers: { 'Content-Type': 'application/json' },
|
|
264
|
+
body: JSON.stringify({ message: 'Read the test-skill skill file and tell me what it says. Be brief.' }),
|
|
265
|
+
signal: AbortSignal.timeout(TIMEOUT),
|
|
266
|
+
});
|
|
267
|
+
const text = await res.text();
|
|
268
|
+
const events = parseSSE(text);
|
|
269
|
+
const init = findEvent(events, 'init');
|
|
270
|
+
expect(init).toBeDefined();
|
|
271
|
+
// Admin agent should use read_repo_file tool
|
|
272
|
+
const toolStarts = findEvents(events, 'tool_call_start');
|
|
273
|
+
const readTool = toolStarts.find((e) => e['tool_name'] === 'read_repo_file');
|
|
274
|
+
expect(readTool).toBeDefined();
|
|
275
|
+
const responseText = allText(events);
|
|
276
|
+
expect(responseText.toLowerCase()).toContain('test');
|
|
277
|
+
}, TIMEOUT);
|
|
278
|
+
// -------------------------------------------------------------------------
|
|
279
|
+
// 10. Write intent enforcement (G8)
|
|
280
|
+
// -------------------------------------------------------------------------
|
|
281
|
+
it('rejects POST with intent "read"', async () => {
|
|
282
|
+
const { events } = await chat('Use the request tool to call POST /items on mock-api with intent "read" and data {"name": "test"}. Do not use "write" intent — use exactly "read".');
|
|
283
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
284
|
+
// Should get an error result about intent mismatch
|
|
285
|
+
const hasError = toolResults.some((e) => e['status'] === 'error');
|
|
286
|
+
const responseText = allText(events);
|
|
287
|
+
const mentionsIntent = responseText.toLowerCase().includes('intent') || responseText.toLowerCase().includes('write');
|
|
288
|
+
// Either the tool returned an error about intent, or the model explained the rejection
|
|
289
|
+
expect(hasError || mentionsIntent).toBe(true);
|
|
290
|
+
}, TIMEOUT);
|
|
291
|
+
// -------------------------------------------------------------------------
|
|
292
|
+
// 11. Store write + query persistence
|
|
293
|
+
// -------------------------------------------------------------------------
|
|
294
|
+
it('persists data across store write and query', async () => {
|
|
295
|
+
// Write
|
|
296
|
+
const writeResult = await chat('Store a test item: use store_test_items with item_id="persist-check", name="Persistence Test", status="active". Call the tool now.');
|
|
297
|
+
const writeToolResults = findEvents(writeResult.events, 'tool_call_result');
|
|
298
|
+
const writeSuccess = writeToolResults.find((e) => e['status'] === 'success');
|
|
299
|
+
if (!writeSuccess) {
|
|
300
|
+
// Model didn't call the tool — skip gracefully
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
// Query back in a NEW session (proves persistence, not just in-memory)
|
|
304
|
+
const queryResult = await chat('You have a tool called query_store. Use it now with store="test-items" and filter={"item_id": "persist-check"}. Then tell me the name field of the result.');
|
|
305
|
+
const queryToolResults = findEvents(queryResult.events, 'tool_call_result');
|
|
306
|
+
if (queryToolResults.length === 0) {
|
|
307
|
+
// Model didn't call query_store despite explicit instruction — LLM non-determinism
|
|
308
|
+
// eslint-disable-next-line no-console -- intentional test diagnostic
|
|
309
|
+
console.warn('[smoke] Model did not call query_store in persistence test — LLM non-determinism');
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
const responseText = allText(queryResult.events);
|
|
313
|
+
expect(responseText).toContain('Persistence Test');
|
|
314
|
+
}, TIMEOUT * 2);
|
|
315
|
+
// -------------------------------------------------------------------------
|
|
316
|
+
// 11b. Store batch write
|
|
317
|
+
// -------------------------------------------------------------------------
|
|
318
|
+
it('batch writes multiple items to store', async () => {
|
|
319
|
+
const { events } = await chat('Write two items to the test-items store using the batch tool: item_id="batch-1", name="First Batch", status="active" and item_id="batch-2", name="Second Batch", status="archived".');
|
|
320
|
+
const toolStarts = findEvents(events, 'tool_call_start');
|
|
321
|
+
const batchTool = toolStarts.find((e) => String(e['tool_name'] ?? '').includes('batch'));
|
|
322
|
+
if (!batchTool) {
|
|
323
|
+
// Model didn't use batch — might have used individual writes, that's OK
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
327
|
+
const batchResult = toolResults.find((e) => e['tool_id'] === batchTool['tool_id']);
|
|
328
|
+
expect(batchResult).toBeDefined();
|
|
329
|
+
expect(batchResult?.['status']).toBe('success');
|
|
330
|
+
}, TIMEOUT);
|
|
331
|
+
// -------------------------------------------------------------------------
|
|
332
|
+
// 11c. Store single document fetch by key
|
|
333
|
+
// -------------------------------------------------------------------------
|
|
334
|
+
it('fetches a single document by key from store', async () => {
|
|
335
|
+
// First write a known item and verify the write succeeded
|
|
336
|
+
const writeResult = await chat('Write to the test-items store: item_id="key-lookup-test", name="Key Lookup Item", status="active".');
|
|
337
|
+
const writeSuccess = findEvents(writeResult.events, 'tool_call_result').find((e) => e['status'] === 'success');
|
|
338
|
+
if (!writeSuccess)
|
|
339
|
+
return; // Write didn't happen — skip
|
|
340
|
+
// Fetch by key in a new session
|
|
341
|
+
const { events } = await chat('Use query_store with store="test-items" and key="key-lookup-test". What is the name field?');
|
|
342
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
343
|
+
if (toolResults.length === 0) {
|
|
344
|
+
// eslint-disable-next-line no-console -- intentional test diagnostic
|
|
345
|
+
console.warn('[smoke] Model did not call query_store for key lookup — LLM non-determinism');
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
const responseText = allText(events);
|
|
349
|
+
expect(responseText).toContain('Key Lookup');
|
|
350
|
+
}, TIMEOUT * 2);
|
|
351
|
+
// -------------------------------------------------------------------------
|
|
352
|
+
// 11d. Store filtered query (multiple results)
|
|
353
|
+
// -------------------------------------------------------------------------
|
|
354
|
+
it('queries store with filter and returns multiple results', async () => {
|
|
355
|
+
// Write two items with a unique status we can filter on
|
|
356
|
+
const w1 = await chat('Write to test-items store: item_id="filter-a", name="Filter Alpha", status="archived".');
|
|
357
|
+
const w2 = await chat('Write to test-items store: item_id="filter-b", name="Filter Beta", status="archived".');
|
|
358
|
+
const w1ok = findEvents(w1.events, 'tool_call_result').some((e) => e['status'] === 'success');
|
|
359
|
+
const w2ok = findEvents(w2.events, 'tool_call_result').some((e) => e['status'] === 'success');
|
|
360
|
+
if (!w1ok || !w2ok)
|
|
361
|
+
return; // Writes didn't happen — skip
|
|
362
|
+
// Query with filter in a new session
|
|
363
|
+
const { events } = await chat('Use query_store with store="test-items" and filter={"status": "archived"}. List the names of all results.');
|
|
364
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
365
|
+
if (toolResults.length === 0) {
|
|
366
|
+
// eslint-disable-next-line no-console -- intentional test diagnostic
|
|
367
|
+
console.warn('[smoke] Model did not call query_store for filtered query — LLM non-determinism');
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
const responseText = allText(events);
|
|
371
|
+
const mentionsAny = responseText.includes('Filter Alpha') || responseText.includes('Filter Beta');
|
|
372
|
+
expect(mentionsAny).toBe(true);
|
|
373
|
+
}, TIMEOUT * 3);
|
|
374
|
+
// -------------------------------------------------------------------------
|
|
375
|
+
// 12. Concurrent sessions don't bleed context
|
|
376
|
+
// -------------------------------------------------------------------------
|
|
377
|
+
it('concurrent sessions are isolated', async () => {
|
|
378
|
+
// Session A: tell it a secret
|
|
379
|
+
const sessionA = await chat('My secret code for this session is ALPHA9999. Just confirm.');
|
|
380
|
+
// Session B: different secret (we don't need session B's ID)
|
|
381
|
+
await chat('My secret code for this session is BETA5555. Just confirm.');
|
|
382
|
+
// Ask session A about B's secret — should NOT know it
|
|
383
|
+
const checkA = await chat('What is the BETA code?', sessionA.sessionId);
|
|
384
|
+
const textA = allText(checkA.events);
|
|
385
|
+
// Session A should not contain session B's secret
|
|
386
|
+
expect(textA).not.toContain('BETA5555');
|
|
387
|
+
}, TIMEOUT * 3);
|
|
388
|
+
// -------------------------------------------------------------------------
|
|
389
|
+
// 13. Automation trigger
|
|
390
|
+
// -------------------------------------------------------------------------
|
|
391
|
+
it('triggers automation via API', async () => {
|
|
392
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/automations`, { signal: AbortSignal.timeout(5000) });
|
|
393
|
+
const body = await res.json();
|
|
394
|
+
// Smoke agent doesn't have automations defined in amodal.json,
|
|
395
|
+
// but the endpoint should still respond
|
|
396
|
+
expect(res.status).toBe(200);
|
|
397
|
+
expect(body.automations).toBeDefined();
|
|
398
|
+
});
|
|
399
|
+
// -------------------------------------------------------------------------
|
|
400
|
+
// 14. Multi-turn tool loop
|
|
401
|
+
// -------------------------------------------------------------------------
|
|
402
|
+
it('handles multi-turn tool interaction', async () => {
|
|
403
|
+
// Ask something that requires a tool call then reasoning about the result
|
|
404
|
+
const { events } = await chat('Fetch items from mock-api using the request tool (GET /items, intent "read"), then tell me how many items have status "active".');
|
|
405
|
+
// Should have tool call AND text response with the count
|
|
406
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
407
|
+
const responseText = allText(events);
|
|
408
|
+
expect(toolResults.length).toBeGreaterThan(0);
|
|
409
|
+
// Mock returns 2 active items (Widget, Doohickey) out of 3
|
|
410
|
+
expect(responseText).toMatch(/2|two/i);
|
|
411
|
+
}, TIMEOUT);
|
|
412
|
+
// -------------------------------------------------------------------------
|
|
413
|
+
// 15. Evals list endpoint
|
|
414
|
+
// -------------------------------------------------------------------------
|
|
415
|
+
it('lists eval suites from repo', async () => {
|
|
416
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/evals/suites`, { signal: AbortSignal.timeout(5000) });
|
|
417
|
+
const body = await res.json();
|
|
418
|
+
expect(res.status).toBe(200);
|
|
419
|
+
expect(body.suites.length).toBeGreaterThan(0);
|
|
420
|
+
expect(body.suites[0]?.['name']).toBe('basic-eval');
|
|
421
|
+
});
|
|
422
|
+
// -------------------------------------------------------------------------
|
|
423
|
+
// 16. Inspect endpoint — connection health
|
|
424
|
+
// -------------------------------------------------------------------------
|
|
425
|
+
it('inspect shows connection status', async () => {
|
|
426
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/inspect/context`, { signal: AbortSignal.timeout(10000) });
|
|
427
|
+
const body = await res.json();
|
|
428
|
+
expect(res.status).toBe(200);
|
|
429
|
+
expect(body['connections']).toBeDefined();
|
|
430
|
+
});
|
|
431
|
+
// -------------------------------------------------------------------------
|
|
432
|
+
// 17. MCP tool call
|
|
433
|
+
// -------------------------------------------------------------------------
|
|
434
|
+
it('calls MCP tool and gets result', async () => {
|
|
435
|
+
const { events } = await chat('Use the mock-mcp__smoke_search tool to search for "test". Call the tool now.');
|
|
436
|
+
const toolStarts = findEvents(events, 'tool_call_start');
|
|
437
|
+
const mcpTool = toolStarts.find((e) => String(e['tool_name'] ?? '').includes('smoke_search'));
|
|
438
|
+
if (!mcpTool) {
|
|
439
|
+
// Check if MCP tools are even available — model might not know about them
|
|
440
|
+
const responseText = allText(events);
|
|
441
|
+
// If the model says it doesn't have that tool, MCP isn't wired
|
|
442
|
+
if (responseText.toLowerCase().includes('not available') || responseText.toLowerCase().includes('don\'t have')) {
|
|
443
|
+
throw new Error('MCP tools not registered — mock-mcp__smoke_search not available to the model');
|
|
444
|
+
}
|
|
445
|
+
// Model just chose not to call it — LLM non-determinism
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
449
|
+
const mcpResult = toolResults.find((e) => e['tool_id'] === mcpTool['tool_id']);
|
|
450
|
+
expect(mcpResult).toBeDefined();
|
|
451
|
+
expect(mcpResult?.['status']).toBe('success');
|
|
452
|
+
}, TIMEOUT);
|
|
453
|
+
// -------------------------------------------------------------------------
|
|
454
|
+
// 18. Custom tool (echo_tool) with ctx.request() + ctx.store()
|
|
455
|
+
// -------------------------------------------------------------------------
|
|
456
|
+
it('custom tool calls ctx.request and ctx.store', async () => {
|
|
457
|
+
const { events } = await chat('Use the echo_tool with message "smoke-test-ping". Call the tool now.');
|
|
458
|
+
const toolStarts = findEvents(events, 'tool_call_start');
|
|
459
|
+
const echoTool = toolStarts.find((e) => e['tool_name'] === 'echo_tool');
|
|
460
|
+
if (!echoTool) {
|
|
461
|
+
const responseText = allText(events);
|
|
462
|
+
if (responseText.toLowerCase().includes('not available') || responseText.toLowerCase().includes('don\'t have')) {
|
|
463
|
+
throw new Error('echo_tool not registered');
|
|
464
|
+
}
|
|
465
|
+
return; // LLM non-determinism
|
|
466
|
+
}
|
|
467
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
468
|
+
const echoResult = toolResults.find((e) => e['tool_id'] === echoTool['tool_id']);
|
|
469
|
+
expect(echoResult).toBeDefined();
|
|
470
|
+
expect(echoResult?.['status']).toBe('success');
|
|
471
|
+
}, TIMEOUT);
|
|
472
|
+
// -------------------------------------------------------------------------
|
|
473
|
+
// 19. Stop execution tool terminates loop
|
|
474
|
+
// -------------------------------------------------------------------------
|
|
475
|
+
it('stop_execution tool is available', async () => {
|
|
476
|
+
// We can't easily force the model to call stop_execution, but we can
|
|
477
|
+
// verify it's in the tool list by asking the model
|
|
478
|
+
const { events } = await chat('Do you have a tool called stop_execution? Answer yes or no, nothing else.');
|
|
479
|
+
const responseText = allText(events).toLowerCase();
|
|
480
|
+
expect(responseText).toContain('yes');
|
|
481
|
+
}, TIMEOUT);
|
|
482
|
+
// -------------------------------------------------------------------------
|
|
483
|
+
// 20. Done event always has usage (G2)
|
|
484
|
+
// -------------------------------------------------------------------------
|
|
485
|
+
it('done event always includes token usage', async () => {
|
|
486
|
+
const { events } = await chat('Reply with exactly the word "pong".');
|
|
487
|
+
const done = findEvent(events, 'done');
|
|
488
|
+
expect(done).toBeDefined();
|
|
489
|
+
const usage = done?.['usage'];
|
|
490
|
+
expect(usage).toBeDefined();
|
|
491
|
+
expect(typeof usage?.['input_tokens']).toBe('number');
|
|
492
|
+
expect(typeof usage?.['output_tokens']).toBe('number');
|
|
493
|
+
expect(usage?.['input_tokens']).toBeGreaterThan(0);
|
|
494
|
+
expect(usage?.['output_tokens']).toBeGreaterThan(0);
|
|
495
|
+
}, TIMEOUT);
|
|
496
|
+
// -------------------------------------------------------------------------
|
|
497
|
+
// 21. Sub-agent dispatch
|
|
498
|
+
// -------------------------------------------------------------------------
|
|
499
|
+
it('dispatch_task spawns child agent and returns result', async () => {
|
|
500
|
+
const { events } = await chat('Use the dispatch_task tool to delegate a sub-task. Set agent_name to "data-fetcher", tools to ["request"], and prompt to "Fetch GET /items from mock-api with intent read and summarize what you find." Call dispatch_task now.');
|
|
501
|
+
// Look for subagent events (child activity)
|
|
502
|
+
const subagentEvents = findEvents(events, 'subagent_event');
|
|
503
|
+
// Look for the dispatch_task tool call result
|
|
504
|
+
const toolStarts = findEvents(events, 'tool_call_start');
|
|
505
|
+
const dispatchStart = toolStarts.find((e) => e['tool_name'] === 'dispatch_task');
|
|
506
|
+
if (!dispatchStart) {
|
|
507
|
+
// Model didn't call dispatch_task — LLM non-determinism
|
|
508
|
+
const responseText = allText(events);
|
|
509
|
+
if (responseText.toLowerCase().includes('not available') || responseText.toLowerCase().includes('don\'t have')) {
|
|
510
|
+
throw new Error('dispatch_task tool not registered');
|
|
511
|
+
}
|
|
512
|
+
return;
|
|
513
|
+
}
|
|
514
|
+
// dispatch_task was called — verify it completed
|
|
515
|
+
const toolResults = findEvents(events, 'tool_call_result');
|
|
516
|
+
const dispatchResult = toolResults.find((e) => e['tool_id'] === dispatchStart['tool_id']);
|
|
517
|
+
expect(dispatchResult).toBeDefined();
|
|
518
|
+
expect(dispatchResult?.['status']).toBe('success');
|
|
519
|
+
// SubagentEvents should have been emitted during child execution
|
|
520
|
+
if (subagentEvents.length > 0) {
|
|
521
|
+
// All should reference the same parent_tool_id
|
|
522
|
+
for (const event of subagentEvents) {
|
|
523
|
+
expect(event['parent_tool_id']).toBe(dispatchStart['tool_id']);
|
|
524
|
+
expect(event['agent_name']).toBe('data-fetcher');
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
// Parent should have incorporated the child's result into its response
|
|
528
|
+
const responseText = allText(events);
|
|
529
|
+
expect(responseText.length).toBeGreaterThan(0);
|
|
530
|
+
}, TIMEOUT * 2);
|
|
531
|
+
it('dispatch_task tool is available to the model', async () => {
|
|
532
|
+
const { events } = await chat('Do you have a tool called dispatch_task? Answer yes or no, nothing else.');
|
|
533
|
+
const responseText = allText(events).toLowerCase();
|
|
534
|
+
expect(responseText).toContain('yes');
|
|
535
|
+
}, TIMEOUT);
|
|
536
|
+
// -------------------------------------------------------------------------
|
|
537
|
+
// 22. Pages — user-defined React pages
|
|
538
|
+
// -------------------------------------------------------------------------
|
|
539
|
+
it('lists pages with metadata from repo', async () => {
|
|
540
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/pages`, { signal: AbortSignal.timeout(5000) });
|
|
541
|
+
const body = await res.json();
|
|
542
|
+
expect(res.status).toBe(200);
|
|
543
|
+
expect(body.pages.length).toBeGreaterThan(0);
|
|
544
|
+
const testPage = body.pages.find((p) => p['name'] === 'TestPage');
|
|
545
|
+
expect(testPage).toBeDefined();
|
|
546
|
+
expect(testPage?.['description']).toBe('Smoke test page fixture');
|
|
547
|
+
expect(testPage?.['stores']).toEqual(['test-items']);
|
|
548
|
+
});
|
|
549
|
+
it('serves compiled page bundle', async () => {
|
|
550
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/pages-bundle/TestPage.js`, { signal: AbortSignal.timeout(5000) });
|
|
551
|
+
expect(res.status).toBe(200);
|
|
552
|
+
const bundle = await res.text();
|
|
553
|
+
// IIFE bundle registers on window.__AMODAL_PAGES__
|
|
554
|
+
expect(bundle).toContain('__AMODAL_PAGES__');
|
|
555
|
+
expect(bundle).toContain('TestPage');
|
|
556
|
+
});
|
|
557
|
+
// -------------------------------------------------------------------------
|
|
558
|
+
// 23. Sessions — listing and history
|
|
559
|
+
// -------------------------------------------------------------------------
|
|
560
|
+
it('sessions endpoint returns a sessions array', async () => {
|
|
561
|
+
// Chat sessions in local dev don't auto-populate the legacy session store
|
|
562
|
+
// used by /sessions (only automation runs do), so we just verify the
|
|
563
|
+
// endpoint returns the expected shape.
|
|
564
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/sessions`, { signal: AbortSignal.timeout(5000) });
|
|
565
|
+
const body = await res.json();
|
|
566
|
+
expect(res.status).toBe(200);
|
|
567
|
+
expect(Array.isArray(body.sessions)).toBe(true);
|
|
568
|
+
});
|
|
569
|
+
it('returns 404 for unknown session', async () => {
|
|
570
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/session/nonexistent-id`, { signal: AbortSignal.timeout(5000) });
|
|
571
|
+
expect(res.status).toBe(404);
|
|
572
|
+
});
|
|
573
|
+
// -------------------------------------------------------------------------
|
|
574
|
+
// 24. Files — browser and editor
|
|
575
|
+
// -------------------------------------------------------------------------
|
|
576
|
+
it('lists repo files as a tree', async () => {
|
|
577
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/files`, { signal: AbortSignal.timeout(5000) });
|
|
578
|
+
const body = await res.json();
|
|
579
|
+
expect(res.status).toBe(200);
|
|
580
|
+
expect(body.tree.length).toBeGreaterThan(0);
|
|
581
|
+
// Should include at least one convention directory
|
|
582
|
+
const names = body.tree.map((n) => String(n['name']));
|
|
583
|
+
expect(names.some((n) => ['skills', 'connections', 'stores', 'tools'].includes(n))).toBe(true);
|
|
584
|
+
});
|
|
585
|
+
it('reads a specific file', async () => {
|
|
586
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/files/amodal.json`, { signal: AbortSignal.timeout(5000) });
|
|
587
|
+
expect(res.status).toBe(200);
|
|
588
|
+
const body = await res.json();
|
|
589
|
+
expect(body.path).toBe('amodal.json');
|
|
590
|
+
expect(body.language).toBe('json');
|
|
591
|
+
expect(body.content).toContain('smoke-test-agent');
|
|
592
|
+
});
|
|
593
|
+
it('writes a file and reads it back', async () => {
|
|
594
|
+
const testPath = 'knowledge/smoke-write-test.md';
|
|
595
|
+
const testContent = '# Smoke Write Test\n\nThis file was written by a smoke test.';
|
|
596
|
+
const writeRes = await fetch(`http://localhost:${AGENT_PORT}/api/files/${testPath}`, {
|
|
597
|
+
method: 'PUT',
|
|
598
|
+
headers: { 'Content-Type': 'application/json' },
|
|
599
|
+
body: JSON.stringify({ content: testContent }),
|
|
600
|
+
signal: AbortSignal.timeout(5000),
|
|
601
|
+
});
|
|
602
|
+
expect(writeRes.status).toBe(200);
|
|
603
|
+
const readRes = await fetch(`http://localhost:${AGENT_PORT}/api/files/${testPath}`, { signal: AbortSignal.timeout(5000) });
|
|
604
|
+
expect(readRes.status).toBe(200);
|
|
605
|
+
const body = await readRes.json();
|
|
606
|
+
expect(body.content).toBe(testContent);
|
|
607
|
+
});
|
|
608
|
+
it('rejects path traversal attempts', async () => {
|
|
609
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/files/..%2F..%2F..%2Fetc%2Fpasswd`, { signal: AbortSignal.timeout(5000) });
|
|
610
|
+
expect([400, 403, 404]).toContain(res.status);
|
|
611
|
+
});
|
|
612
|
+
// -------------------------------------------------------------------------
|
|
613
|
+
// 25. Webhooks — inbound automation trigger
|
|
614
|
+
// -------------------------------------------------------------------------
|
|
615
|
+
it('rejects webhook for unknown automation with 404', async () => {
|
|
616
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/webhooks/nonexistent-automation`, {
|
|
617
|
+
method: 'POST',
|
|
618
|
+
headers: { 'Content-Type': 'application/json' },
|
|
619
|
+
body: JSON.stringify({ event: 'test' }),
|
|
620
|
+
signal: AbortSignal.timeout(5000),
|
|
621
|
+
});
|
|
622
|
+
expect(res.status).toBe(404);
|
|
623
|
+
const body = await res.json();
|
|
624
|
+
expect(body.error).toContain('not found');
|
|
625
|
+
});
|
|
626
|
+
// -------------------------------------------------------------------------
|
|
627
|
+
// 26. Store REST API — CRUD outside chat
|
|
628
|
+
// -------------------------------------------------------------------------
|
|
629
|
+
it('lists stores with document counts', async () => {
|
|
630
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/stores`, { signal: AbortSignal.timeout(5000) });
|
|
631
|
+
expect(res.status).toBe(200);
|
|
632
|
+
const body = await res.json();
|
|
633
|
+
expect(body.stores.length).toBeGreaterThan(0);
|
|
634
|
+
const testItems = body.stores.find((s) => s['name'] === 'test-items');
|
|
635
|
+
expect(testItems).toBeDefined();
|
|
636
|
+
expect(typeof testItems?.['documentCount']).toBe('number');
|
|
637
|
+
});
|
|
638
|
+
it('writes and retrieves a document via REST', async () => {
|
|
639
|
+
const writeRes = await fetch(`http://localhost:${AGENT_PORT}/api/stores/test-items`, {
|
|
640
|
+
method: 'POST',
|
|
641
|
+
headers: { 'Content-Type': 'application/json' },
|
|
642
|
+
body: JSON.stringify({ item_id: 'rest-api-test', name: 'REST API Item', status: 'active' }),
|
|
643
|
+
signal: AbortSignal.timeout(5000),
|
|
644
|
+
});
|
|
645
|
+
expect(writeRes.status).toBe(201);
|
|
646
|
+
const writeBody = await writeRes.json();
|
|
647
|
+
expect(writeBody.key).toBe('rest-api-test');
|
|
648
|
+
const readRes = await fetch(`http://localhost:${AGENT_PORT}/api/stores/test-items/rest-api-test`, { signal: AbortSignal.timeout(5000) });
|
|
649
|
+
expect(readRes.status).toBe(200);
|
|
650
|
+
const readBody = await readRes.json();
|
|
651
|
+
expect(readBody.document.payload['name']).toBe('REST API Item');
|
|
652
|
+
});
|
|
653
|
+
it('lists documents in a store', async () => {
|
|
654
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/stores/test-items?limit=10`, { signal: AbortSignal.timeout(5000) });
|
|
655
|
+
expect(res.status).toBe(200);
|
|
656
|
+
const body = await res.json();
|
|
657
|
+
expect(Array.isArray(body.documents)).toBe(true);
|
|
658
|
+
expect(typeof body.total).toBe('number');
|
|
659
|
+
});
|
|
660
|
+
it('returns 404 for unknown store', async () => {
|
|
661
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/stores/nonexistent-store`, { signal: AbortSignal.timeout(5000) });
|
|
662
|
+
expect(res.status).toBe(404);
|
|
663
|
+
});
|
|
664
|
+
// -------------------------------------------------------------------------
|
|
665
|
+
// 27. Feedback
|
|
666
|
+
// -------------------------------------------------------------------------
|
|
667
|
+
it('saves feedback rating', async () => {
|
|
668
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/feedback`, {
|
|
669
|
+
method: 'POST',
|
|
670
|
+
headers: { 'Content-Type': 'application/json' },
|
|
671
|
+
body: JSON.stringify({
|
|
672
|
+
sessionId: 'smoke-session',
|
|
673
|
+
messageId: 'smoke-msg-1',
|
|
674
|
+
rating: 'up',
|
|
675
|
+
query: 'Test query',
|
|
676
|
+
response: 'Test response',
|
|
677
|
+
}),
|
|
678
|
+
signal: AbortSignal.timeout(5000),
|
|
679
|
+
});
|
|
680
|
+
expect(res.status).toBe(200);
|
|
681
|
+
const body = await res.json();
|
|
682
|
+
expect(body.ok).toBe(true);
|
|
683
|
+
expect(body.id).toBeTruthy();
|
|
684
|
+
});
|
|
685
|
+
it('returns feedback summary stats', async () => {
|
|
686
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/feedback/summary`, { signal: AbortSignal.timeout(5000) });
|
|
687
|
+
expect(res.status).toBe(200);
|
|
688
|
+
const body = await res.json();
|
|
689
|
+
expect(typeof body.total).toBe('number');
|
|
690
|
+
expect(typeof body.thumbsUp).toBe('number');
|
|
691
|
+
expect(typeof body.thumbsDown).toBe('number');
|
|
692
|
+
});
|
|
693
|
+
it('rejects invalid feedback rating', async () => {
|
|
694
|
+
const res = await fetch(`http://localhost:${AGENT_PORT}/api/feedback`, {
|
|
695
|
+
method: 'POST',
|
|
696
|
+
headers: { 'Content-Type': 'application/json' },
|
|
697
|
+
body: JSON.stringify({ sessionId: 'x', messageId: 'y', rating: 'invalid' }),
|
|
698
|
+
signal: AbortSignal.timeout(5000),
|
|
699
|
+
});
|
|
700
|
+
expect(res.status).toBe(400);
|
|
701
|
+
});
|
|
702
|
+
});
|
|
703
|
+
// ---------------------------------------------------------------------------
|
|
704
|
+
// SSE parser helper
|
|
705
|
+
// ---------------------------------------------------------------------------
|
|
706
|
+
function parseSSE(text) {
|
|
707
|
+
const events = [];
|
|
708
|
+
for (const line of text.split('\n')) {
|
|
709
|
+
if (!line.startsWith('data: '))
|
|
710
|
+
continue;
|
|
711
|
+
try {
|
|
712
|
+
events.push(JSON.parse(line.slice(6)));
|
|
713
|
+
}
|
|
714
|
+
catch { /* skip */ }
|
|
715
|
+
}
|
|
716
|
+
return events;
|
|
717
|
+
}
|
|
718
|
+
//# sourceMappingURL=smoke.test.js.map
|