@lleverage-ai/agent-sdk 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +2321 -0
- package/dist/agent.d.ts +52 -0
- package/dist/agent.d.ts.map +1 -0
- package/dist/agent.js +2122 -0
- package/dist/agent.js.map +1 -0
- package/dist/backend.d.ts +378 -0
- package/dist/backend.d.ts.map +1 -0
- package/dist/backend.js +71 -0
- package/dist/backend.js.map +1 -0
- package/dist/backends/composite.d.ts +258 -0
- package/dist/backends/composite.d.ts.map +1 -0
- package/dist/backends/composite.js +437 -0
- package/dist/backends/composite.js.map +1 -0
- package/dist/backends/filesystem.d.ts +268 -0
- package/dist/backends/filesystem.d.ts.map +1 -0
- package/dist/backends/filesystem.js +623 -0
- package/dist/backends/filesystem.js.map +1 -0
- package/dist/backends/index.d.ts +14 -0
- package/dist/backends/index.d.ts.map +1 -0
- package/dist/backends/index.js +14 -0
- package/dist/backends/index.js.map +1 -0
- package/dist/backends/persistent.d.ts +312 -0
- package/dist/backends/persistent.d.ts.map +1 -0
- package/dist/backends/persistent.js +519 -0
- package/dist/backends/persistent.js.map +1 -0
- package/dist/backends/sandbox.d.ts +315 -0
- package/dist/backends/sandbox.d.ts.map +1 -0
- package/dist/backends/sandbox.js +490 -0
- package/dist/backends/sandbox.js.map +1 -0
- package/dist/backends/state.d.ts +225 -0
- package/dist/backends/state.d.ts.map +1 -0
- package/dist/backends/state.js +396 -0
- package/dist/backends/state.js.map +1 -0
- package/dist/checkpointer/file-saver.d.ts +182 -0
- package/dist/checkpointer/file-saver.d.ts.map +1 -0
- package/dist/checkpointer/file-saver.js +298 -0
- package/dist/checkpointer/file-saver.js.map +1 -0
- package/dist/checkpointer/index.d.ts +40 -0
- package/dist/checkpointer/index.d.ts.map +1 -0
- package/dist/checkpointer/index.js +40 -0
- package/dist/checkpointer/index.js.map +1 -0
- package/dist/checkpointer/kv-saver.d.ts +142 -0
- package/dist/checkpointer/kv-saver.d.ts.map +1 -0
- package/dist/checkpointer/kv-saver.js +176 -0
- package/dist/checkpointer/kv-saver.js.map +1 -0
- package/dist/checkpointer/memory-saver.d.ts +158 -0
- package/dist/checkpointer/memory-saver.d.ts.map +1 -0
- package/dist/checkpointer/memory-saver.js +222 -0
- package/dist/checkpointer/memory-saver.js.map +1 -0
- package/dist/checkpointer/types.d.ts +353 -0
- package/dist/checkpointer/types.d.ts.map +1 -0
- package/dist/checkpointer/types.js +159 -0
- package/dist/checkpointer/types.js.map +1 -0
- package/dist/context-manager.d.ts +627 -0
- package/dist/context-manager.d.ts.map +1 -0
- package/dist/context-manager.js +1039 -0
- package/dist/context-manager.js.map +1 -0
- package/dist/context.d.ts +57 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +76 -0
- package/dist/context.js.map +1 -0
- package/dist/errors/index.d.ts +611 -0
- package/dist/errors/index.d.ts.map +1 -0
- package/dist/errors/index.js +1023 -0
- package/dist/errors/index.js.map +1 -0
- package/dist/generation-helpers.d.ts +126 -0
- package/dist/generation-helpers.d.ts.map +1 -0
- package/dist/generation-helpers.js +181 -0
- package/dist/generation-helpers.js.map +1 -0
- package/dist/hooks/audit.d.ts +210 -0
- package/dist/hooks/audit.d.ts.map +1 -0
- package/dist/hooks/audit.js +305 -0
- package/dist/hooks/audit.js.map +1 -0
- package/dist/hooks/cache.d.ts +180 -0
- package/dist/hooks/cache.d.ts.map +1 -0
- package/dist/hooks/cache.js +273 -0
- package/dist/hooks/cache.js.map +1 -0
- package/dist/hooks/guardrails.d.ts +145 -0
- package/dist/hooks/guardrails.d.ts.map +1 -0
- package/dist/hooks/guardrails.js +326 -0
- package/dist/hooks/guardrails.js.map +1 -0
- package/dist/hooks/index.d.ts +18 -0
- package/dist/hooks/index.d.ts.map +1 -0
- package/dist/hooks/index.js +32 -0
- package/dist/hooks/index.js.map +1 -0
- package/dist/hooks/logging.d.ts +193 -0
- package/dist/hooks/logging.d.ts.map +1 -0
- package/dist/hooks/logging.js +345 -0
- package/dist/hooks/logging.js.map +1 -0
- package/dist/hooks/parallel-guardrails.d.ts +268 -0
- package/dist/hooks/parallel-guardrails.d.ts.map +1 -0
- package/dist/hooks/parallel-guardrails.js +416 -0
- package/dist/hooks/parallel-guardrails.js.map +1 -0
- package/dist/hooks/rate-limit.d.ts +305 -0
- package/dist/hooks/rate-limit.d.ts.map +1 -0
- package/dist/hooks/rate-limit.js +372 -0
- package/dist/hooks/rate-limit.js.map +1 -0
- package/dist/hooks/retry.d.ts +144 -0
- package/dist/hooks/retry.d.ts.map +1 -0
- package/dist/hooks/retry.js +210 -0
- package/dist/hooks/retry.js.map +1 -0
- package/dist/hooks/secrets.d.ts +174 -0
- package/dist/hooks/secrets.d.ts.map +1 -0
- package/dist/hooks/secrets.js +306 -0
- package/dist/hooks/secrets.js.map +1 -0
- package/dist/hooks.d.ts +229 -0
- package/dist/hooks.d.ts.map +1 -0
- package/dist/hooks.js +352 -0
- package/dist/hooks.js.map +1 -0
- package/dist/index.d.ts +97 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +182 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp/env.d.ts +25 -0
- package/dist/mcp/env.d.ts.map +1 -0
- package/dist/mcp/env.js +18 -0
- package/dist/mcp/env.js.map +1 -0
- package/dist/mcp/index.d.ts +16 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +17 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/manager.d.ts +184 -0
- package/dist/mcp/manager.d.ts.map +1 -0
- package/dist/mcp/manager.js +446 -0
- package/dist/mcp/manager.js.map +1 -0
- package/dist/mcp/types.d.ts +58 -0
- package/dist/mcp/types.d.ts.map +1 -0
- package/dist/mcp/types.js +7 -0
- package/dist/mcp/types.js.map +1 -0
- package/dist/mcp/validation.d.ts +119 -0
- package/dist/mcp/validation.d.ts.map +1 -0
- package/dist/mcp/validation.js +407 -0
- package/dist/mcp/validation.js.map +1 -0
- package/dist/mcp/virtual-server.d.ts +78 -0
- package/dist/mcp/virtual-server.d.ts.map +1 -0
- package/dist/mcp/virtual-server.js +137 -0
- package/dist/mcp/virtual-server.js.map +1 -0
- package/dist/memory/filesystem-store.d.ts +217 -0
- package/dist/memory/filesystem-store.d.ts.map +1 -0
- package/dist/memory/filesystem-store.js +343 -0
- package/dist/memory/filesystem-store.js.map +1 -0
- package/dist/memory/index.d.ts +46 -0
- package/dist/memory/index.d.ts.map +1 -0
- package/dist/memory/index.js +46 -0
- package/dist/memory/index.js.map +1 -0
- package/dist/memory/loader.d.ts +396 -0
- package/dist/memory/loader.d.ts.map +1 -0
- package/dist/memory/loader.js +419 -0
- package/dist/memory/loader.js.map +1 -0
- package/dist/memory/permissions.d.ts +282 -0
- package/dist/memory/permissions.d.ts.map +1 -0
- package/dist/memory/permissions.js +297 -0
- package/dist/memory/permissions.js.map +1 -0
- package/dist/memory/rules.d.ts +249 -0
- package/dist/memory/rules.d.ts.map +1 -0
- package/dist/memory/rules.js +362 -0
- package/dist/memory/rules.js.map +1 -0
- package/dist/memory/store.d.ts +286 -0
- package/dist/memory/store.d.ts.map +1 -0
- package/dist/memory/store.js +263 -0
- package/dist/memory/store.js.map +1 -0
- package/dist/middleware/apply.d.ts +73 -0
- package/dist/middleware/apply.d.ts.map +1 -0
- package/dist/middleware/apply.js +219 -0
- package/dist/middleware/apply.js.map +1 -0
- package/dist/middleware/context.d.ts +33 -0
- package/dist/middleware/context.d.ts.map +1 -0
- package/dist/middleware/context.js +176 -0
- package/dist/middleware/context.js.map +1 -0
- package/dist/middleware/index.d.ts +31 -0
- package/dist/middleware/index.d.ts.map +1 -0
- package/dist/middleware/index.js +32 -0
- package/dist/middleware/index.js.map +1 -0
- package/dist/middleware/logging.d.ts +137 -0
- package/dist/middleware/logging.d.ts.map +1 -0
- package/dist/middleware/logging.js +374 -0
- package/dist/middleware/logging.js.map +1 -0
- package/dist/middleware/types.d.ts +183 -0
- package/dist/middleware/types.d.ts.map +1 -0
- package/dist/middleware/types.js +11 -0
- package/dist/middleware/types.js.map +1 -0
- package/dist/observability/events.d.ts +183 -0
- package/dist/observability/events.d.ts.map +1 -0
- package/dist/observability/events.js +305 -0
- package/dist/observability/events.js.map +1 -0
- package/dist/observability/index.d.ts +55 -0
- package/dist/observability/index.d.ts.map +1 -0
- package/dist/observability/index.js +87 -0
- package/dist/observability/index.js.map +1 -0
- package/dist/observability/logger.d.ts +318 -0
- package/dist/observability/logger.d.ts.map +1 -0
- package/dist/observability/logger.js +436 -0
- package/dist/observability/logger.js.map +1 -0
- package/dist/observability/metrics.d.ts +341 -0
- package/dist/observability/metrics.d.ts.map +1 -0
- package/dist/observability/metrics.js +490 -0
- package/dist/observability/metrics.js.map +1 -0
- package/dist/observability/preset.d.ts +161 -0
- package/dist/observability/preset.d.ts.map +1 -0
- package/dist/observability/preset.js +133 -0
- package/dist/observability/preset.js.map +1 -0
- package/dist/observability/streaming.d.ts +113 -0
- package/dist/observability/streaming.d.ts.map +1 -0
- package/dist/observability/streaming.js +114 -0
- package/dist/observability/streaming.js.map +1 -0
- package/dist/observability/tracing.d.ts +378 -0
- package/dist/observability/tracing.d.ts.map +1 -0
- package/dist/observability/tracing.js +539 -0
- package/dist/observability/tracing.js.map +1 -0
- package/dist/plugins.d.ts +55 -0
- package/dist/plugins.d.ts.map +1 -0
- package/dist/plugins.js +63 -0
- package/dist/plugins.js.map +1 -0
- package/dist/presets/index.d.ts +7 -0
- package/dist/presets/index.d.ts.map +1 -0
- package/dist/presets/index.js +7 -0
- package/dist/presets/index.js.map +1 -0
- package/dist/presets/production.d.ts +262 -0
- package/dist/presets/production.d.ts.map +1 -0
- package/dist/presets/production.js +295 -0
- package/dist/presets/production.js.map +1 -0
- package/dist/security/index.d.ts +179 -0
- package/dist/security/index.d.ts.map +1 -0
- package/dist/security/index.js +323 -0
- package/dist/security/index.js.map +1 -0
- package/dist/subagents/advanced.d.ts +413 -0
- package/dist/subagents/advanced.d.ts.map +1 -0
- package/dist/subagents/advanced.js +396 -0
- package/dist/subagents/advanced.js.map +1 -0
- package/dist/subagents/index.d.ts +14 -0
- package/dist/subagents/index.d.ts.map +1 -0
- package/dist/subagents/index.js +15 -0
- package/dist/subagents/index.js.map +1 -0
- package/dist/subagents.d.ts +73 -0
- package/dist/subagents.d.ts.map +1 -0
- package/dist/subagents.js +213 -0
- package/dist/subagents.js.map +1 -0
- package/dist/task-store/file-store.d.ts +76 -0
- package/dist/task-store/file-store.d.ts.map +1 -0
- package/dist/task-store/file-store.js +190 -0
- package/dist/task-store/file-store.js.map +1 -0
- package/dist/task-store/index.d.ts +11 -0
- package/dist/task-store/index.d.ts.map +1 -0
- package/dist/task-store/index.js +10 -0
- package/dist/task-store/index.js.map +1 -0
- package/dist/task-store/kv-store.d.ts +140 -0
- package/dist/task-store/kv-store.d.ts.map +1 -0
- package/dist/task-store/kv-store.js +169 -0
- package/dist/task-store/kv-store.js.map +1 -0
- package/dist/task-store/memory-store.d.ts +66 -0
- package/dist/task-store/memory-store.d.ts.map +1 -0
- package/dist/task-store/memory-store.js +125 -0
- package/dist/task-store/memory-store.js.map +1 -0
- package/dist/task-store/types.d.ts +235 -0
- package/dist/task-store/types.d.ts.map +1 -0
- package/dist/task-store/types.js +110 -0
- package/dist/task-store/types.js.map +1 -0
- package/dist/testing/assertions.d.ts +401 -0
- package/dist/testing/assertions.d.ts.map +1 -0
- package/dist/testing/assertions.js +630 -0
- package/dist/testing/assertions.js.map +1 -0
- package/dist/testing/index.d.ts +343 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +360 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/mock-agent.d.ts +214 -0
- package/dist/testing/mock-agent.d.ts.map +1 -0
- package/dist/testing/mock-agent.js +448 -0
- package/dist/testing/mock-agent.js.map +1 -0
- package/dist/testing/recorder.d.ts +288 -0
- package/dist/testing/recorder.d.ts.map +1 -0
- package/dist/testing/recorder.js +499 -0
- package/dist/testing/recorder.js.map +1 -0
- package/dist/tools/execute.d.ts +104 -0
- package/dist/tools/execute.d.ts.map +1 -0
- package/dist/tools/execute.js +191 -0
- package/dist/tools/execute.js.map +1 -0
- package/dist/tools/factory.d.ts +260 -0
- package/dist/tools/factory.d.ts.map +1 -0
- package/dist/tools/factory.js +241 -0
- package/dist/tools/factory.js.map +1 -0
- package/dist/tools/filesystem.d.ts +215 -0
- package/dist/tools/filesystem.d.ts.map +1 -0
- package/dist/tools/filesystem.js +311 -0
- package/dist/tools/filesystem.js.map +1 -0
- package/dist/tools/index.d.ts +33 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +33 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/search.d.ts +59 -0
- package/dist/tools/search.d.ts.map +1 -0
- package/dist/tools/search.js +94 -0
- package/dist/tools/search.js.map +1 -0
- package/dist/tools/skills.d.ts +354 -0
- package/dist/tools/skills.d.ts.map +1 -0
- package/dist/tools/skills.js +413 -0
- package/dist/tools/skills.js.map +1 -0
- package/dist/tools/task.d.ts +272 -0
- package/dist/tools/task.d.ts.map +1 -0
- package/dist/tools/task.js +521 -0
- package/dist/tools/task.js.map +1 -0
- package/dist/tools/todos.d.ts +131 -0
- package/dist/tools/todos.d.ts.map +1 -0
- package/dist/tools/todos.js +120 -0
- package/dist/tools/todos.js.map +1 -0
- package/dist/tools/tool-registry.d.ts +424 -0
- package/dist/tools/tool-registry.d.ts.map +1 -0
- package/dist/tools/tool-registry.js +607 -0
- package/dist/tools/tool-registry.js.map +1 -0
- package/dist/tools/user-interaction.d.ts +116 -0
- package/dist/tools/user-interaction.d.ts.map +1 -0
- package/dist/tools/user-interaction.js +147 -0
- package/dist/tools/user-interaction.js.map +1 -0
- package/dist/tools/utils.d.ts +124 -0
- package/dist/tools/utils.d.ts.map +1 -0
- package/dist/tools/utils.js +189 -0
- package/dist/tools/utils.js.map +1 -0
- package/dist/tools.d.ts +74 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +73 -0
- package/dist/tools.js.map +1 -0
- package/dist/types.d.ts +2421 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +55 -0
- package/dist/types.js.map +1 -0
- package/package.json +81 -0
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Testing utilities for the Agent SDK.
|
|
3
|
+
*
|
|
4
|
+
* This module provides comprehensive testing support including:
|
|
5
|
+
* - Mock agents with configurable responses
|
|
6
|
+
* - Recording and playback of agent interactions
|
|
7
|
+
* - Assertion helpers for common test scenarios
|
|
8
|
+
*
|
|
9
|
+
* ## Installation
|
|
10
|
+
*
|
|
11
|
+
* The testing utilities are included in the main package:
|
|
12
|
+
*
|
|
13
|
+
* ```typescript
|
|
14
|
+
* import {
|
|
15
|
+
* createMockAgent,
|
|
16
|
+
* createPlaybackAgent,
|
|
17
|
+
* assertResponseContains,
|
|
18
|
+
* } from "@lleverage-ai/agent-sdk/testing";
|
|
19
|
+
* ```
|
|
20
|
+
*
|
|
21
|
+
* ## Integration Test Patterns
|
|
22
|
+
*
|
|
23
|
+
* ### Pattern 1: Mock Agent for Unit Tests
|
|
24
|
+
*
|
|
25
|
+
* Use mock agents when testing code that depends on agent behavior:
|
|
26
|
+
*
|
|
27
|
+
* ```typescript
|
|
28
|
+
* import { createMockAgent, assertAgentBehavior } from "@lleverage-ai/agent-sdk/testing";
|
|
29
|
+
*
|
|
30
|
+
* describe("UserAssistant", () => {
|
|
31
|
+
* it("handles greeting", async () => {
|
|
32
|
+
* const agent = createMockAgent({
|
|
33
|
+
* response: { text: "Hello! How can I help?" },
|
|
34
|
+
* });
|
|
35
|
+
*
|
|
36
|
+
* const assistant = new UserAssistant(agent);
|
|
37
|
+
* const response = await assistant.greet();
|
|
38
|
+
*
|
|
39
|
+
* expect(response).toContain("Hello");
|
|
40
|
+
* expect(agent.generateCalls).toHaveLength(1);
|
|
41
|
+
* });
|
|
42
|
+
*
|
|
43
|
+
* it("handles multiple interactions", async () => {
|
|
44
|
+
* const agent = createMockAgent();
|
|
45
|
+
* agent.queueResponses(
|
|
46
|
+
* { text: "First response" },
|
|
47
|
+
* { text: "Second response" },
|
|
48
|
+
* { text: "Third response" },
|
|
49
|
+
* );
|
|
50
|
+
*
|
|
51
|
+
* const results = await Promise.all([
|
|
52
|
+
* assistant.query("1"),
|
|
53
|
+
* assistant.query("2"),
|
|
54
|
+
* assistant.query("3"),
|
|
55
|
+
* ]);
|
|
56
|
+
*
|
|
57
|
+
* expect(results[0]).toContain("First");
|
|
58
|
+
* expect(agent.getGenerateCallCount()).toBe(3);
|
|
59
|
+
* });
|
|
60
|
+
* });
|
|
61
|
+
* ```
|
|
62
|
+
*
|
|
63
|
+
* ### Pattern 2: Dynamic Responses Based on Input
|
|
64
|
+
*
|
|
65
|
+
* Use response handlers for context-aware mocking:
|
|
66
|
+
*
|
|
67
|
+
* ```typescript
|
|
68
|
+
* const agent = createMockAgent({
|
|
69
|
+
* responseHandler: (opts) => {
|
|
70
|
+
* if (opts.prompt?.includes("weather")) {
|
|
71
|
+
* return { text: "The weather is sunny!" };
|
|
72
|
+
* }
|
|
73
|
+
* if (opts.prompt?.includes("time")) {
|
|
74
|
+
* return { text: "The time is 3:00 PM." };
|
|
75
|
+
* }
|
|
76
|
+
* return { text: "I don't understand." };
|
|
77
|
+
* },
|
|
78
|
+
* });
|
|
79
|
+
*
|
|
80
|
+
* const r1 = await agent.generate({ prompt: "What's the weather?" });
|
|
81
|
+
* expect(r1.text).toContain("sunny");
|
|
82
|
+
*
|
|
83
|
+
* const r2 = await agent.generate({ prompt: "What's the time?" });
|
|
84
|
+
* expect(r2.text).toContain("3:00 PM");
|
|
85
|
+
* ```
|
|
86
|
+
*
|
|
87
|
+
* ### Pattern 3: Recording Real Interactions
|
|
88
|
+
*
|
|
89
|
+
* Record interactions from a real agent for later playback:
|
|
90
|
+
*
|
|
91
|
+
* ```typescript
|
|
92
|
+
* import { createRecordingAgent } from "@lleverage-ai/agent-sdk/testing";
|
|
93
|
+
* import { createAgent } from "@lleverage-ai/agent-sdk";
|
|
94
|
+
* import fs from "fs";
|
|
95
|
+
*
|
|
96
|
+
* // Create recording agent wrapping a real agent
|
|
97
|
+
* const realAgent = createAgent({
|
|
98
|
+
* model: anthropic("claude-sonnet-4-20250514"),
|
|
99
|
+
* systemPrompt: "You are a helpful assistant.",
|
|
100
|
+
* });
|
|
101
|
+
*
|
|
102
|
+
* const recordingAgent = createRecordingAgent(realAgent, {
|
|
103
|
+
* description: "User onboarding flow",
|
|
104
|
+
* tags: ["onboarding", "regression"],
|
|
105
|
+
* });
|
|
106
|
+
*
|
|
107
|
+
* // Make real API calls
|
|
108
|
+
* await recordingAgent.generate({ prompt: "Hello" });
|
|
109
|
+
* await recordingAgent.generate({ prompt: "What can you do?" });
|
|
110
|
+
* await recordingAgent.generate({ prompt: "Help me get started" });
|
|
111
|
+
*
|
|
112
|
+
* // Save recording for future tests
|
|
113
|
+
* const recording = recordingAgent.exportRecording();
|
|
114
|
+
* fs.writeFileSync("fixtures/onboarding.json", recording);
|
|
115
|
+
* ```
|
|
116
|
+
*
|
|
117
|
+
* ### Pattern 4: Playback for Regression Tests
|
|
118
|
+
*
|
|
119
|
+
* Replay recorded interactions for deterministic tests:
|
|
120
|
+
*
|
|
121
|
+
* ```typescript
|
|
122
|
+
* import { createPlaybackAgent, parseRecording } from "@lleverage-ai/agent-sdk/testing";
|
|
123
|
+
* import onboardingFixture from "./fixtures/onboarding.json";
|
|
124
|
+
*
|
|
125
|
+
* describe("Onboarding Flow", () => {
|
|
126
|
+
* let agent;
|
|
127
|
+
*
|
|
128
|
+
* beforeEach(() => {
|
|
129
|
+
* agent = createPlaybackAgent({
|
|
130
|
+
* recording: parseRecording(onboardingFixture),
|
|
131
|
+
* matchMode: "sequence",
|
|
132
|
+
* });
|
|
133
|
+
* });
|
|
134
|
+
*
|
|
135
|
+
* it("handles the complete onboarding flow", async () => {
|
|
136
|
+
* const r1 = await agent.generate({ prompt: "Hello" });
|
|
137
|
+
* const r2 = await agent.generate({ prompt: "What can you do?" });
|
|
138
|
+
* const r3 = await agent.generate({ prompt: "Help me get started" });
|
|
139
|
+
*
|
|
140
|
+
* expect(agent.isComplete()).toBe(true);
|
|
141
|
+
* expect(r1.text).toBeDefined();
|
|
142
|
+
* expect(r2.text).toBeDefined();
|
|
143
|
+
* expect(r3.text).toBeDefined();
|
|
144
|
+
* });
|
|
145
|
+
* });
|
|
146
|
+
* ```
|
|
147
|
+
*
|
|
148
|
+
* ### Pattern 5: Testing Tool Calls
|
|
149
|
+
*
|
|
150
|
+
* Verify that agents use tools correctly:
|
|
151
|
+
*
|
|
152
|
+
* ```typescript
|
|
153
|
+
* import {
|
|
154
|
+
* createMockAgent,
|
|
155
|
+
* assertToolCalled,
|
|
156
|
+
* assertToolCalledWith,
|
|
157
|
+
* assertToolNotCalled,
|
|
158
|
+
* } from "@lleverage-ai/agent-sdk/testing";
|
|
159
|
+
*
|
|
160
|
+
* it("uses the search tool for queries", async () => {
|
|
161
|
+
* const agent = createMockAgent({
|
|
162
|
+
* response: {
|
|
163
|
+
* text: "Found results for Tokyo",
|
|
164
|
+
* steps: [{
|
|
165
|
+
* text: "",
|
|
166
|
+
* toolCalls: [{
|
|
167
|
+
* toolCallId: "call-1",
|
|
168
|
+
* toolName: "search",
|
|
169
|
+
* input: { query: "Tokyo weather" },
|
|
170
|
+
* }],
|
|
171
|
+
* toolResults: [{
|
|
172
|
+
* toolCallId: "call-1",
|
|
173
|
+
* toolName: "search",
|
|
174
|
+
* output: "Tokyo: 25°C, sunny",
|
|
175
|
+
* }],
|
|
176
|
+
* finishReason: "tool-calls",
|
|
177
|
+
* }],
|
|
178
|
+
* },
|
|
179
|
+
* });
|
|
180
|
+
*
|
|
181
|
+
* const result = await agent.generate({ prompt: "Weather in Tokyo?" });
|
|
182
|
+
*
|
|
183
|
+
* assertToolCalled(result, "search");
|
|
184
|
+
* assertToolCalledWith(result, "search", { query: "Tokyo weather" });
|
|
185
|
+
* assertToolNotCalled(result, "dangerousTool");
|
|
186
|
+
* });
|
|
187
|
+
* ```
|
|
188
|
+
*
|
|
189
|
+
* ### Pattern 6: Testing State Changes
|
|
190
|
+
*
|
|
191
|
+
* Verify agent state modifications:
|
|
192
|
+
*
|
|
193
|
+
* ```typescript
|
|
194
|
+
* import {
|
|
195
|
+
* createMockAgent,
|
|
196
|
+
* assertStateHasFile,
|
|
197
|
+
* assertStateHasTodo,
|
|
198
|
+
* assertTodoCount,
|
|
199
|
+
* } from "@lleverage-ai/agent-sdk/testing";
|
|
200
|
+
*
|
|
201
|
+
* it("tracks todos correctly", async () => {
|
|
202
|
+
* const agent = createMockAgent({
|
|
203
|
+
* initialState: {
|
|
204
|
+
* todos: [
|
|
205
|
+
* { id: "1", content: "Fix bug", status: "pending", createdAt: new Date().toISOString() },
|
|
206
|
+
* ],
|
|
207
|
+
* },
|
|
208
|
+
* });
|
|
209
|
+
*
|
|
210
|
+
* // Simulate agent modifying state
|
|
211
|
+
* agent.state.todos[0].status = "completed";
|
|
212
|
+
* agent.state.todos.push({
|
|
213
|
+
* id: "2",
|
|
214
|
+
* content: "Write tests",
|
|
215
|
+
* status: "pending",
|
|
216
|
+
* createdAt: new Date().toISOString(),
|
|
217
|
+
* });
|
|
218
|
+
*
|
|
219
|
+
* assertTodoCount(agent.state, 2);
|
|
220
|
+
* assertTodoCount(agent.state, 1, "completed");
|
|
221
|
+
* assertStateHasTodo(agent.state, { content: "Fix bug", status: "completed" });
|
|
222
|
+
* });
|
|
223
|
+
* ```
|
|
224
|
+
*
|
|
225
|
+
* ### Pattern 7: Testing Streaming
|
|
226
|
+
*
|
|
227
|
+
* Test streaming responses:
|
|
228
|
+
*
|
|
229
|
+
* ```typescript
|
|
230
|
+
* import {
|
|
231
|
+
* createMockAgent,
|
|
232
|
+
* collectStreamChunks,
|
|
233
|
+
* assertStreamHasText,
|
|
234
|
+
* assertStreamFinished,
|
|
235
|
+
* getStreamText,
|
|
236
|
+
* } from "@lleverage-ai/agent-sdk/testing";
|
|
237
|
+
*
|
|
238
|
+
* it("streams response correctly", async () => {
|
|
239
|
+
* const agent = createMockAgent({
|
|
240
|
+
* response: {
|
|
241
|
+
* text: "Hello world",
|
|
242
|
+
* streamChunks: [
|
|
243
|
+
* { type: "text-delta", text: "Hello " },
|
|
244
|
+
* { type: "text-delta", text: "world" },
|
|
245
|
+
* { type: "finish", finishReason: "stop" },
|
|
246
|
+
* ],
|
|
247
|
+
* },
|
|
248
|
+
* });
|
|
249
|
+
*
|
|
250
|
+
* const stream = agent.stream({ prompt: "Say hello" });
|
|
251
|
+
* const chunks = await collectStreamChunks(stream);
|
|
252
|
+
*
|
|
253
|
+
* assertStreamHasText(chunks);
|
|
254
|
+
* assertStreamFinished(chunks, "stop");
|
|
255
|
+
*
|
|
256
|
+
* const text = getStreamText(chunks);
|
|
257
|
+
* expect(text).toBe("Hello world");
|
|
258
|
+
* });
|
|
259
|
+
* ```
|
|
260
|
+
*
|
|
261
|
+
* ### Pattern 8: Error Simulation
|
|
262
|
+
*
|
|
263
|
+
* Test error handling:
|
|
264
|
+
*
|
|
265
|
+
* ```typescript
|
|
266
|
+
* import { createMockAgent } from "@lleverage-ai/agent-sdk/testing";
|
|
267
|
+
*
|
|
268
|
+
* it("handles API errors gracefully", async () => {
|
|
269
|
+
* const agent = createMockAgent({
|
|
270
|
+
* response: {
|
|
271
|
+
* text: "",
|
|
272
|
+
* error: new Error("API rate limit exceeded"),
|
|
273
|
+
* },
|
|
274
|
+
* });
|
|
275
|
+
*
|
|
276
|
+
* await expect(agent.generate({ prompt: "Hi" })).rejects.toThrow("rate limit");
|
|
277
|
+
* });
|
|
278
|
+
*
|
|
279
|
+
* it("retries on transient errors", async () => {
|
|
280
|
+
* const agent = createMockAgent();
|
|
281
|
+
* agent.queueResponses(
|
|
282
|
+
* { text: "", error: new Error("Temporary failure") },
|
|
283
|
+
* { text: "", error: new Error("Temporary failure") },
|
|
284
|
+
* { text: "Success!" },
|
|
285
|
+
* );
|
|
286
|
+
*
|
|
287
|
+
* // Your retry logic would call generate multiple times
|
|
288
|
+
* let result;
|
|
289
|
+
* for (let i = 0; i < 3; i++) {
|
|
290
|
+
* try {
|
|
291
|
+
* result = await agent.generate({ prompt: "Hi" });
|
|
292
|
+
* break;
|
|
293
|
+
* } catch {
|
|
294
|
+
* // Continue retrying
|
|
295
|
+
* }
|
|
296
|
+
* }
|
|
297
|
+
*
|
|
298
|
+
* expect(result?.text).toBe("Success!");
|
|
299
|
+
* });
|
|
300
|
+
* ```
|
|
301
|
+
*
|
|
302
|
+
* ### Pattern 9: Composite Assertions
|
|
303
|
+
*
|
|
304
|
+
* Use assertAgentBehavior for comprehensive checks:
|
|
305
|
+
*
|
|
306
|
+
* ```typescript
|
|
307
|
+
* import { createMockAgent, assertAgentBehavior } from "@lleverage-ai/agent-sdk/testing";
|
|
308
|
+
*
|
|
309
|
+
* it("behaves correctly for weather query", async () => {
|
|
310
|
+
* const agent = createMockAgent({
|
|
311
|
+
* response: {
|
|
312
|
+
* text: "The weather in Tokyo is sunny and 25°C.",
|
|
313
|
+
* finishReason: "stop",
|
|
314
|
+
* usage: { inputTokens: 50, outputTokens: 20 },
|
|
315
|
+
* steps: [{
|
|
316
|
+
* text: "",
|
|
317
|
+
* toolCalls: [{ toolCallId: "1", toolName: "weather", input: { city: "Tokyo" } }],
|
|
318
|
+
* toolResults: [{ toolCallId: "1", toolName: "weather", output: "25°C sunny" }],
|
|
319
|
+
* finishReason: "tool-calls",
|
|
320
|
+
* }],
|
|
321
|
+
* },
|
|
322
|
+
* });
|
|
323
|
+
*
|
|
324
|
+
* const result = await agent.generate({ prompt: "Weather in Tokyo?" });
|
|
325
|
+
*
|
|
326
|
+
* assertAgentBehavior(result, {
|
|
327
|
+
* responseContains: ["Tokyo", "sunny"],
|
|
328
|
+
* finishReason: "stop",
|
|
329
|
+
* toolsCalled: ["weather"],
|
|
330
|
+
* toolsNotCalled: ["search", "calculator"],
|
|
331
|
+
* minSteps: 1,
|
|
332
|
+
* hasUsage: true,
|
|
333
|
+
* });
|
|
334
|
+
* });
|
|
335
|
+
* ```
|
|
336
|
+
*
|
|
337
|
+
* @packageDocumentation
|
|
338
|
+
* @module testing
|
|
339
|
+
*/
|
|
340
|
+
export { AgentAssertionError, type AgentBehaviorOptions, assertAgentBehavior, assertFinishReason, assertHasOutput, assertHasUsage, assertMockAgentCallCount, assertMockAgentCalled, assertMockAgentCalledWith, assertResponseContains, assertResponseMatches, assertStateHasFile, assertStateHasTodo, assertStepCount, assertStreamFinished, assertStreamHasText, assertStreamTextContains, assertTodoCount, assertToolCallCount, assertToolCalled, assertToolCalledWith, assertToolNotCalled, assertUsageWithin, collectStreamChunks, getStreamText, } from "./assertions.js";
|
|
341
|
+
export { createMockAgent, createMockModel, type MockAgent, type MockAgentOptions, type MockModelOptions, type MockResponse, } from "./mock-agent.js";
|
|
342
|
+
export { createPlaybackAgent, createRecording, createRecordingAgent, filterRecording, mergeRecordings, type PlaybackAgent, type PlaybackAgentOptions, parseRecording, type RecordedInteraction, type Recording, type RecordingAgent, type RecordingAgentOptions, } from "./recorder.js";
|
|
343
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkVG;AAGH,OAAO,EAEL,mBAAmB,EACnB,KAAK,oBAAoB,EAEzB,mBAAmB,EACnB,kBAAkB,EAClB,eAAe,EACf,cAAc,EACd,wBAAwB,EAExB,qBAAqB,EACrB,yBAAyB,EAEzB,sBAAsB,EACtB,qBAAqB,EAErB,kBAAkB,EAClB,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACnB,wBAAwB,EACxB,eAAe,EACf,mBAAmB,EAEnB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EAEjB,mBAAmB,EACnB,aAAa,GACd,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,eAAe,EACf,eAAe,EACf,KAAK,SAAS,EACd,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,YAAY,GAClB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,oBAAoB,EACpB,eAAe,EACf,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,cAAc,EACd,KAAK,mBAAmB,EACxB,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,qBAAqB,GAC3B,MAAM,eAAe,CAAC"}
|
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Testing utilities for the Agent SDK.
|
|
3
|
+
*
|
|
4
|
+
* This module provides comprehensive testing support including:
|
|
5
|
+
* - Mock agents with configurable responses
|
|
6
|
+
* - Recording and playback of agent interactions
|
|
7
|
+
* - Assertion helpers for common test scenarios
|
|
8
|
+
*
|
|
9
|
+
* ## Installation
|
|
10
|
+
*
|
|
11
|
+
* The testing utilities are included in the main package:
|
|
12
|
+
*
|
|
13
|
+
* ```typescript
|
|
14
|
+
* import {
|
|
15
|
+
* createMockAgent,
|
|
16
|
+
* createPlaybackAgent,
|
|
17
|
+
* assertResponseContains,
|
|
18
|
+
* } from "@lleverage-ai/agent-sdk/testing";
|
|
19
|
+
* ```
|
|
20
|
+
*
|
|
21
|
+
* ## Integration Test Patterns
|
|
22
|
+
*
|
|
23
|
+
* ### Pattern 1: Mock Agent for Unit Tests
|
|
24
|
+
*
|
|
25
|
+
* Use mock agents when testing code that depends on agent behavior:
|
|
26
|
+
*
|
|
27
|
+
* ```typescript
|
|
28
|
+
* import { createMockAgent, assertAgentBehavior } from "@lleverage-ai/agent-sdk/testing";
|
|
29
|
+
*
|
|
30
|
+
* describe("UserAssistant", () => {
|
|
31
|
+
* it("handles greeting", async () => {
|
|
32
|
+
* const agent = createMockAgent({
|
|
33
|
+
* response: { text: "Hello! How can I help?" },
|
|
34
|
+
* });
|
|
35
|
+
*
|
|
36
|
+
* const assistant = new UserAssistant(agent);
|
|
37
|
+
* const response = await assistant.greet();
|
|
38
|
+
*
|
|
39
|
+
* expect(response).toContain("Hello");
|
|
40
|
+
* expect(agent.generateCalls).toHaveLength(1);
|
|
41
|
+
* });
|
|
42
|
+
*
|
|
43
|
+
* it("handles multiple interactions", async () => {
|
|
44
|
+
* const agent = createMockAgent();
|
|
45
|
+
* agent.queueResponses(
|
|
46
|
+
* { text: "First response" },
|
|
47
|
+
* { text: "Second response" },
|
|
48
|
+
* { text: "Third response" },
|
|
49
|
+
* );
|
|
50
|
+
*
|
|
51
|
+
* const results = await Promise.all([
|
|
52
|
+
* assistant.query("1"),
|
|
53
|
+
* assistant.query("2"),
|
|
54
|
+
* assistant.query("3"),
|
|
55
|
+
* ]);
|
|
56
|
+
*
|
|
57
|
+
* expect(results[0]).toContain("First");
|
|
58
|
+
* expect(agent.getGenerateCallCount()).toBe(3);
|
|
59
|
+
* });
|
|
60
|
+
* });
|
|
61
|
+
* ```
|
|
62
|
+
*
|
|
63
|
+
* ### Pattern 2: Dynamic Responses Based on Input
|
|
64
|
+
*
|
|
65
|
+
* Use response handlers for context-aware mocking:
|
|
66
|
+
*
|
|
67
|
+
* ```typescript
|
|
68
|
+
* const agent = createMockAgent({
|
|
69
|
+
* responseHandler: (opts) => {
|
|
70
|
+
* if (opts.prompt?.includes("weather")) {
|
|
71
|
+
* return { text: "The weather is sunny!" };
|
|
72
|
+
* }
|
|
73
|
+
* if (opts.prompt?.includes("time")) {
|
|
74
|
+
* return { text: "The time is 3:00 PM." };
|
|
75
|
+
* }
|
|
76
|
+
* return { text: "I don't understand." };
|
|
77
|
+
* },
|
|
78
|
+
* });
|
|
79
|
+
*
|
|
80
|
+
* const r1 = await agent.generate({ prompt: "What's the weather?" });
|
|
81
|
+
* expect(r1.text).toContain("sunny");
|
|
82
|
+
*
|
|
83
|
+
* const r2 = await agent.generate({ prompt: "What's the time?" });
|
|
84
|
+
* expect(r2.text).toContain("3:00 PM");
|
|
85
|
+
* ```
|
|
86
|
+
*
|
|
87
|
+
* ### Pattern 3: Recording Real Interactions
|
|
88
|
+
*
|
|
89
|
+
* Record interactions from a real agent for later playback:
|
|
90
|
+
*
|
|
91
|
+
* ```typescript
|
|
92
|
+
* import { createRecordingAgent } from "@lleverage-ai/agent-sdk/testing";
|
|
93
|
+
* import { createAgent } from "@lleverage-ai/agent-sdk";
|
|
94
|
+
* import fs from "fs";
|
|
95
|
+
*
|
|
96
|
+
* // Create recording agent wrapping a real agent
|
|
97
|
+
* const realAgent = createAgent({
|
|
98
|
+
* model: anthropic("claude-sonnet-4-20250514"),
|
|
99
|
+
* systemPrompt: "You are a helpful assistant.",
|
|
100
|
+
* });
|
|
101
|
+
*
|
|
102
|
+
* const recordingAgent = createRecordingAgent(realAgent, {
|
|
103
|
+
* description: "User onboarding flow",
|
|
104
|
+
* tags: ["onboarding", "regression"],
|
|
105
|
+
* });
|
|
106
|
+
*
|
|
107
|
+
* // Make real API calls
|
|
108
|
+
* await recordingAgent.generate({ prompt: "Hello" });
|
|
109
|
+
* await recordingAgent.generate({ prompt: "What can you do?" });
|
|
110
|
+
* await recordingAgent.generate({ prompt: "Help me get started" });
|
|
111
|
+
*
|
|
112
|
+
* // Save recording for future tests
|
|
113
|
+
* const recording = recordingAgent.exportRecording();
|
|
114
|
+
* fs.writeFileSync("fixtures/onboarding.json", recording);
|
|
115
|
+
* ```
|
|
116
|
+
*
|
|
117
|
+
* ### Pattern 4: Playback for Regression Tests
|
|
118
|
+
*
|
|
119
|
+
* Replay recorded interactions for deterministic tests:
|
|
120
|
+
*
|
|
121
|
+
* ```typescript
|
|
122
|
+
* import { createPlaybackAgent, parseRecording } from "@lleverage-ai/agent-sdk/testing";
|
|
123
|
+
* import onboardingFixture from "./fixtures/onboarding.json";
|
|
124
|
+
*
|
|
125
|
+
* describe("Onboarding Flow", () => {
|
|
126
|
+
* let agent;
|
|
127
|
+
*
|
|
128
|
+
* beforeEach(() => {
|
|
129
|
+
* agent = createPlaybackAgent({
|
|
130
|
+
* recording: parseRecording(onboardingFixture),
|
|
131
|
+
* matchMode: "sequence",
|
|
132
|
+
* });
|
|
133
|
+
* });
|
|
134
|
+
*
|
|
135
|
+
* it("handles the complete onboarding flow", async () => {
|
|
136
|
+
* const r1 = await agent.generate({ prompt: "Hello" });
|
|
137
|
+
* const r2 = await agent.generate({ prompt: "What can you do?" });
|
|
138
|
+
* const r3 = await agent.generate({ prompt: "Help me get started" });
|
|
139
|
+
*
|
|
140
|
+
* expect(agent.isComplete()).toBe(true);
|
|
141
|
+
* expect(r1.text).toBeDefined();
|
|
142
|
+
* expect(r2.text).toBeDefined();
|
|
143
|
+
* expect(r3.text).toBeDefined();
|
|
144
|
+
* });
|
|
145
|
+
* });
|
|
146
|
+
* ```
|
|
147
|
+
*
|
|
148
|
+
* ### Pattern 5: Testing Tool Calls
|
|
149
|
+
*
|
|
150
|
+
* Verify that agents use tools correctly:
|
|
151
|
+
*
|
|
152
|
+
* ```typescript
|
|
153
|
+
* import {
|
|
154
|
+
* createMockAgent,
|
|
155
|
+
* assertToolCalled,
|
|
156
|
+
* assertToolCalledWith,
|
|
157
|
+
* assertToolNotCalled,
|
|
158
|
+
* } from "@lleverage-ai/agent-sdk/testing";
|
|
159
|
+
*
|
|
160
|
+
* it("uses the search tool for queries", async () => {
|
|
161
|
+
* const agent = createMockAgent({
|
|
162
|
+
* response: {
|
|
163
|
+
* text: "Found results for Tokyo",
|
|
164
|
+
* steps: [{
|
|
165
|
+
* text: "",
|
|
166
|
+
* toolCalls: [{
|
|
167
|
+
* toolCallId: "call-1",
|
|
168
|
+
* toolName: "search",
|
|
169
|
+
* input: { query: "Tokyo weather" },
|
|
170
|
+
* }],
|
|
171
|
+
* toolResults: [{
|
|
172
|
+
* toolCallId: "call-1",
|
|
173
|
+
* toolName: "search",
|
|
174
|
+
* output: "Tokyo: 25°C, sunny",
|
|
175
|
+
* }],
|
|
176
|
+
* finishReason: "tool-calls",
|
|
177
|
+
* }],
|
|
178
|
+
* },
|
|
179
|
+
* });
|
|
180
|
+
*
|
|
181
|
+
* const result = await agent.generate({ prompt: "Weather in Tokyo?" });
|
|
182
|
+
*
|
|
183
|
+
* assertToolCalled(result, "search");
|
|
184
|
+
* assertToolCalledWith(result, "search", { query: "Tokyo weather" });
|
|
185
|
+
* assertToolNotCalled(result, "dangerousTool");
|
|
186
|
+
* });
|
|
187
|
+
* ```
|
|
188
|
+
*
|
|
189
|
+
* ### Pattern 6: Testing State Changes
|
|
190
|
+
*
|
|
191
|
+
* Verify agent state modifications:
|
|
192
|
+
*
|
|
193
|
+
* ```typescript
|
|
194
|
+
* import {
|
|
195
|
+
* createMockAgent,
|
|
196
|
+
* assertStateHasFile,
|
|
197
|
+
* assertStateHasTodo,
|
|
198
|
+
* assertTodoCount,
|
|
199
|
+
* } from "@lleverage-ai/agent-sdk/testing";
|
|
200
|
+
*
|
|
201
|
+
* it("tracks todos correctly", async () => {
|
|
202
|
+
* const agent = createMockAgent({
|
|
203
|
+
* initialState: {
|
|
204
|
+
* todos: [
|
|
205
|
+
* { id: "1", content: "Fix bug", status: "pending", createdAt: new Date().toISOString() },
|
|
206
|
+
* ],
|
|
207
|
+
* },
|
|
208
|
+
* });
|
|
209
|
+
*
|
|
210
|
+
* // Simulate agent modifying state
|
|
211
|
+
* agent.state.todos[0].status = "completed";
|
|
212
|
+
* agent.state.todos.push({
|
|
213
|
+
* id: "2",
|
|
214
|
+
* content: "Write tests",
|
|
215
|
+
* status: "pending",
|
|
216
|
+
* createdAt: new Date().toISOString(),
|
|
217
|
+
* });
|
|
218
|
+
*
|
|
219
|
+
* assertTodoCount(agent.state, 2);
|
|
220
|
+
* assertTodoCount(agent.state, 1, "completed");
|
|
221
|
+
* assertStateHasTodo(agent.state, { content: "Fix bug", status: "completed" });
|
|
222
|
+
* });
|
|
223
|
+
* ```
|
|
224
|
+
*
|
|
225
|
+
* ### Pattern 7: Testing Streaming
|
|
226
|
+
*
|
|
227
|
+
* Test streaming responses:
|
|
228
|
+
*
|
|
229
|
+
* ```typescript
|
|
230
|
+
* import {
|
|
231
|
+
* createMockAgent,
|
|
232
|
+
* collectStreamChunks,
|
|
233
|
+
* assertStreamHasText,
|
|
234
|
+
* assertStreamFinished,
|
|
235
|
+
* getStreamText,
|
|
236
|
+
* } from "@lleverage-ai/agent-sdk/testing";
|
|
237
|
+
*
|
|
238
|
+
* it("streams response correctly", async () => {
|
|
239
|
+
* const agent = createMockAgent({
|
|
240
|
+
* response: {
|
|
241
|
+
* text: "Hello world",
|
|
242
|
+
* streamChunks: [
|
|
243
|
+
* { type: "text-delta", text: "Hello " },
|
|
244
|
+
* { type: "text-delta", text: "world" },
|
|
245
|
+
* { type: "finish", finishReason: "stop" },
|
|
246
|
+
* ],
|
|
247
|
+
* },
|
|
248
|
+
* });
|
|
249
|
+
*
|
|
250
|
+
* const stream = agent.stream({ prompt: "Say hello" });
|
|
251
|
+
* const chunks = await collectStreamChunks(stream);
|
|
252
|
+
*
|
|
253
|
+
* assertStreamHasText(chunks);
|
|
254
|
+
* assertStreamFinished(chunks, "stop");
|
|
255
|
+
*
|
|
256
|
+
* const text = getStreamText(chunks);
|
|
257
|
+
* expect(text).toBe("Hello world");
|
|
258
|
+
* });
|
|
259
|
+
* ```
|
|
260
|
+
*
|
|
261
|
+
* ### Pattern 8: Error Simulation
|
|
262
|
+
*
|
|
263
|
+
* Test error handling:
|
|
264
|
+
*
|
|
265
|
+
* ```typescript
|
|
266
|
+
* import { createMockAgent } from "@lleverage-ai/agent-sdk/testing";
|
|
267
|
+
*
|
|
268
|
+
* it("handles API errors gracefully", async () => {
|
|
269
|
+
* const agent = createMockAgent({
|
|
270
|
+
* response: {
|
|
271
|
+
* text: "",
|
|
272
|
+
* error: new Error("API rate limit exceeded"),
|
|
273
|
+
* },
|
|
274
|
+
* });
|
|
275
|
+
*
|
|
276
|
+
* await expect(agent.generate({ prompt: "Hi" })).rejects.toThrow("rate limit");
|
|
277
|
+
* });
|
|
278
|
+
*
|
|
279
|
+
* it("retries on transient errors", async () => {
|
|
280
|
+
* const agent = createMockAgent();
|
|
281
|
+
* agent.queueResponses(
|
|
282
|
+
* { text: "", error: new Error("Temporary failure") },
|
|
283
|
+
* { text: "", error: new Error("Temporary failure") },
|
|
284
|
+
* { text: "Success!" },
|
|
285
|
+
* );
|
|
286
|
+
*
|
|
287
|
+
* // Your retry logic would call generate multiple times
|
|
288
|
+
* let result;
|
|
289
|
+
* for (let i = 0; i < 3; i++) {
|
|
290
|
+
* try {
|
|
291
|
+
* result = await agent.generate({ prompt: "Hi" });
|
|
292
|
+
* break;
|
|
293
|
+
* } catch {
|
|
294
|
+
* // Continue retrying
|
|
295
|
+
* }
|
|
296
|
+
* }
|
|
297
|
+
*
|
|
298
|
+
* expect(result?.text).toBe("Success!");
|
|
299
|
+
* });
|
|
300
|
+
* ```
|
|
301
|
+
*
|
|
302
|
+
* ### Pattern 9: Composite Assertions
|
|
303
|
+
*
|
|
304
|
+
* Use assertAgentBehavior for comprehensive checks:
|
|
305
|
+
*
|
|
306
|
+
* ```typescript
|
|
307
|
+
* import { createMockAgent, assertAgentBehavior } from "@lleverage-ai/agent-sdk/testing";
|
|
308
|
+
*
|
|
309
|
+
* it("behaves correctly for weather query", async () => {
|
|
310
|
+
* const agent = createMockAgent({
|
|
311
|
+
* response: {
|
|
312
|
+
* text: "The weather in Tokyo is sunny and 25°C.",
|
|
313
|
+
* finishReason: "stop",
|
|
314
|
+
* usage: { inputTokens: 50, outputTokens: 20 },
|
|
315
|
+
* steps: [{
|
|
316
|
+
* text: "",
|
|
317
|
+
* toolCalls: [{ toolCallId: "1", toolName: "weather", input: { city: "Tokyo" } }],
|
|
318
|
+
* toolResults: [{ toolCallId: "1", toolName: "weather", output: "25°C sunny" }],
|
|
319
|
+
* finishReason: "tool-calls",
|
|
320
|
+
* }],
|
|
321
|
+
* },
|
|
322
|
+
* });
|
|
323
|
+
*
|
|
324
|
+
* const result = await agent.generate({ prompt: "Weather in Tokyo?" });
|
|
325
|
+
*
|
|
326
|
+
* assertAgentBehavior(result, {
|
|
327
|
+
* responseContains: ["Tokyo", "sunny"],
|
|
328
|
+
* finishReason: "stop",
|
|
329
|
+
* toolsCalled: ["weather"],
|
|
330
|
+
* toolsNotCalled: ["search", "calculator"],
|
|
331
|
+
* minSteps: 1,
|
|
332
|
+
* hasUsage: true,
|
|
333
|
+
* });
|
|
334
|
+
* });
|
|
335
|
+
* ```
|
|
336
|
+
*
|
|
337
|
+
* @packageDocumentation
|
|
338
|
+
* @module testing
|
|
339
|
+
*/
|
|
340
|
+
// Assertions
|
|
341
|
+
export {
|
|
342
|
+
// Error class
|
|
343
|
+
AgentAssertionError,
|
|
344
|
+
// Composite assertions
|
|
345
|
+
assertAgentBehavior, assertFinishReason, assertHasOutput, assertHasUsage, assertMockAgentCallCount,
|
|
346
|
+
// Mock agent assertions
|
|
347
|
+
assertMockAgentCalled, assertMockAgentCalledWith,
|
|
348
|
+
// Response assertions
|
|
349
|
+
assertResponseContains, assertResponseMatches,
|
|
350
|
+
// State assertions
|
|
351
|
+
assertStateHasFile, assertStateHasTodo, assertStepCount, assertStreamFinished, assertStreamHasText, assertStreamTextContains, assertTodoCount, assertToolCallCount,
|
|
352
|
+
// Tool assertions
|
|
353
|
+
assertToolCalled, assertToolCalledWith, assertToolNotCalled, assertUsageWithin,
|
|
354
|
+
// Stream assertions
|
|
355
|
+
collectStreamChunks, getStreamText, } from "./assertions.js";
|
|
356
|
+
// Mock Agent
|
|
357
|
+
export { createMockAgent, createMockModel, } from "./mock-agent.js";
|
|
358
|
+
// Recording and Playback
|
|
359
|
+
export { createPlaybackAgent, createRecording, createRecordingAgent, filterRecording, mergeRecordings, parseRecording, } from "./recorder.js";
|
|
360
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkVG;AAEH,aAAa;AACb,OAAO;AACL,cAAc;AACd,mBAAmB;AAEnB,uBAAuB;AACvB,mBAAmB,EACnB,kBAAkB,EAClB,eAAe,EACf,cAAc,EACd,wBAAwB;AACxB,wBAAwB;AACxB,qBAAqB,EACrB,yBAAyB;AACzB,sBAAsB;AACtB,sBAAsB,EACtB,qBAAqB;AACrB,mBAAmB;AACnB,kBAAkB,EAClB,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACnB,wBAAwB,EACxB,eAAe,EACf,mBAAmB;AACnB,kBAAkB;AAClB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB;AACjB,oBAAoB;AACpB,mBAAmB,EACnB,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,aAAa;AACb,OAAO,EACL,eAAe,EACf,eAAe,GAKhB,MAAM,iBAAiB,CAAC;AACzB,yBAAyB;AACzB,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,oBAAoB,EACpB,eAAe,EACf,eAAe,EAGf,cAAc,GAKf,MAAM,eAAe,CAAC"}
|