@amplitude/ai 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/instrument-with-amplitude-ai.md +12 -0
- package/.cursor/skills/instrument-with-amplitude-ai/SKILL.md +4 -42
- package/AGENTS.md +86 -28
- package/README.md +190 -111
- package/amplitude-ai.md +463 -0
- package/bin/amplitude-ai-doctor.mjs +0 -0
- package/bin/amplitude-ai-instrument.mjs +0 -0
- package/bin/amplitude-ai-mcp.mjs +0 -0
- package/bin/amplitude-ai-register-catalog.mjs +0 -0
- package/bin/amplitude-ai-status.mjs +0 -0
- package/bin/amplitude-ai.mjs +14 -5
- package/data/agent_event_catalog.json +52 -2
- package/dist/bound-agent.d.ts +18 -14
- package/dist/bound-agent.d.ts.map +1 -1
- package/dist/bound-agent.js +4 -1
- package/dist/bound-agent.js.map +1 -1
- package/dist/cli/status.d.ts.map +1 -1
- package/dist/cli/status.js +31 -19
- package/dist/cli/status.js.map +1 -1
- package/dist/client.d.ts +14 -14
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +38 -0
- package/dist/client.js.map +1 -1
- package/dist/context.d.ts +5 -0
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +4 -0
- package/dist/context.js.map +1 -1
- package/dist/core/constants.d.ts +3 -1
- package/dist/core/constants.d.ts.map +1 -1
- package/dist/core/constants.js +3 -1
- package/dist/core/constants.js.map +1 -1
- package/dist/core/tracking.d.ts +12 -2
- package/dist/core/tracking.d.ts.map +1 -1
- package/dist/core/tracking.js +13 -2
- package/dist/core/tracking.js.map +1 -1
- package/dist/decorators.d.ts +1 -1
- package/dist/decorators.d.ts.map +1 -1
- package/dist/decorators.js +4 -3
- package/dist/decorators.js.map +1 -1
- package/dist/index.d.ts +7 -4
- package/dist/index.js +5 -2
- package/dist/mcp/contract.d.ts +4 -0
- package/dist/mcp/contract.d.ts.map +1 -1
- package/dist/mcp/contract.js +6 -2
- package/dist/mcp/contract.js.map +1 -1
- package/dist/mcp/generate-verify-test.d.ts +7 -0
- package/dist/mcp/generate-verify-test.d.ts.map +1 -0
- package/dist/mcp/generate-verify-test.js +25 -0
- package/dist/mcp/generate-verify-test.js.map +1 -0
- package/dist/mcp/index.d.ts +2 -1
- package/dist/mcp/index.js +2 -1
- package/dist/mcp/instrument-file.d.ts +14 -0
- package/dist/mcp/instrument-file.d.ts.map +1 -0
- package/dist/mcp/instrument-file.js +139 -0
- package/dist/mcp/instrument-file.js.map +1 -0
- package/dist/mcp/scan-project.d.ts +52 -0
- package/dist/mcp/scan-project.d.ts.map +1 -0
- package/dist/mcp/scan-project.js +309 -0
- package/dist/mcp/scan-project.js.map +1 -0
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +79 -4
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/validate-file.d.ts +4 -0
- package/dist/mcp/validate-file.d.ts.map +1 -1
- package/dist/mcp/validate-file.js +559 -11
- package/dist/mcp/validate-file.js.map +1 -1
- package/dist/middleware.js +2 -1
- package/dist/middleware.js.map +1 -1
- package/dist/node_modules/.pnpm/acorn-typescript@1.4.13_acorn@8.16.0/node_modules/acorn-typescript/lib/index.js +2389 -0
- package/dist/node_modules/.pnpm/acorn-typescript@1.4.13_acorn@8.16.0/node_modules/acorn-typescript/lib/index.js.map +1 -0
- package/dist/node_modules/.pnpm/acorn@8.16.0/node_modules/acorn/dist/acorn.js +5128 -0
- package/dist/node_modules/.pnpm/acorn@8.16.0/node_modules/acorn/dist/acorn.js.map +1 -0
- package/dist/node_modules/.pnpm/zod@3.25.76/node_modules/zod/v3/types.js +1 -1
- package/dist/patching.d.ts.map +1 -1
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +1 -0
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/base.d.ts +2 -1
- package/dist/providers/base.d.ts.map +1 -1
- package/dist/providers/base.js +4 -0
- package/dist/providers/base.js.map +1 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +2 -0
- package/dist/providers/openai.js.map +1 -1
- package/dist/serverless.d.ts +19 -0
- package/dist/serverless.d.ts.map +1 -0
- package/dist/serverless.js +35 -0
- package/dist/serverless.js.map +1 -0
- package/dist/session.d.ts +24 -8
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +20 -1
- package/dist/session.js.map +1 -1
- package/dist/types.d.ts +1 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/llms-full.txt +353 -69
- package/llms.txt +6 -2
- package/mcp.schema.json +7 -3
- package/package.json +10 -5
- package/bin/amplitude-ai-init.mjs +0 -27
- package/dist/cli/init.d.ts +0 -14
- package/dist/cli/init.d.ts.map +0 -1
- package/dist/cli/init.js +0 -40
- package/dist/cli/init.js.map +0 -1
package/amplitude-ai.md
ADDED
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
# Instrument with @amplitude/ai
|
|
2
|
+
|
|
3
|
+
Auto-instrument a JS/TS AI app with `@amplitude/ai` in 4 phases: **Detect → Discover → Instrument → Verify**. The result is a fully instrumented app with provider wrappers, session lifecycle, multi-agent delegation (when detected), and a verification test proving correctness — all before deploying anything.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Phase 1: Detect Environment
|
|
8
|
+
|
|
9
|
+
1. Read `package.json` for dependencies
|
|
10
|
+
2. Detect framework: `next` → Next.js, `express` → Express, `fastify` → Fastify, `hono` → Hono
|
|
11
|
+
3. Detect LLM providers: `openai`, `@anthropic-ai/sdk`, `@google/generative-ai`, `@aws-sdk/client-bedrock-runtime`, `@mistralai/mistralai`
|
|
12
|
+
4. Detect agent frameworks: `langchain`, `@langchain/core`, `llamaindex`, `@openai/agents`, `crewai`
|
|
13
|
+
5. Detect existing instrumentation: `@amplitude/ai` in deps, `patch({` or `AmplitudeAI` in source
|
|
14
|
+
6. Check for multi-agent signals: multiple files with LLM calls, tool definitions that call other LLM-calling functions, delegation patterns
|
|
15
|
+
7. Check for streaming: `stream: true` in provider calls
|
|
16
|
+
8. Check for frontend deps: `react`, `vue`, `svelte` in deps
|
|
17
|
+
9. Check for Vercel AI SDK: `@ai-sdk/*` in deps
|
|
18
|
+
10. Check for edge runtime: `runtime = 'edge'` in route files
|
|
19
|
+
|
|
20
|
+
**Output to the developer:**
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
Detected environment:
|
|
24
|
+
Runtime: Node.js (TypeScript)
|
|
25
|
+
Framework: [framework or "none"]
|
|
26
|
+
Providers: [list]
|
|
27
|
+
Agent frameworks: [list or "none"]
|
|
28
|
+
Existing instrumentation: [yes/no]
|
|
29
|
+
Multi-agent signals: [yes/no]
|
|
30
|
+
Streaming: [yes/no]
|
|
31
|
+
Frontend deps: [yes/no]
|
|
32
|
+
Recommended tier: [quick_start / standard / advanced]
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
**Decision point:** Ask the developer to confirm the detection and choose a tier:
|
|
36
|
+
- **Quick start** — `patch({ amplitudeAI: ai })`, zero code changes, good for getting data flowing
|
|
37
|
+
- **Standard** — Provider wrappers + session middleware (recommended for most apps)
|
|
38
|
+
- **Advanced** — Multi-agent `runAs`, agent descriptions, scoring, tool tracking
|
|
39
|
+
|
|
40
|
+
If multi-agent signals are detected, recommend Advanced.
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## Phase 2: Discover Agents and Call Sites
|
|
45
|
+
|
|
46
|
+
For **Quick start** tier, skip to Phase 3 — discovery is just "which providers are imported."
|
|
47
|
+
|
|
48
|
+
For **Standard** and **Advanced** tiers:
|
|
49
|
+
|
|
50
|
+
1. Identify files with LLM call sites (search for `chat.completions.create`, `messages.create`, `generateContent`, `streamText`, `generateText`)
|
|
51
|
+
2. For each file with call sites, read the actual source and review:
|
|
52
|
+
- Is it a route handler / API endpoint?
|
|
53
|
+
- What provider(s) does it use?
|
|
54
|
+
- Does it call other files with LLM call sites? (delegation → multi-agent)
|
|
55
|
+
3. For **Advanced** tier, also identify:
|
|
56
|
+
- Agent boundaries (each distinct orchestration unit = one agent)
|
|
57
|
+
- Delegation patterns (parent calls child → `runAs`)
|
|
58
|
+
- Feedback handlers (thumbs up/down UI components)
|
|
59
|
+
- Tool functions (functions called by the LLM via function calling)
|
|
60
|
+
4. For each event emission you plan to add, trace **all code paths** that should emit the same event type. Look for error handlers, retry-exhaustion paths, timeout handlers, and fallback branches that represent the same logical operation failing — these should also emit the event (typically with `success: false` or an `errorMessage`).
|
|
61
|
+
|
|
62
|
+
### Multi-Agent Detection
|
|
63
|
+
|
|
64
|
+
Look for these patterns by reading the source files:
|
|
65
|
+
1. Do any tool functions call other functions that make LLM calls? → **delegation-as-tools** (A2A) pattern
|
|
66
|
+
2. Does a parent function call a function in another file that has LLM call sites? → **sequential delegation**
|
|
67
|
+
3. Are there multiple distinct agent roles or personas with separate system prompts?
|
|
68
|
+
4. Identify **orchestration wrappers** — functions that invoke sub-agents and should measure their execution. These are candidates for `observe()` / `trackSpan()` to capture delegation latency, input/output summaries, and error status. Look for: try/catch blocks around sub-agent calls, functions that dispatch to multiple agents in sequence or parallel, and any function that measures duration of a delegated operation.
|
|
69
|
+
|
|
70
|
+
Only mark the architecture as multi-agent once you've confirmed one of these patterns by reading the source.
|
|
71
|
+
|
|
72
|
+
**Output to the developer:**
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
Found N agents across M files:
|
|
76
|
+
|
|
77
|
+
Agent 1: "chat-handler"
|
|
78
|
+
Description: "Handles user chat requests via streaming OpenAI GPT-4o"
|
|
79
|
+
File: src/app/api/chat/route.ts
|
|
80
|
+
Provider: OpenAI (chat.completions.create)
|
|
81
|
+
Entry point: POST /api/chat
|
|
82
|
+
|
|
83
|
+
Agent 2: "recipe-agent" (child of chat-handler, called as a tool)
|
|
84
|
+
Description: "Specialized recipe planning agent called by the orchestrator"
|
|
85
|
+
File: src/lib/recipe-agent.ts
|
|
86
|
+
Provider: OpenAI (chat.completions.create)
|
|
87
|
+
Delegation: ask_recipe_agent() tool in Agent 1 delegates to this file
|
|
88
|
+
|
|
89
|
+
Multi-agent architecture: delegation-as-tools (A2A)
|
|
90
|
+
→ will instrument with ai.agent().child() + session.runAs()
|
|
91
|
+
|
|
92
|
+
Proceed with instrumentation? [Review changes first / Apply / Skip]
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
**PAUSE HERE.** Let the developer review the agent names, descriptions, and structure before proceeding. They can edit names and descriptions.
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
## Phase 3: Instrument
|
|
100
|
+
|
|
101
|
+
### Step 3a: Install dependencies
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
pnpm add @amplitude/ai # or npm install / yarn add
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
### Step 3b: Create bootstrap file
|
|
108
|
+
|
|
109
|
+
Create `src/lib/amplitude.ts` (or the project's conventional lib path):
|
|
110
|
+
|
|
111
|
+
**Choose `contentMode` based on privacy needs:**
|
|
112
|
+
|
|
113
|
+
- **`full`** — captures full prompt/response text. Best for debugging and enrichment. Always pair with `redactPii: true` unless the customer has explicit consent.
|
|
114
|
+
- **`metadata_only`** — captures token counts, latency, model, cost, but no text. Use for sensitive PII or regulated data.
|
|
115
|
+
- **`customer_enriched`** — no text captured by default, but the customer can send enriched summaries via `trackSessionEnrichment()`.
|
|
116
|
+
|
|
117
|
+
```typescript
|
|
118
|
+
import { AmplitudeAI, AIConfig, OpenAI } from '@amplitude/ai';
|
|
119
|
+
|
|
120
|
+
export const ai = new AmplitudeAI({
|
|
121
|
+
apiKey: process.env.AMPLITUDE_AI_API_KEY!,
|
|
122
|
+
config: new AIConfig({
|
|
123
|
+
contentMode: 'full',
|
|
124
|
+
redactPii: true,
|
|
125
|
+
}),
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
// One wrapped client per provider detected in Phase 1
|
|
129
|
+
export const openai = new OpenAI({
|
|
130
|
+
apiKey: process.env.OPENAI_API_KEY!,
|
|
131
|
+
amplitude: ai,
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
// Add more providers as detected:
|
|
135
|
+
// export const anthropic = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY!, amplitude: ai });
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
**Alternative: `wrap()` for existing clients.** If the project creates provider clients dynamically:
|
|
139
|
+
|
|
140
|
+
```typescript
|
|
141
|
+
import { wrap } from '@amplitude/ai';
|
|
142
|
+
import OpenAI from 'openai';
|
|
143
|
+
const rawClient = new OpenAI({ apiKey: process.env.OPENAI_API_KEY! });
|
|
144
|
+
export const openai = wrap(rawClient, ai);
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Add `AMPLITUDE_AI_API_KEY` to `.env.example`. Check `.gitignore` includes `.env`.
|
|
148
|
+
|
|
149
|
+
### Step 3c: Swap provider imports
|
|
150
|
+
|
|
151
|
+
Replace direct provider instantiation with imports from the bootstrap file:
|
|
152
|
+
|
|
153
|
+
**Before:**
|
|
154
|
+
```typescript
|
|
155
|
+
import OpenAI from 'openai';
|
|
156
|
+
const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
**After:**
|
|
160
|
+
```typescript
|
|
161
|
+
import { openai as client } from '@/lib/amplitude';
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### Step 3d: Add session context
|
|
165
|
+
|
|
166
|
+
**Standard tier** — wrap route handlers with agent + session:
|
|
167
|
+
|
|
168
|
+
```typescript
|
|
169
|
+
import { ai } from '@/lib/amplitude';
|
|
170
|
+
const agent = ai.agent('chat-handler', {
|
|
171
|
+
description: 'Handles user chat requests via streaming OpenAI GPT-4o',
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
export async function POST(req: Request) {
|
|
175
|
+
const { messages, userId } = await req.json();
|
|
176
|
+
return agent.session({ userId }).run(async (s) => {
|
|
177
|
+
s.trackUserMessage(messages[messages.length - 1].content);
|
|
178
|
+
const response = await client.chat.completions.create({ model: 'gpt-4o', messages });
|
|
179
|
+
return Response.json(response);
|
|
180
|
+
});
|
|
181
|
+
// session.run() auto-flushes in serverless (Vercel, Lambda, Netlify, etc.)
|
|
182
|
+
// For non-serverless, or tracking outside session.run(), call: await ai.flush()
|
|
183
|
+
}
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
**Advanced tier** — add multi-agent delegation with `runAs`:
|
|
187
|
+
|
|
188
|
+
```typescript
|
|
189
|
+
const orchestrator = ai.agent('shopping-agent', { description: 'Orchestrates shopping requests' });
|
|
190
|
+
const recipeAgent = orchestrator.child('recipe-agent', { description: 'Finds recipes' });
|
|
191
|
+
|
|
192
|
+
// Inside parent's session.run():
|
|
193
|
+
const result = await s.runAs(recipeAgent, async (cs) => {
|
|
194
|
+
cs.trackUserMessage(delegatedQuery);
|
|
195
|
+
return openai.chat.completions.create({ model: 'gpt-4o', messages: [...] });
|
|
196
|
+
});
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
### Step 3e: Track tools and explicit AI responses
|
|
200
|
+
|
|
201
|
+
**Tool tracking** with the `tool()` higher-order function:
|
|
202
|
+
|
|
203
|
+
```typescript
|
|
204
|
+
import { tool } from '@amplitude/ai';
|
|
205
|
+
|
|
206
|
+
const searchProducts = tool(searchDB, { name: 'search_products' });
|
|
207
|
+
|
|
208
|
+
// Inside session.run(), just call the wrapped function:
|
|
209
|
+
const result = await searchProducts(query);
|
|
210
|
+
// [Agent] Tool Call event automatically emitted with duration, success, input/output
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
**Explicit AI response capture** (when provider wrappers can't auto-capture):
|
|
214
|
+
|
|
215
|
+
```typescript
|
|
216
|
+
s.trackAiMessage(completedMessage.content, 'gpt-4o', 'openai', latencyMs, {
|
|
217
|
+
inputTokens: usage.prompt_tokens,
|
|
218
|
+
outputTokens: usage.completion_tokens,
|
|
219
|
+
totalTokens: usage.total_tokens,
|
|
220
|
+
});
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
### Step 3f: Add span tracking for orchestration (Advanced tier)
|
|
224
|
+
|
|
225
|
+
When a parent agent delegates work to a child agent, wrap the delegation call with span tracking to measure latency and capture errors. Look for existing try/catch blocks around sub-agent execution — these are natural places to add span tracking with both success and error paths:
|
|
226
|
+
|
|
227
|
+
```typescript
|
|
228
|
+
import { observe } from '@amplitude/ai';
|
|
229
|
+
|
|
230
|
+
// Option A: higher-order function on orchestration functions
|
|
231
|
+
const runSubAgent = observe(async (prompt: string) => {
|
|
232
|
+
return await subAgent.execute(prompt);
|
|
233
|
+
}, { name: 'sub-agent-execution' });
|
|
234
|
+
|
|
235
|
+
// Option B: explicit tracking when you need more control
|
|
236
|
+
const start = Date.now();
|
|
237
|
+
try {
|
|
238
|
+
const result = await subAgent.execute(prompt);
|
|
239
|
+
s.trackSpan({
|
|
240
|
+
name: childAgentName,
|
|
241
|
+
latencyMs: Date.now() - start,
|
|
242
|
+
inputState: { prompt: prompt.slice(0, 1000) },
|
|
243
|
+
outputState: { response: result.slice(0, 1000) },
|
|
244
|
+
});
|
|
245
|
+
} catch (e) {
|
|
246
|
+
s.trackSpan({
|
|
247
|
+
name: childAgentName,
|
|
248
|
+
latencyMs: Date.now() - start,
|
|
249
|
+
isError: true,
|
|
250
|
+
errorMessage: `${(e as Error).name}: ${(e as Error).message}`,
|
|
251
|
+
});
|
|
252
|
+
throw e;
|
|
253
|
+
}
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
### Step 3g: Add scoring (Advanced tier only)
|
|
257
|
+
|
|
258
|
+
If feedback handlers were detected (thumbs up/down UI), check whether the handler receives a `messageId` — if so, target the specific message for finer-grained scoring. Otherwise fall back to session-level scoring:
|
|
259
|
+
|
|
260
|
+
```typescript
|
|
261
|
+
const targetId = messageId ?? sessionId;
|
|
262
|
+
const targetType = messageId ? 'message' : 'session';
|
|
263
|
+
ai.score({
|
|
264
|
+
userId, name: 'user-feedback', value: thumbsUp ? 1 : 0,
|
|
265
|
+
targetId, targetType, source: 'user',
|
|
266
|
+
sessionId, comment: feedbackText,
|
|
267
|
+
});
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
### Step 3h: Streaming session lifecycle
|
|
271
|
+
|
|
272
|
+
If the app uses streaming, the session must stay open until the stream is fully consumed:
|
|
273
|
+
|
|
274
|
+
```typescript
|
|
275
|
+
// WRONG: session ends before stream is consumed
|
|
276
|
+
return agent.session({ userId }).run(async (s) => {
|
|
277
|
+
const stream = await openai.chat.completions.create({ model: 'gpt-4o', messages, stream: true });
|
|
278
|
+
return new Response(stream.toReadableStream());
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
// CORRECT: session stays open until stream completes
|
|
282
|
+
return agent.session({ userId }).run(async (s) => {
|
|
283
|
+
const stream = await openai.chat.completions.create({ model: 'gpt-4o', messages, stream: true });
|
|
284
|
+
const readable = stream.toReadableStream();
|
|
285
|
+
const [passthrough, forClient] = readable.tee();
|
|
286
|
+
const reader = passthrough.getReader();
|
|
287
|
+
(async () => { while (!(await reader.read()).done) {} })();
|
|
288
|
+
return new Response(forClient);
|
|
289
|
+
});
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
### Step 3i: Browser-server session linking
|
|
293
|
+
|
|
294
|
+
If frontend deps were detected, extract browser IDs from request headers:
|
|
295
|
+
|
|
296
|
+
```typescript
|
|
297
|
+
const browserSessionId = req.headers.get('x-amplitude-session-id');
|
|
298
|
+
const deviceId = req.headers.get('x-amplitude-device-id');
|
|
299
|
+
const session = agent.session({ userId, browserSessionId, deviceId });
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
### Step 3j: Framework-specific notes
|
|
303
|
+
|
|
304
|
+
**Next.js App Router**: Session wrapping goes inside each route handler. Add `@amplitude/ai` to `serverExternalPackages` in `next.config.ts`.
|
|
305
|
+
|
|
306
|
+
**Express/Fastify/Hono**: Use middleware:
|
|
307
|
+
```typescript
|
|
308
|
+
import { createAmplitudeAIMiddleware } from '@amplitude/ai';
|
|
309
|
+
app.use(createAmplitudeAIMiddleware({
|
|
310
|
+
amplitudeAI: ai,
|
|
311
|
+
userIdResolver: (req) => req.headers['x-user-id'] ?? null,
|
|
312
|
+
}));
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
---
|
|
316
|
+
|
|
317
|
+
## Phase 4: Verify
|
|
318
|
+
|
|
319
|
+
### Step 4a: Create verification test
|
|
320
|
+
|
|
321
|
+
Create `__amplitude_verify__.test.ts` that verifies:
|
|
322
|
+
- Each agent emits `[Agent] User Message` with correct `[Agent] Agent ID`
|
|
323
|
+
- Sessions are properly closed (`assertSessionClosed`)
|
|
324
|
+
- Multi-agent delegation preserves session ID across `runAs`
|
|
325
|
+
|
|
326
|
+
```typescript
|
|
327
|
+
import { AIConfig, tool } from '@amplitude/ai';
|
|
328
|
+
import { MockAmplitudeAI } from '@amplitude/ai/testing';
|
|
329
|
+
|
|
330
|
+
const mock = new MockAmplitudeAI(new AIConfig({ contentMode: 'full' }));
|
|
331
|
+
const agent = mock.agent('test-agent', { userId: 'u1' });
|
|
332
|
+
|
|
333
|
+
await agent.session({ sessionId: 's1' }).run(async (s) => {
|
|
334
|
+
s.trackUserMessage('hello');
|
|
335
|
+
s.trackAiMessage('response', 'gpt-4o-mini', 'openai', 150);
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
mock.assertEventTracked('[Agent] User Message', { userId: 'u1' });
|
|
339
|
+
mock.assertSessionClosed('s1');
|
|
340
|
+
|
|
341
|
+
// For multi-agent: verify child agent events
|
|
342
|
+
mock.eventsForAgent('child-agent-id'); // filter by agent
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
### Step 4b: Run verification
|
|
346
|
+
|
|
347
|
+
```bash
|
|
348
|
+
npx vitest run __amplitude_verify__.test.ts
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
### Step 4c: Run doctor
|
|
352
|
+
|
|
353
|
+
```bash
|
|
354
|
+
npx amplitude-ai doctor
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
### Step 4d: Run project checks
|
|
358
|
+
|
|
359
|
+
```bash
|
|
360
|
+
npx tsc --noEmit # TypeScript compiles
|
|
361
|
+
npm test # Existing tests still pass
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
### Step 4e: Show confidence report
|
|
365
|
+
|
|
366
|
+
```
|
|
367
|
+
Verification complete:
|
|
368
|
+
Doctor checks: 5/5 passed
|
|
369
|
+
Event sequence test: PASSED (N events captured)
|
|
370
|
+
TypeScript check: PASSED
|
|
371
|
+
Existing tests: PASSED
|
|
372
|
+
|
|
373
|
+
Content mode: full (PII redacted)
|
|
374
|
+
|
|
375
|
+
Next steps:
|
|
376
|
+
1. Set AMPLITUDE_AI_API_KEY in your environment
|
|
377
|
+
2. Keep __amplitude_verify__.test.ts for CI regression testing
|
|
378
|
+
3. Deploy and verify live events in Amplitude
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
---
|
|
382
|
+
|
|
383
|
+
## API Quick Reference
|
|
384
|
+
|
|
385
|
+
### Core Classes
|
|
386
|
+
|
|
387
|
+
| API | What it does |
|
|
388
|
+
|-----|-------------|
|
|
389
|
+
| `new AmplitudeAI({ apiKey, config? })` | Initialize SDK |
|
|
390
|
+
| `new AIConfig({ contentMode?, redactPii?, debug? })` | Privacy/debug config |
|
|
391
|
+
| `ai.agent(agentId, opts?)` | Create bound agent |
|
|
392
|
+
| `agent.child(agentId, opts?)` | Create child agent |
|
|
393
|
+
| `agent.session(opts?)` | Create session (`autoFlush` auto-detects serverless) |
|
|
394
|
+
| `session.run(fn)` | Execute with session context (auto-flushes in serverless) |
|
|
395
|
+
| `s.runAs(childAgent, fn)` | Delegate to child agent |
|
|
396
|
+
| `ai.flush()` | Flush events (serverless) |
|
|
397
|
+
|
|
398
|
+
### Tracking Methods (on session `s`)
|
|
399
|
+
|
|
400
|
+
| Method | Event Emitted |
|
|
401
|
+
|--------|--------------|
|
|
402
|
+
| `s.trackUserMessage(content)` | `[Agent] User Message` |
|
|
403
|
+
| `s.trackAiMessage(content, model, provider, latencyMs)` | `[Agent] AI Response` |
|
|
404
|
+
| `s.trackToolCall(toolName, latencyMs, success)` | `[Agent] Tool Call` |
|
|
405
|
+
| `s.score(name, value, targetId)` | `[Agent] Score` |
|
|
406
|
+
|
|
407
|
+
### Higher-Order Functions
|
|
408
|
+
|
|
409
|
+
| HOF | Event Emitted | Usage |
|
|
410
|
+
|-----|--------------|-------|
|
|
411
|
+
| `tool(fn, { name })` | `[Agent] Tool Call` | Wrap tool functions |
|
|
412
|
+
| `observe(fn, { name })` | `[Agent] Span` | Wrap any function for observability |
|
|
413
|
+
|
|
414
|
+
### Provider Wrappers
|
|
415
|
+
|
|
416
|
+
All imported from `@amplitude/ai`:
|
|
417
|
+
|
|
418
|
+
| Provider | Constructor |
|
|
419
|
+
|----------|------------|
|
|
420
|
+
| OpenAI | `new OpenAI({ apiKey, amplitude: ai })` |
|
|
421
|
+
| Anthropic | `new Anthropic({ apiKey, amplitude: ai })` |
|
|
422
|
+
| Gemini | `new Gemini({ apiKey, amplitude: ai })` |
|
|
423
|
+
| AzureOpenAI | `new AzureOpenAI({ apiKey, amplitude: ai })` |
|
|
424
|
+
| Bedrock | `new Bedrock({ amplitude: ai })` |
|
|
425
|
+
| Mistral | `new Mistral({ apiKey, amplitude: ai })` |
|
|
426
|
+
|
|
427
|
+
### Other APIs
|
|
428
|
+
|
|
429
|
+
| API | Usage |
|
|
430
|
+
|-----|-------|
|
|
431
|
+
| `patch({ amplitudeAI: ai })` / `unpatch()` | Zero-code instrumentation |
|
|
432
|
+
| `wrap(client, ai)` | Wrap existing provider client |
|
|
433
|
+
| `injectContext()` / `extractContext(headers)` | Cross-service propagation |
|
|
434
|
+
| `createAmplitudeAIMiddleware(opts)` | Express/Fastify/Hono middleware |
|
|
435
|
+
| `MockAmplitudeAI` (from `@amplitude/ai/testing`) | Deterministic test double |
|
|
436
|
+
|
|
437
|
+
---
|
|
438
|
+
|
|
439
|
+
## Ecosystem-Specific Guidance
|
|
440
|
+
|
|
441
|
+
### Vercel AI SDK
|
|
442
|
+
- Provider wrappers instrument the underlying SDK (`openai`), not the Vercel abstraction
|
|
443
|
+
- If only `@ai-sdk/openai` is present (no direct `openai`), recommend `patch()` or adding `openai` as a direct dep
|
|
444
|
+
|
|
445
|
+
### Edge Runtime / Cloudflare Workers
|
|
446
|
+
- `session.run()` relies on `AsyncLocalStorage` which is unavailable in Edge Runtime
|
|
447
|
+
- Use explicit context: `agent.trackUserMessage(content, { sessionId })` instead
|
|
448
|
+
|
|
449
|
+
### OpenAI Assistants API
|
|
450
|
+
- Provider wrappers do NOT auto-instrument the Assistants API (async/polling-based)
|
|
451
|
+
- Use manual tracking: `trackUserMessage()` when creating a message, `trackAiMessage()` when polling
|
|
452
|
+
|
|
453
|
+
---
|
|
454
|
+
|
|
455
|
+
## Safety Rules
|
|
456
|
+
|
|
457
|
+
- **Never modify unrelated files.** Only touch files with LLM call sites and the bootstrap file.
|
|
458
|
+
- **Never duplicate instrumentation.** Check for existing `patch()` or wrapper calls before adding new ones.
|
|
459
|
+
- **Pause before Phase 3.** Always show the discovery report and get developer confirmation.
|
|
460
|
+
- **Prefer additive changes.** Add imports and wrappers rather than rewriting entire files.
|
|
461
|
+
- **Keep content mode explicit.** Default is `full` + `redactPii: true`. Never silently downgrade.
|
|
462
|
+
- **Preserve existing tests.** Instrumentation must not break the test suite.
|
|
463
|
+
- **Idempotent.** Running this twice should not double-instrument. Detect existing instrumentation and skip.
|
|
File without changes
|
|
File without changes
|
package/bin/amplitude-ai-mcp.mjs
CHANGED
|
File without changes
|
|
File without changes
|
|
File without changes
|
package/bin/amplitude-ai.mjs
CHANGED
|
@@ -9,22 +9,31 @@ const [, , command, ...rest] = process.argv;
|
|
|
9
9
|
|
|
10
10
|
const commandToBin = {
|
|
11
11
|
mcp: './amplitude-ai-mcp.mjs',
|
|
12
|
-
init: './amplitude-ai-init.mjs',
|
|
13
12
|
doctor: './amplitude-ai-doctor.mjs',
|
|
14
13
|
status: './amplitude-ai-status.mjs',
|
|
15
14
|
};
|
|
16
15
|
|
|
16
|
+
const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json');
|
|
17
|
+
|
|
17
18
|
if (command === '--help' || command === '-h' || command === undefined) {
|
|
18
|
-
const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json');
|
|
19
19
|
const version = JSON.parse(readFileSync(pkgPath, 'utf8')).version;
|
|
20
20
|
process.stdout.write(
|
|
21
|
-
|
|
21
|
+
`@amplitude/ai v${version}
|
|
22
|
+
|
|
23
|
+
Paste this into your AI coding agent (Cursor, Claude Code, Copilot, etc.):
|
|
24
|
+
|
|
25
|
+
Instrument this app with @amplitude/ai. Follow node_modules/@amplitude/ai/amplitude-ai.md
|
|
26
|
+
|
|
27
|
+
CLI commands:
|
|
28
|
+
mcp Start the MCP server (optional, for advanced tooling)
|
|
29
|
+
doctor Validate environment, deps, and event pipeline
|
|
30
|
+
status Show SDK version, installed providers, and env config
|
|
31
|
+
`
|
|
22
32
|
);
|
|
23
33
|
process.exit(0);
|
|
24
34
|
}
|
|
25
35
|
|
|
26
36
|
if (command === '--version' || command === '-v') {
|
|
27
|
-
const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json');
|
|
28
37
|
const version = JSON.parse(readFileSync(pkgPath, 'utf8')).version;
|
|
29
38
|
process.stdout.write(`${version}\n`);
|
|
30
39
|
process.exit(0);
|
|
@@ -39,5 +48,5 @@ if (command in commandToBin) {
|
|
|
39
48
|
process.exit(result.status ?? 1);
|
|
40
49
|
}
|
|
41
50
|
|
|
42
|
-
process.stderr.write(`Unknown command: ${command}\nUsage: amplitude-ai <mcp|
|
|
51
|
+
process.stderr.write(`Unknown command: ${command}\nUsage: amplitude-ai <mcp|doctor|status>\n`);
|
|
43
52
|
process.exit(1);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_meta": {
|
|
3
|
-
"schema_version":
|
|
4
|
-
"generated_at": "2026-03-
|
|
3
|
+
"schema_version": 2,
|
|
4
|
+
"generated_at": "2026-03-20T19:12:10.988879+00:00",
|
|
5
5
|
"source": "langley/langley/llm_analytics/taxonomy_catalog.py",
|
|
6
6
|
"category": "Agent Analytics"
|
|
7
7
|
},
|
|
@@ -46,6 +46,11 @@
|
|
|
46
46
|
"type": "string",
|
|
47
47
|
"description": "Agent code version (e.g., 'v4.2'). Enables version-over-version quality comparison."
|
|
48
48
|
},
|
|
49
|
+
{
|
|
50
|
+
"name": "[Agent] Agent Description",
|
|
51
|
+
"type": "string",
|
|
52
|
+
"description": "Human-readable description of the agent's purpose (e.g., 'Handles user chat requests via OpenAI GPT-4o'). Enables observability-driven agent registry from event streams."
|
|
53
|
+
},
|
|
49
54
|
{
|
|
50
55
|
"name": "[Agent] Context",
|
|
51
56
|
"type": "string",
|
|
@@ -135,6 +140,11 @@
|
|
|
135
140
|
"name": "[Agent] Message Labels",
|
|
136
141
|
"type": "string",
|
|
137
142
|
"description": "Serialized JSON array of MessageLabel objects (key-value pairs with optional confidence). Used for routing tags, classifier output, business context."
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
"name": "[Agent] Message Source",
|
|
146
|
+
"type": "string",
|
|
147
|
+
"description": "Origin of the message: 'user' for direct user input, 'agent' for messages delegated from a parent agent (auto-set based on parentAgentId)."
|
|
138
148
|
}
|
|
139
149
|
]
|
|
140
150
|
},
|
|
@@ -178,6 +188,11 @@
|
|
|
178
188
|
"type": "string",
|
|
179
189
|
"description": "Agent code version (e.g., 'v4.2'). Enables version-over-version quality comparison."
|
|
180
190
|
},
|
|
191
|
+
{
|
|
192
|
+
"name": "[Agent] Agent Description",
|
|
193
|
+
"type": "string",
|
|
194
|
+
"description": "Human-readable description of the agent's purpose (e.g., 'Handles user chat requests via OpenAI GPT-4o'). Enables observability-driven agent registry from event streams."
|
|
195
|
+
},
|
|
181
196
|
{
|
|
182
197
|
"name": "[Agent] Context",
|
|
183
198
|
"type": "string",
|
|
@@ -444,6 +459,11 @@
|
|
|
444
459
|
"type": "string",
|
|
445
460
|
"description": "Agent code version (e.g., 'v4.2'). Enables version-over-version quality comparison."
|
|
446
461
|
},
|
|
462
|
+
{
|
|
463
|
+
"name": "[Agent] Agent Description",
|
|
464
|
+
"type": "string",
|
|
465
|
+
"description": "Human-readable description of the agent's purpose (e.g., 'Handles user chat requests via OpenAI GPT-4o'). Enables observability-driven agent registry from event streams."
|
|
466
|
+
},
|
|
447
467
|
{
|
|
448
468
|
"name": "[Agent] Context",
|
|
449
469
|
"type": "string",
|
|
@@ -579,6 +599,11 @@
|
|
|
579
599
|
"type": "string",
|
|
580
600
|
"description": "Agent code version (e.g., 'v4.2'). Enables version-over-version quality comparison."
|
|
581
601
|
},
|
|
602
|
+
{
|
|
603
|
+
"name": "[Agent] Agent Description",
|
|
604
|
+
"type": "string",
|
|
605
|
+
"description": "Human-readable description of the agent's purpose (e.g., 'Handles user chat requests via OpenAI GPT-4o'). Enables observability-driven agent registry from event streams."
|
|
606
|
+
},
|
|
582
607
|
{
|
|
583
608
|
"name": "[Agent] Context",
|
|
584
609
|
"type": "string",
|
|
@@ -688,6 +713,11 @@
|
|
|
688
713
|
"type": "string",
|
|
689
714
|
"description": "Agent code version (e.g., 'v4.2'). Enables version-over-version quality comparison."
|
|
690
715
|
},
|
|
716
|
+
{
|
|
717
|
+
"name": "[Agent] Agent Description",
|
|
718
|
+
"type": "string",
|
|
719
|
+
"description": "Human-readable description of the agent's purpose (e.g., 'Handles user chat requests via OpenAI GPT-4o'). Enables observability-driven agent registry from event streams."
|
|
720
|
+
},
|
|
691
721
|
{
|
|
692
722
|
"name": "[Agent] Context",
|
|
693
723
|
"type": "string",
|
|
@@ -796,6 +826,11 @@
|
|
|
796
826
|
"type": "string",
|
|
797
827
|
"description": "Agent code version (e.g., 'v4.2'). Enables version-over-version quality comparison."
|
|
798
828
|
},
|
|
829
|
+
{
|
|
830
|
+
"name": "[Agent] Agent Description",
|
|
831
|
+
"type": "string",
|
|
832
|
+
"description": "Human-readable description of the agent's purpose (e.g., 'Handles user chat requests via OpenAI GPT-4o'). Enables observability-driven agent registry from event streams."
|
|
833
|
+
},
|
|
799
834
|
{
|
|
800
835
|
"name": "[Agent] Context",
|
|
801
836
|
"type": "string",
|
|
@@ -875,6 +910,11 @@
|
|
|
875
910
|
"type": "string",
|
|
876
911
|
"description": "Agent code version (e.g., 'v4.2'). Enables version-over-version quality comparison."
|
|
877
912
|
},
|
|
913
|
+
{
|
|
914
|
+
"name": "[Agent] Agent Description",
|
|
915
|
+
"type": "string",
|
|
916
|
+
"description": "Human-readable description of the agent's purpose (e.g., 'Handles user chat requests via OpenAI GPT-4o'). Enables observability-driven agent registry from event streams."
|
|
917
|
+
},
|
|
878
918
|
{
|
|
879
919
|
"name": "[Agent] Context",
|
|
880
920
|
"type": "string",
|
|
@@ -945,6 +985,11 @@
|
|
|
945
985
|
"type": "string",
|
|
946
986
|
"description": "Agent code version (e.g., 'v4.2'). Enables version-over-version quality comparison."
|
|
947
987
|
},
|
|
988
|
+
{
|
|
989
|
+
"name": "[Agent] Agent Description",
|
|
990
|
+
"type": "string",
|
|
991
|
+
"description": "Human-readable description of the agent's purpose (e.g., 'Handles user chat requests via OpenAI GPT-4o'). Enables observability-driven agent registry from event streams."
|
|
992
|
+
},
|
|
948
993
|
{
|
|
949
994
|
"name": "[Agent] Context",
|
|
950
995
|
"type": "string",
|
|
@@ -1199,6 +1244,11 @@
|
|
|
1199
1244
|
"name": "[Agent] Agent Version",
|
|
1200
1245
|
"type": "string",
|
|
1201
1246
|
"description": "Agent code version (e.g., 'v4.2'). Enables version-over-version quality comparison."
|
|
1247
|
+
},
|
|
1248
|
+
{
|
|
1249
|
+
"name": "[Agent] Agent Description",
|
|
1250
|
+
"type": "string",
|
|
1251
|
+
"description": "Human-readable description of the agent's purpose (e.g., 'Handles user chat requests via OpenAI GPT-4o'). Enables observability-driven agent registry from event streams."
|
|
1202
1252
|
}
|
|
1203
1253
|
]
|
|
1204
1254
|
},
|