@rudderjs/ai 1.17.3 → 1.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -1274
- package/dist/budget-orm/index.d.ts +1 -95
- package/dist/budget-orm/index.d.ts.map +1 -1
- package/dist/budget-orm/index.js +4 -176
- package/dist/budget-orm/index.js.map +1 -1
- package/dist/chat-mentions.d.ts +1 -58
- package/dist/chat-mentions.d.ts.map +1 -1
- package/dist/chat-mentions.js +4 -80
- package/dist/chat-mentions.js.map +1 -1
- package/dist/commands/ai-eval.d.ts +1 -92
- package/dist/commands/ai-eval.d.ts.map +1 -1
- package/dist/commands/ai-eval.js +4 -377
- package/dist/commands/ai-eval.js.map +1 -1
- package/dist/commands/make-agent.d.ts +1 -2
- package/dist/commands/make-agent.d.ts.map +1 -1
- package/dist/commands/make-agent.js +4 -22
- package/dist/commands/make-agent.js.map +1 -1
- package/dist/computer-use/index.d.ts +1 -52
- package/dist/computer-use/index.d.ts.map +1 -1
- package/dist/computer-use/index.js +4 -50
- package/dist/computer-use/index.js.map +1 -1
- package/dist/conversation-orm/index.d.ts +1 -108
- package/dist/conversation-orm/index.d.ts.map +1 -1
- package/dist/conversation-orm/index.js +4 -214
- package/dist/conversation-orm/index.js.map +1 -1
- package/dist/doctor.d.ts +1 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +4 -65
- package/dist/doctor.js.map +1 -1
- package/dist/eval/index.d.ts +1 -270
- package/dist/eval/index.d.ts.map +1 -1
- package/dist/eval/index.js +4 -509
- package/dist/eval/index.js.map +1 -1
- package/dist/gateway/index.d.ts +1 -10
- package/dist/gateway/index.d.ts.map +1 -1
- package/dist/gateway/index.js +4 -10
- package/dist/gateway/index.js.map +1 -1
- package/dist/index.d.ts +1 -66
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -78
- package/dist/index.js.map +1 -1
- package/dist/mcp/index.d.ts +1 -15
- package/dist/mcp/index.d.ts.map +1 -1
- package/dist/mcp/index.js +4 -14
- package/dist/mcp/index.js.map +1 -1
- package/dist/memory-embedding/index.d.ts +1 -120
- package/dist/memory-embedding/index.d.ts.map +1 -1
- package/dist/memory-embedding/index.js +4 -228
- package/dist/memory-embedding/index.js.map +1 -1
- package/dist/memory-orm/index.d.ts +1 -117
- package/dist/memory-orm/index.d.ts.map +1 -1
- package/dist/memory-orm/index.js +4 -186
- package/dist/memory-orm/index.js.map +1 -1
- package/dist/node/index.d.ts +1 -2
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +4 -2
- package/dist/node/index.js.map +1 -1
- package/dist/observers.d.ts +1 -129
- package/dist/observers.d.ts.map +1 -1
- package/dist/observers.js +4 -39
- package/dist/observers.js.map +1 -1
- package/dist/react/index.d.ts +1 -15
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +4 -15
- package/dist/react/index.js.map +1 -1
- package/dist/server/index.d.ts +1 -1
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +4 -1
- package/dist/server/index.js.map +1 -1
- package/package.json +9 -13
- package/boost/guidelines.md +0 -260
- package/boost/skills/ai-agents/SKILL.md +0 -240
- package/boost/skills/ai-tools/SKILL.md +0 -260
- package/dist/agent-run-store.d.ts +0 -161
- package/dist/agent-run-store.d.ts.map +0 -1
- package/dist/agent-run-store.js +0 -98
- package/dist/agent-run-store.js.map +0 -1
- package/dist/agent-sse.d.ts +0 -153
- package/dist/agent-sse.d.ts.map +0 -1
- package/dist/agent-sse.js +0 -282
- package/dist/agent-sse.js.map +0 -1
- package/dist/agent.d.ts +0 -508
- package/dist/agent.d.ts.map +0 -1
- package/dist/agent.js +0 -1538
- package/dist/agent.js.map +0 -1
- package/dist/attachment.d.ts +0 -31
- package/dist/attachment.d.ts.map +0 -1
- package/dist/attachment.js +0 -89
- package/dist/attachment.js.map +0 -1
- package/dist/audio.d.ts +0 -45
- package/dist/audio.d.ts.map +0 -1
- package/dist/audio.js +0 -93
- package/dist/audio.js.map +0 -1
- package/dist/base64.d.ts +0 -7
- package/dist/base64.d.ts.map +0 -1
- package/dist/base64.js +0 -39
- package/dist/base64.js.map +0 -1
- package/dist/budget/pricing.d.ts +0 -124
- package/dist/budget/pricing.d.ts.map +0 -1
- package/dist/budget/pricing.js +0 -175
- package/dist/budget/pricing.js.map +0 -1
- package/dist/budget/storage.d.ts +0 -104
- package/dist/budget/storage.d.ts.map +0 -1
- package/dist/budget/storage.js +0 -0
- package/dist/budget/storage.js.map +0 -1
- package/dist/budget/with-budget.d.ts +0 -119
- package/dist/budget/with-budget.d.ts.map +0 -1
- package/dist/budget/with-budget.js +0 -175
- package/dist/budget/with-budget.js.map +0 -1
- package/dist/cached-embedding.d.ts +0 -14
- package/dist/cached-embedding.d.ts.map +0 -1
- package/dist/cached-embedding.js +0 -44
- package/dist/cached-embedding.js.map +0 -1
- package/dist/computer-use/actions.d.ts +0 -214
- package/dist/computer-use/actions.d.ts.map +0 -1
- package/dist/computer-use/actions.js +0 -48
- package/dist/computer-use/actions.js.map +0 -1
- package/dist/computer-use/errors.d.ts +0 -57
- package/dist/computer-use/errors.d.ts.map +0 -1
- package/dist/computer-use/errors.js +0 -76
- package/dist/computer-use/errors.js.map +0 -1
- package/dist/computer-use/playwright.d.ts +0 -76
- package/dist/computer-use/playwright.d.ts.map +0 -1
- package/dist/computer-use/playwright.js +0 -270
- package/dist/computer-use/playwright.js.map +0 -1
- package/dist/computer-use/tool.d.ts +0 -154
- package/dist/computer-use/tool.d.ts.map +0 -1
- package/dist/computer-use/tool.js +0 -210
- package/dist/computer-use/tool.js.map +0 -1
- package/dist/continuation-validation.d.ts +0 -85
- package/dist/continuation-validation.d.ts.map +0 -1
- package/dist/continuation-validation.js +0 -166
- package/dist/continuation-validation.js.map +0 -1
- package/dist/conversation-persistence.d.ts +0 -46
- package/dist/conversation-persistence.d.ts.map +0 -1
- package/dist/conversation-persistence.js +0 -176
- package/dist/conversation-persistence.js.map +0 -1
- package/dist/conversation.d.ts +0 -11
- package/dist/conversation.d.ts.map +0 -1
- package/dist/conversation.js +0 -55
- package/dist/conversation.js.map +0 -1
- package/dist/eval/fixtures.d.ts +0 -65
- package/dist/eval/fixtures.d.ts.map +0 -1
- package/dist/eval/fixtures.js +0 -110
- package/dist/eval/fixtures.js.map +0 -1
- package/dist/eval/html-reporter.d.ts +0 -25
- package/dist/eval/html-reporter.d.ts.map +0 -1
- package/dist/eval/html-reporter.js +0 -209
- package/dist/eval/html-reporter.js.map +0 -1
- package/dist/eval/json-reporter.d.ts +0 -43
- package/dist/eval/json-reporter.d.ts.map +0 -1
- package/dist/eval/json-reporter.js +0 -40
- package/dist/eval/json-reporter.js.map +0 -1
- package/dist/facade.d.ts +0 -96
- package/dist/facade.d.ts.map +0 -1
- package/dist/facade.js +0 -146
- package/dist/facade.js.map +0 -1
- package/dist/fake.d.ts +0 -201
- package/dist/fake.d.ts.map +0 -1
- package/dist/fake.js +0 -428
- package/dist/fake.js.map +0 -1
- package/dist/file-search.d.ts +0 -168
- package/dist/file-search.d.ts.map +0 -1
- package/dist/file-search.js +0 -158
- package/dist/file-search.js.map +0 -1
- package/dist/files.d.ts +0 -27
- package/dist/files.d.ts.map +0 -1
- package/dist/files.js +0 -44
- package/dist/files.js.map +0 -1
- package/dist/gateway/http-gateway-adapter.d.ts +0 -94
- package/dist/gateway/http-gateway-adapter.d.ts.map +0 -1
- package/dist/gateway/http-gateway-adapter.js +0 -106
- package/dist/gateway/http-gateway-adapter.js.map +0 -1
- package/dist/gateway/sse.d.ts +0 -28
- package/dist/gateway/sse.d.ts.map +0 -1
- package/dist/gateway/sse.js +0 -78
- package/dist/gateway/sse.js.map +0 -1
- package/dist/handoff.d.ts +0 -95
- package/dist/handoff.d.ts.map +0 -1
- package/dist/handoff.js +0 -78
- package/dist/handoff.js.map +0 -1
- package/dist/handoffs-driver.d.ts +0 -58
- package/dist/handoffs-driver.d.ts.map +0 -1
- package/dist/handoffs-driver.js +0 -103
- package/dist/handoffs-driver.js.map +0 -1
- package/dist/image.d.ts +0 -40
- package/dist/image.d.ts.map +0 -1
- package/dist/image.js +0 -109
- package/dist/image.js.map +0 -1
- package/dist/mcp/client-tools.d.ts +0 -39
- package/dist/mcp/client-tools.d.ts.map +0 -1
- package/dist/mcp/client-tools.js +0 -147
- package/dist/mcp/client-tools.js.map +0 -1
- package/dist/mcp/server-from-agent.d.ts +0 -24
- package/dist/mcp/server-from-agent.d.ts.map +0 -1
- package/dist/mcp/server-from-agent.js +0 -113
- package/dist/mcp/server-from-agent.js.map +0 -1
- package/dist/mcp/types.d.ts +0 -64
- package/dist/mcp/types.d.ts.map +0 -1
- package/dist/mcp/types.js +0 -6
- package/dist/mcp/types.js.map +0 -1
- package/dist/memory-extract.d.ts +0 -60
- package/dist/memory-extract.d.ts.map +0 -1
- package/dist/memory-extract.js +0 -163
- package/dist/memory-extract.js.map +0 -1
- package/dist/memory-inject.d.ts +0 -39
- package/dist/memory-inject.d.ts.map +0 -1
- package/dist/memory-inject.js +0 -135
- package/dist/memory-inject.js.map +0 -1
- package/dist/memory.d.ts +0 -55
- package/dist/memory.d.ts.map +0 -1
- package/dist/memory.js +0 -132
- package/dist/memory.js.map +0 -1
- package/dist/middleware.d.ts +0 -18
- package/dist/middleware.d.ts.map +0 -1
- package/dist/middleware.js +0 -72
- package/dist/middleware.js.map +0 -1
- package/dist/node/attachment.d.ts +0 -6
- package/dist/node/attachment.d.ts.map +0 -1
- package/dist/node/attachment.js +0 -35
- package/dist/node/attachment.js.map +0 -1
- package/dist/node/transcription.d.ts +0 -4
- package/dist/node/transcription.d.ts.map +0 -1
- package/dist/node/transcription.js +0 -8
- package/dist/node/transcription.js.map +0 -1
- package/dist/output.d.ts +0 -22
- package/dist/output.d.ts.map +0 -1
- package/dist/output.js +0 -60
- package/dist/output.js.map +0 -1
- package/dist/provider-tools.d.ts +0 -87
- package/dist/provider-tools.d.ts.map +0 -1
- package/dist/provider-tools.js +0 -189
- package/dist/provider-tools.js.map +0 -1
- package/dist/providers/anthropic.d.ts +0 -24
- package/dist/providers/anthropic.d.ts.map +0 -1
- package/dist/providers/anthropic.js +0 -405
- package/dist/providers/anthropic.js.map +0 -1
- package/dist/providers/azure.d.ts +0 -13
- package/dist/providers/azure.d.ts.map +0 -1
- package/dist/providers/azure.js +0 -15
- package/dist/providers/azure.js.map +0 -1
- package/dist/providers/bedrock.d.ts +0 -75
- package/dist/providers/bedrock.d.ts.map +0 -1
- package/dist/providers/bedrock.js +0 -181
- package/dist/providers/bedrock.js.map +0 -1
- package/dist/providers/cohere.d.ts +0 -13
- package/dist/providers/cohere.d.ts.map +0 -1
- package/dist/providers/cohere.js +0 -87
- package/dist/providers/cohere.js.map +0 -1
- package/dist/providers/deepseek.d.ts +0 -12
- package/dist/providers/deepseek.d.ts.map +0 -1
- package/dist/providers/deepseek.js +0 -15
- package/dist/providers/deepseek.js.map +0 -1
- package/dist/providers/elevenlabs.d.ts +0 -98
- package/dist/providers/elevenlabs.d.ts.map +0 -1
- package/dist/providers/elevenlabs.js +0 -229
- package/dist/providers/elevenlabs.js.map +0 -1
- package/dist/providers/google-cache-registry.d.ts +0 -132
- package/dist/providers/google-cache-registry.d.ts.map +0 -1
- package/dist/providers/google-cache-registry.js +0 -209
- package/dist/providers/google-cache-registry.js.map +0 -1
- package/dist/providers/google.d.ts +0 -38
- package/dist/providers/google.d.ts.map +0 -1
- package/dist/providers/google.js +0 -903
- package/dist/providers/google.js.map +0 -1
- package/dist/providers/groq.d.ts +0 -12
- package/dist/providers/groq.d.ts.map +0 -1
- package/dist/providers/groq.js +0 -15
- package/dist/providers/groq.js.map +0 -1
- package/dist/providers/jina.d.ts +0 -13
- package/dist/providers/jina.d.ts.map +0 -1
- package/dist/providers/jina.js +0 -90
- package/dist/providers/jina.js.map +0 -1
- package/dist/providers/mistral.d.ts +0 -13
- package/dist/providers/mistral.d.ts.map +0 -1
- package/dist/providers/mistral.js +0 -46
- package/dist/providers/mistral.js.map +0 -1
- package/dist/providers/ollama.d.ts +0 -11
- package/dist/providers/ollama.d.ts.map +0 -1
- package/dist/providers/ollama.js +0 -15
- package/dist/providers/ollama.js.map +0 -1
- package/dist/providers/openai.d.ts +0 -79
- package/dist/providers/openai.d.ts.map +0 -1
- package/dist/providers/openai.js +0 -792
- package/dist/providers/openai.js.map +0 -1
- package/dist/providers/openrouter.d.ts +0 -43
- package/dist/providers/openrouter.d.ts.map +0 -1
- package/dist/providers/openrouter.js +0 -21
- package/dist/providers/openrouter.js.map +0 -1
- package/dist/providers/voyage.d.ts +0 -91
- package/dist/providers/voyage.d.ts.map +0 -1
- package/dist/providers/voyage.js +0 -166
- package/dist/providers/voyage.js.map +0 -1
- package/dist/providers/xai.d.ts +0 -12
- package/dist/providers/xai.d.ts.map +0 -1
- package/dist/providers/xai.js +0 -15
- package/dist/providers/xai.js.map +0 -1
- package/dist/queue-job.d.ts +0 -100
- package/dist/queue-job.d.ts.map +0 -1
- package/dist/queue-job.js +0 -185
- package/dist/queue-job.js.map +0 -1
- package/dist/react/agent-run.d.ts +0 -111
- package/dist/react/agent-run.d.ts.map +0 -1
- package/dist/react/agent-run.js +0 -107
- package/dist/react/agent-run.js.map +0 -1
- package/dist/react/useAgentRun.d.ts +0 -68
- package/dist/react/useAgentRun.d.ts.map +0 -1
- package/dist/react/useAgentRun.js +0 -125
- package/dist/react/useAgentRun.js.map +0 -1
- package/dist/registry.d.ts +0 -45
- package/dist/registry.d.ts.map +0 -1
- package/dist/registry.js +0 -131
- package/dist/registry.js.map +0 -1
- package/dist/rerank.d.ts +0 -20
- package/dist/rerank.d.ts.map +0 -1
- package/dist/rerank.js +0 -40
- package/dist/rerank.js.map +0 -1
- package/dist/resume-approval.d.ts +0 -30
- package/dist/resume-approval.d.ts.map +0 -1
- package/dist/resume-approval.js +0 -147
- package/dist/resume-approval.js.map +0 -1
- package/dist/sanitize-conversation.d.ts +0 -43
- package/dist/sanitize-conversation.d.ts.map +0 -1
- package/dist/sanitize-conversation.js +0 -85
- package/dist/sanitize-conversation.js.map +0 -1
- package/dist/scoped-tool.d.ts +0 -98
- package/dist/scoped-tool.d.ts.map +0 -1
- package/dist/scoped-tool.js +0 -174
- package/dist/scoped-tool.js.map +0 -1
- package/dist/server/provider.d.ts +0 -22
- package/dist/server/provider.d.ts.map +0 -1
- package/dist/server/provider.js +0 -194
- package/dist/server/provider.js.map +0 -1
- package/dist/similarity-search.d.ts +0 -163
- package/dist/similarity-search.d.ts.map +0 -1
- package/dist/similarity-search.js +0 -147
- package/dist/similarity-search.js.map +0 -1
- package/dist/sub-agent-run-store.d.ts +0 -157
- package/dist/sub-agent-run-store.d.ts.map +0 -1
- package/dist/sub-agent-run-store.js +0 -87
- package/dist/sub-agent-run-store.js.map +0 -1
- package/dist/tool-execution.d.ts +0 -16
- package/dist/tool-execution.d.ts.map +0 -1
- package/dist/tool-execution.js +0 -498
- package/dist/tool-execution.js.map +0 -1
- package/dist/tool-helpers.d.ts +0 -77
- package/dist/tool-helpers.d.ts.map +0 -1
- package/dist/tool-helpers.js +0 -117
- package/dist/tool-helpers.js.map +0 -1
- package/dist/tool.d.ts +0 -216
- package/dist/tool.d.ts.map +0 -1
- package/dist/tool.js +0 -175
- package/dist/tool.js.map +0 -1
- package/dist/transcription.d.ts +0 -42
- package/dist/transcription.d.ts.map +0 -1
- package/dist/transcription.js +0 -77
- package/dist/transcription.js.map +0 -1
- package/dist/types.d.ts +0 -1020
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -2
- package/dist/types.js.map +0 -1
- package/dist/util/hash.d.ts +0 -11
- package/dist/util/hash.d.ts.map +0 -1
- package/dist/util/hash.js +0 -23
- package/dist/util/hash.js.map +0 -1
- package/dist/vector-stores/index.d.ts +0 -96
- package/dist/vector-stores/index.d.ts.map +0 -1
- package/dist/vector-stores/index.js +0 -153
- package/dist/vector-stores/index.js.map +0 -1
- package/dist/vercel-protocol.d.ts +0 -18
- package/dist/vercel-protocol.d.ts.map +0 -1
- package/dist/vercel-protocol.js +0 -75
- package/dist/vercel-protocol.js.map +0 -1
- package/dist/zod-to-json-schema.d.ts +0 -16
- package/dist/zod-to-json-schema.d.ts.map +0 -1
- package/dist/zod-to-json-schema.js +0 -17
- package/dist/zod-to-json-schema.js.map +0 -1
package/dist/cached-embedding.js
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* In-memory caching wrapper for any EmbeddingAdapter.
|
|
3
|
-
*
|
|
4
|
-
* Caches embeddings by `model:text` key so repeated inputs skip the provider call.
|
|
5
|
-
* Cache hits report zero token usage since no API call is made.
|
|
6
|
-
*/
|
|
7
|
-
export class CachedEmbeddingAdapter {
|
|
8
|
-
inner;
|
|
9
|
-
cache = new Map();
|
|
10
|
-
constructor(inner) {
|
|
11
|
-
this.inner = inner;
|
|
12
|
-
}
|
|
13
|
-
async embed(input, model) {
|
|
14
|
-
const inputs = Array.isArray(input) ? input : [input];
|
|
15
|
-
const results = [];
|
|
16
|
-
const uncached = [];
|
|
17
|
-
const uncachedIndices = [];
|
|
18
|
-
for (let i = 0; i < inputs.length; i++) {
|
|
19
|
-
const key = `${model}:${inputs[i]}`;
|
|
20
|
-
const cached = this.cache.get(key);
|
|
21
|
-
if (cached) {
|
|
22
|
-
results[i] = cached;
|
|
23
|
-
}
|
|
24
|
-
else {
|
|
25
|
-
uncached.push(inputs[i]);
|
|
26
|
-
uncachedIndices.push(i);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
if (uncached.length > 0) {
|
|
30
|
-
const fresh = await this.inner.embed(uncached, model);
|
|
31
|
-
for (let j = 0; j < uncachedIndices.length; j++) {
|
|
32
|
-
const idx = uncachedIndices[j];
|
|
33
|
-
const embedding = fresh.embeddings[j];
|
|
34
|
-
results[idx] = embedding;
|
|
35
|
-
this.cache.set(`${model}:${uncached[j]}`, embedding);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
return {
|
|
39
|
-
embeddings: results,
|
|
40
|
-
usage: { promptTokens: 0, totalTokens: 0 },
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
//# sourceMappingURL=cached-embedding.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cached-embedding.js","sourceRoot":"","sources":["../src/cached-embedding.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,MAAM,OAAO,sBAAsB;IAGb;IAFZ,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE3C,YAAoB,KAAuB;QAAvB,UAAK,GAAL,KAAK,CAAkB;IAAG,CAAC;IAE/C,KAAK,CAAC,KAAK,CAAC,KAAwB,EAAE,KAAa;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACrD,MAAM,OAAO,GAAe,EAAE,CAAA;QAC9B,MAAM,QAAQ,GAAa,EAAE,CAAA;QAC7B,MAAM,eAAe,GAAa,EAAE,CAAA;QAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,GAAG,KAAK,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAA;YACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClC,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,CAAA;YACrB,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAA;gBACzB,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACzB,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,GAAG,GAAG,eAAe,CAAC,CAAC,CAAE,CAAA;gBAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAE,CAAA;gBACtC,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,CAAA;gBACxB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;YACtD,CAAC;QACH,CAAC;QAED,OAAO;YACL,UAAU,EAAE,OAAO;YACnB,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE;SAC3C,CAAA;IACH,CAAC;CACF"}
|
|
@@ -1,214 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Action vocabulary for `@rudderjs/ai/computer-use` (#A7 Phase 1).
|
|
3
|
-
*
|
|
4
|
-
* Mirrors Anthropic's `computer_20250124` tool schema verbatim — see
|
|
5
|
-
* https://docs.anthropic.com/en/docs/build-with-claude/tool-use/computer-use-tool
|
|
6
|
-
*
|
|
7
|
-
* The model emits a {@link ComputerAction} as the JSON argument of a
|
|
8
|
-
* `computer` tool call; the executor (see `./playwright.ts`) dispatches
|
|
9
|
-
* the action against a Playwright `Page` and returns a
|
|
10
|
-
* {@link ComputerActionResult} the agent loop forwards back to the model.
|
|
11
|
-
*
|
|
12
|
-
* # Why mirror Anthropic exactly?
|
|
13
|
-
*
|
|
14
|
-
* Computer-use is `Anthropic-only` in v1 (see plan
|
|
15
|
-
* `docs/plans/2026-05-10-ai-computer-use.md`). Claude is fine-tuned on
|
|
16
|
-
* this exact action vocabulary; reusing the schema means:
|
|
17
|
-
*
|
|
18
|
-
* - Phase 2's `computerUseTool({ page })` factory maps cleanly to
|
|
19
|
-
* Anthropic's native `{ type: 'computer_20250124', name: 'computer',
|
|
20
|
-
* display_width_px, display_height_px }` block — zero translation
|
|
21
|
-
* layer.
|
|
22
|
-
* - When OpenAI or Google's native computer-use APIs mature enough to
|
|
23
|
-
* route through, only the provider adapter changes; the schema and
|
|
24
|
-
* executor stay put.
|
|
25
|
-
*
|
|
26
|
-
* # Coordinates
|
|
27
|
-
*
|
|
28
|
-
* `[x, y]` in viewport pixels. Origin is top-left. Anthropic suggests a
|
|
29
|
-
* 1280×800 viewport for Claude's training distribution; the executor
|
|
30
|
-
* does not validate bounds — out-of-viewport coordinates pass through
|
|
31
|
-
* to Playwright (which clips the cursor to viewport edges).
|
|
32
|
-
*
|
|
33
|
-
* # Modifier text on click / scroll actions
|
|
34
|
-
*
|
|
35
|
-
* Anthropic encodes "hold these modifier keys while clicking" as a `+`
|
|
36
|
-
* separated string on the `text` field — e.g.
|
|
37
|
-
* `{ action: 'left_click', coordinate: [400, 200], text: 'shift+ctrl' }`.
|
|
38
|
-
* The executor translates `cmd`/`super`/`meta` → `Meta`,
|
|
39
|
-
* `ctrl`/`control` → `Control`, etc. before passing to Playwright.
|
|
40
|
-
*/
|
|
41
|
-
/** Viewport pixel coordinate, top-left origin. `[x, y]`. */
|
|
42
|
-
export type Coordinate = readonly [number, number];
|
|
43
|
-
/**
|
|
44
|
-
* Discriminated union of every action Anthropic's `computer_20250124`
|
|
45
|
-
* tool can emit. The executor dispatches on `action`; downstream code
|
|
46
|
-
* should treat unknown variants as a forward-compatibility hazard
|
|
47
|
-
* rather than a bug — Anthropic versions the schema with dated suffixes
|
|
48
|
-
* (e.g. a future `computer_20260101` may add new variants we'll route
|
|
49
|
-
* here once supported).
|
|
50
|
-
*/
|
|
51
|
-
export type ComputerAction = {
|
|
52
|
-
action: 'screenshot';
|
|
53
|
-
} | {
|
|
54
|
-
action: 'cursor_position';
|
|
55
|
-
}
|
|
56
|
-
/** Pause for `duration` seconds. Useful between an action that triggers async UI and the screenshot that should observe the result. */
|
|
57
|
-
| {
|
|
58
|
-
action: 'wait';
|
|
59
|
-
duration: number;
|
|
60
|
-
} | {
|
|
61
|
-
action: 'mouse_move';
|
|
62
|
-
coordinate: Coordinate;
|
|
63
|
-
}
|
|
64
|
-
/** Single left-button click. `text` holds optional modifier keys, `+`-separated (e.g. `'shift+ctrl'`). */
|
|
65
|
-
| {
|
|
66
|
-
action: 'left_click';
|
|
67
|
-
coordinate: Coordinate;
|
|
68
|
-
text?: string;
|
|
69
|
-
} | {
|
|
70
|
-
action: 'right_click';
|
|
71
|
-
coordinate: Coordinate;
|
|
72
|
-
text?: string;
|
|
73
|
-
} | {
|
|
74
|
-
action: 'middle_click';
|
|
75
|
-
coordinate: Coordinate;
|
|
76
|
-
text?: string;
|
|
77
|
-
} | {
|
|
78
|
-
action: 'double_click';
|
|
79
|
-
coordinate: Coordinate;
|
|
80
|
-
text?: string;
|
|
81
|
-
} | {
|
|
82
|
-
action: 'triple_click';
|
|
83
|
-
coordinate: Coordinate;
|
|
84
|
-
text?: string;
|
|
85
|
-
}
|
|
86
|
-
/** Press the left mouse button. Pair with `left_mouse_up` to drag. `coordinate` (optional) moves first. */
|
|
87
|
-
| {
|
|
88
|
-
action: 'left_mouse_down';
|
|
89
|
-
coordinate?: Coordinate;
|
|
90
|
-
}
|
|
91
|
-
/** Release the left mouse button. `coordinate` (optional) moves first. */
|
|
92
|
-
| {
|
|
93
|
-
action: 'left_mouse_up';
|
|
94
|
-
coordinate?: Coordinate;
|
|
95
|
-
}
|
|
96
|
-
/** Type literal text. No modifier handling — use `key` for shortcuts. */
|
|
97
|
-
| {
|
|
98
|
-
action: 'type';
|
|
99
|
-
text: string;
|
|
100
|
-
}
|
|
101
|
-
/** Press a key chord. `text` is `+`-separated (e.g. `'ctrl+a'`, `'Return'`, `'cmd+shift+t'`). */
|
|
102
|
-
| {
|
|
103
|
-
action: 'key';
|
|
104
|
-
text: string;
|
|
105
|
-
}
|
|
106
|
-
/** Hold a single key for `duration` seconds. `text` is one key, not a chord. */
|
|
107
|
-
| {
|
|
108
|
-
action: 'hold_key';
|
|
109
|
-
text: string;
|
|
110
|
-
duration: number;
|
|
111
|
-
}
|
|
112
|
-
/** Scroll at `coordinate`. `scroll_amount` is in mouse-wheel "clicks" (~100px each). `text` holds optional modifiers. */
|
|
113
|
-
| {
|
|
114
|
-
action: 'scroll';
|
|
115
|
-
coordinate: Coordinate;
|
|
116
|
-
scroll_direction: 'up' | 'down' | 'left' | 'right';
|
|
117
|
-
scroll_amount: number;
|
|
118
|
-
text?: string;
|
|
119
|
-
};
|
|
120
|
-
/**
|
|
121
|
-
* Tool-result content the agent loop hands back to the model after each
|
|
122
|
-
* action. Shape matches Anthropic's tool-result block content vocabulary:
|
|
123
|
-
*
|
|
124
|
-
* - `image` carries a PNG screenshot. The agent loop wraps this as
|
|
125
|
-
* `{ type: 'image', source: { type: 'base64', media_type, data } }`
|
|
126
|
-
* when serializing to the API.
|
|
127
|
-
* - `text` carries a one-line confirmation (`"left-clicked at (400, 200)"`)
|
|
128
|
-
* or the response of an introspective action (`cursor_position`).
|
|
129
|
-
* - `error` carries the failure text from a Playwright throw. Caller
|
|
130
|
-
* typically maps this onto `{ is_error: true, content: [...] }` in
|
|
131
|
-
* the tool-result block so the model knows the action failed and can
|
|
132
|
-
* retry / recover.
|
|
133
|
-
*/
|
|
134
|
-
export type ComputerActionResult = {
|
|
135
|
-
type: 'image';
|
|
136
|
-
media_type: 'image/png';
|
|
137
|
-
data: Uint8Array;
|
|
138
|
-
} | {
|
|
139
|
-
type: 'text';
|
|
140
|
-
text: string;
|
|
141
|
-
} | {
|
|
142
|
-
type: 'error';
|
|
143
|
-
text: string;
|
|
144
|
-
};
|
|
145
|
-
/**
|
|
146
|
-
* Per-run state the executor mutates. Tracks the last known cursor
|
|
147
|
-
* position so the `cursor_position` action can answer it (Playwright
|
|
148
|
-
* has no native API to read the synthesized mouse position).
|
|
149
|
-
*
|
|
150
|
-
* Create one with {@link makeExecutorState} per agent run; pass the
|
|
151
|
-
* same instance to every {@link executeComputerAction} call.
|
|
152
|
-
*/
|
|
153
|
-
export interface ComputerExecutorState {
|
|
154
|
-
/** Last known cursor position in viewport pixels. Updated on move/click/drag. */
|
|
155
|
-
cursor: {
|
|
156
|
-
x: number;
|
|
157
|
-
y: number;
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
/**
|
|
161
|
-
* Construct a fresh {@link ComputerExecutorState}. Optional `initial`
|
|
162
|
-
* seeds the cursor (defaults to `(0, 0)`).
|
|
163
|
-
*/
|
|
164
|
-
export declare function makeExecutorState(initial?: {
|
|
165
|
-
x: number;
|
|
166
|
-
y: number;
|
|
167
|
-
}): ComputerExecutorState;
|
|
168
|
-
/**
|
|
169
|
-
* Structural subset of Playwright's `Page` the executor calls. Defining
|
|
170
|
-
* this here means {@link executeComputerAction} types correctly without
|
|
171
|
-
* `@rudderjs/ai` taking a hard dependency on the `playwright` package
|
|
172
|
-
* (which carries a 300MB+ Chromium download). Apps install Playwright
|
|
173
|
-
* themselves and pass `page` in.
|
|
174
|
-
*
|
|
175
|
-
* Any object with this shape works — real Playwright `Page`,
|
|
176
|
-
* `puppeteer-core`'s `Page` (close enough), or a hand-rolled mock for
|
|
177
|
-
* tests. Phase 2's `computerUseTool({ page })` accepts the same type.
|
|
178
|
-
*/
|
|
179
|
-
export interface PageLike {
|
|
180
|
-
mouse: PageMouseLike;
|
|
181
|
-
keyboard: PageKeyboardLike;
|
|
182
|
-
/** Take a screenshot of the current viewport. Returns PNG bytes by default. */
|
|
183
|
-
screenshot(options?: {
|
|
184
|
-
type?: 'png' | 'jpeg';
|
|
185
|
-
}): Promise<Uint8Array>;
|
|
186
|
-
}
|
|
187
|
-
export interface PageMouseLike {
|
|
188
|
-
move(x: number, y: number, options?: {
|
|
189
|
-
steps?: number;
|
|
190
|
-
}): Promise<void>;
|
|
191
|
-
click(x: number, y: number, options?: {
|
|
192
|
-
button?: 'left' | 'right' | 'middle';
|
|
193
|
-
clickCount?: number;
|
|
194
|
-
delay?: number;
|
|
195
|
-
}): Promise<void>;
|
|
196
|
-
down(options?: {
|
|
197
|
-
button?: 'left' | 'right' | 'middle';
|
|
198
|
-
}): Promise<void>;
|
|
199
|
-
up(options?: {
|
|
200
|
-
button?: 'left' | 'right' | 'middle';
|
|
201
|
-
}): Promise<void>;
|
|
202
|
-
wheel(deltaX: number, deltaY: number): Promise<void>;
|
|
203
|
-
}
|
|
204
|
-
export interface PageKeyboardLike {
|
|
205
|
-
type(text: string, options?: {
|
|
206
|
-
delay?: number;
|
|
207
|
-
}): Promise<void>;
|
|
208
|
-
press(key: string, options?: {
|
|
209
|
-
delay?: number;
|
|
210
|
-
}): Promise<void>;
|
|
211
|
-
down(key: string): Promise<void>;
|
|
212
|
-
up(key: string): Promise<void>;
|
|
213
|
-
}
|
|
214
|
-
//# sourceMappingURL=actions.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../../src/computer-use/actions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAEH,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;AAElD;;;;;;;GAOG;AACH,MAAM,MAAM,cAAc,GAEtB;IAAE,MAAM,EAAE,YAAY,CAAA;CAAE,GACxB;IAAE,MAAM,EAAE,iBAAiB,CAAA;CAAE;AAG/B,uIAAuI;GACrI;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAGpC;IAAE,MAAM,EAAE,YAAY,CAAC;IAAC,UAAU,EAAE,UAAU,CAAA;CAAE;AAClD,0GAA0G;GACxG;IAAE,MAAM,EAAE,YAAY,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/D;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAChE;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GACjE;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GACjE;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE;AACnE,2GAA2G;GACzG;IAAE,MAAM,EAAE,iBAAiB,CAAC;IAAC,UAAU,CAAC,EAAE,UAAU,CAAA;CAAE;AACxD,0EAA0E;GACxE;IAAE,MAAM,EAAE,eAAe,CAAC;IAAC,UAAU,CAAC,EAAE,UAAU,CAAA;CAAE;AAGtD,yEAAyE;GACvE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE;AAClC,iGAAiG;GAC/F;IAAE,MAAM,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE;AACjC,gFAAgF;GAC9E;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE;AAGxD,yHAAyH;GACvH;IACE,MAAM,EAAE,QAAQ,CAAA;IAChB,UAAU,EAAE,UAAU,CAAA;IACtB,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAA;IAClD,aAAa,EAAE,MAAM,CAAA;IACrB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd,CAAA;AAEL;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,oBAAoB,GAC5B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GAC5D;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACpC,iFAAiF;IACjF,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CACjC;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,qBAAqB,CAE3F;AAID;;;;;;;;;;GAUG;AACH,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,aAAa,CAAA;IACpB,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,+EAA+E;IAC/E,UAAU,CAAC,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;CACrE;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACvE,KAAK,CACH,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GACtF,OAAO,CAAC,IAAI,CAAC,CAAA;IAChB,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACvE,EAAE,CAAC,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACrE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CACrD;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/D,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/D,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChC,EAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC/B"}
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Action vocabulary for `@rudderjs/ai/computer-use` (#A7 Phase 1).
|
|
3
|
-
*
|
|
4
|
-
* Mirrors Anthropic's `computer_20250124` tool schema verbatim — see
|
|
5
|
-
* https://docs.anthropic.com/en/docs/build-with-claude/tool-use/computer-use-tool
|
|
6
|
-
*
|
|
7
|
-
* The model emits a {@link ComputerAction} as the JSON argument of a
|
|
8
|
-
* `computer` tool call; the executor (see `./playwright.ts`) dispatches
|
|
9
|
-
* the action against a Playwright `Page` and returns a
|
|
10
|
-
* {@link ComputerActionResult} the agent loop forwards back to the model.
|
|
11
|
-
*
|
|
12
|
-
* # Why mirror Anthropic exactly?
|
|
13
|
-
*
|
|
14
|
-
* Computer-use is `Anthropic-only` in v1 (see plan
|
|
15
|
-
* `docs/plans/2026-05-10-ai-computer-use.md`). Claude is fine-tuned on
|
|
16
|
-
* this exact action vocabulary; reusing the schema means:
|
|
17
|
-
*
|
|
18
|
-
* - Phase 2's `computerUseTool({ page })` factory maps cleanly to
|
|
19
|
-
* Anthropic's native `{ type: 'computer_20250124', name: 'computer',
|
|
20
|
-
* display_width_px, display_height_px }` block — zero translation
|
|
21
|
-
* layer.
|
|
22
|
-
* - When OpenAI or Google's native computer-use APIs mature enough to
|
|
23
|
-
* route through, only the provider adapter changes; the schema and
|
|
24
|
-
* executor stay put.
|
|
25
|
-
*
|
|
26
|
-
* # Coordinates
|
|
27
|
-
*
|
|
28
|
-
* `[x, y]` in viewport pixels. Origin is top-left. Anthropic suggests a
|
|
29
|
-
* 1280×800 viewport for Claude's training distribution; the executor
|
|
30
|
-
* does not validate bounds — out-of-viewport coordinates pass through
|
|
31
|
-
* to Playwright (which clips the cursor to viewport edges).
|
|
32
|
-
*
|
|
33
|
-
* # Modifier text on click / scroll actions
|
|
34
|
-
*
|
|
35
|
-
* Anthropic encodes "hold these modifier keys while clicking" as a `+`
|
|
36
|
-
* separated string on the `text` field — e.g.
|
|
37
|
-
* `{ action: 'left_click', coordinate: [400, 200], text: 'shift+ctrl' }`.
|
|
38
|
-
* The executor translates `cmd`/`super`/`meta` → `Meta`,
|
|
39
|
-
* `ctrl`/`control` → `Control`, etc. before passing to Playwright.
|
|
40
|
-
*/
|
|
41
|
-
/**
|
|
42
|
-
* Construct a fresh {@link ComputerExecutorState}. Optional `initial`
|
|
43
|
-
* seeds the cursor (defaults to `(0, 0)`).
|
|
44
|
-
*/
|
|
45
|
-
export function makeExecutorState(initial) {
|
|
46
|
-
return { cursor: { x: initial?.x ?? 0, y: initial?.y ?? 0 } };
|
|
47
|
-
}
|
|
48
|
-
//# sourceMappingURL=actions.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"actions.js","sourceRoot":"","sources":["../../src/computer-use/actions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAqFH;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAkC;IAClE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,CAAA;AAC/D,CAAC"}
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Error classes for `@rudderjs/ai/computer-use` (#A7 Phase 2).
|
|
3
|
-
*
|
|
4
|
-
* Both extend native `Error` so apps can `instanceof`-check from
|
|
5
|
-
* `try/catch`, error middleware, observers, etc.
|
|
6
|
-
*/
|
|
7
|
-
/**
|
|
8
|
-
* Thrown by {@link computerUseTool} at construction time when the
|
|
9
|
-
* caller supplies a `model` that isn't an Anthropic-family model
|
|
10
|
-
* (matches `anthropic/*` or `bedrock/<region.>?anthropic.*`).
|
|
11
|
-
*
|
|
12
|
-
* Computer-use is Anthropic-only in v1 — see plan
|
|
13
|
-
* `docs/plans/2026-05-10-ai-computer-use.md`. Other providers either
|
|
14
|
-
* lack native computer-use entirely (Google, Cohere, …) or only have
|
|
15
|
-
* preview-quality versions that aren't worth shipping yet (OpenAI's
|
|
16
|
-
* `computer_use_preview`).
|
|
17
|
-
*
|
|
18
|
-
* Apps that wire `computerUseTool({ page, model: this.model() })`
|
|
19
|
-
* inside `Agent.tools()` get this error at agent boot — fail loud
|
|
20
|
-
* before the model gets a chance to hallucinate tool calls against a
|
|
21
|
-
* provider that can't execute them.
|
|
22
|
-
*/
|
|
23
|
-
export declare class ComputerUseProviderError extends Error {
|
|
24
|
-
readonly code: "COMPUTER_USE_PROVIDER_MISMATCH";
|
|
25
|
-
readonly model: string;
|
|
26
|
-
constructor(model: string);
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Thrown by {@link computerUseTool}'s execute when the per-run action
|
|
30
|
-
* counter exceeds {@link ComputerUseToolOptions.maxActions}.
|
|
31
|
-
*
|
|
32
|
-
* Bounds runaway agent loops (e.g. a model that keeps clicking the
|
|
33
|
-
* same broken button forever). Default cap is 50 — most real
|
|
34
|
-
* computer-use tasks finish well under that. Override via
|
|
35
|
-
* `computerUseTool({ page, maxActions: 100 })`.
|
|
36
|
-
*/
|
|
37
|
-
export declare class ComputerUseLimitError extends Error {
|
|
38
|
-
readonly code: "COMPUTER_USE_LIMIT_EXCEEDED";
|
|
39
|
-
readonly maxActions: number;
|
|
40
|
-
constructor(maxActions: number);
|
|
41
|
-
}
|
|
42
|
-
/**
|
|
43
|
-
* Returns true when `model` is an Anthropic-family model id —
|
|
44
|
-
* `anthropic/*` or `bedrock/<region.>?anthropic.*` (covers cross-region
|
|
45
|
-
* inference profiles like `us.anthropic.*`, `eu.anthropic.*`,
|
|
46
|
-
* `apac.anthropic.*`).
|
|
47
|
-
*
|
|
48
|
-
* Used by {@link computerUseTool}'s upfront `model` check. Exported so
|
|
49
|
-
* apps can guard their own code path symmetrically.
|
|
50
|
-
*
|
|
51
|
-
* **Excludes** OpenRouter-routed Anthropic models
|
|
52
|
-
* (`openrouter/anthropic/*`) — OpenRouter goes through the OpenAI SDK
|
|
53
|
-
* with a different base URL, so the request never hits Anthropic's
|
|
54
|
-
* native API and the native computer-use tool block can't be sent.
|
|
55
|
-
*/
|
|
56
|
-
export declare function isAnthropicLikeModel(model: string): boolean;
|
|
57
|
-
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/computer-use/errors.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,EAAG,gCAAgC,CAAS;IACzD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;gBAEV,KAAK,EAAE,MAAM;CAQ1B;AAED;;;;;;;;GAQG;AACH,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,QAAQ,CAAC,IAAI,EAAG,6BAA6B,CAAS;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;gBAEf,UAAU,EAAE,MAAM;CAQ/B;AAID;;;;;;;;;;;;;GAaG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAM3D"}
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Error classes for `@rudderjs/ai/computer-use` (#A7 Phase 2).
|
|
3
|
-
*
|
|
4
|
-
* Both extend native `Error` so apps can `instanceof`-check from
|
|
5
|
-
* `try/catch`, error middleware, observers, etc.
|
|
6
|
-
*/
|
|
7
|
-
/**
|
|
8
|
-
* Thrown by {@link computerUseTool} at construction time when the
|
|
9
|
-
* caller supplies a `model` that isn't an Anthropic-family model
|
|
10
|
-
* (matches `anthropic/*` or `bedrock/<region.>?anthropic.*`).
|
|
11
|
-
*
|
|
12
|
-
* Computer-use is Anthropic-only in v1 — see plan
|
|
13
|
-
* `docs/plans/2026-05-10-ai-computer-use.md`. Other providers either
|
|
14
|
-
* lack native computer-use entirely (Google, Cohere, …) or only have
|
|
15
|
-
* preview-quality versions that aren't worth shipping yet (OpenAI's
|
|
16
|
-
* `computer_use_preview`).
|
|
17
|
-
*
|
|
18
|
-
* Apps that wire `computerUseTool({ page, model: this.model() })`
|
|
19
|
-
* inside `Agent.tools()` get this error at agent boot — fail loud
|
|
20
|
-
* before the model gets a chance to hallucinate tool calls against a
|
|
21
|
-
* provider that can't execute them.
|
|
22
|
-
*/
|
|
23
|
-
export class ComputerUseProviderError extends Error {
|
|
24
|
-
code = 'COMPUTER_USE_PROVIDER_MISMATCH';
|
|
25
|
-
model;
|
|
26
|
-
constructor(model) {
|
|
27
|
-
super(`[Rudder AI] computerUseTool is Anthropic-only in v1; got model "${model}". ` +
|
|
28
|
-
`Use an "anthropic/*" or "bedrock/<region.>?anthropic.*" model, or remove the tool.`);
|
|
29
|
-
this.name = 'ComputerUseProviderError';
|
|
30
|
-
this.model = model;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
/**
|
|
34
|
-
* Thrown by {@link computerUseTool}'s execute when the per-run action
|
|
35
|
-
* counter exceeds {@link ComputerUseToolOptions.maxActions}.
|
|
36
|
-
*
|
|
37
|
-
* Bounds runaway agent loops (e.g. a model that keeps clicking the
|
|
38
|
-
* same broken button forever). Default cap is 50 — most real
|
|
39
|
-
* computer-use tasks finish well under that. Override via
|
|
40
|
-
* `computerUseTool({ page, maxActions: 100 })`.
|
|
41
|
-
*/
|
|
42
|
-
export class ComputerUseLimitError extends Error {
|
|
43
|
-
code = 'COMPUTER_USE_LIMIT_EXCEEDED';
|
|
44
|
-
maxActions;
|
|
45
|
-
constructor(maxActions) {
|
|
46
|
-
super(`[Rudder AI] computerUseTool exceeded maxActions cap of ${maxActions}. ` +
|
|
47
|
-
`Bump the cap with computerUseTool({ page, maxActions: <n> }) if your agent legitimately needs more steps.`);
|
|
48
|
-
this.name = 'ComputerUseLimitError';
|
|
49
|
-
this.maxActions = maxActions;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
// ─── Model classification ─────────────────────────────────
|
|
53
|
-
/**
|
|
54
|
-
* Returns true when `model` is an Anthropic-family model id —
|
|
55
|
-
* `anthropic/*` or `bedrock/<region.>?anthropic.*` (covers cross-region
|
|
56
|
-
* inference profiles like `us.anthropic.*`, `eu.anthropic.*`,
|
|
57
|
-
* `apac.anthropic.*`).
|
|
58
|
-
*
|
|
59
|
-
* Used by {@link computerUseTool}'s upfront `model` check. Exported so
|
|
60
|
-
* apps can guard their own code path symmetrically.
|
|
61
|
-
*
|
|
62
|
-
* **Excludes** OpenRouter-routed Anthropic models
|
|
63
|
-
* (`openrouter/anthropic/*`) — OpenRouter goes through the OpenAI SDK
|
|
64
|
-
* with a different base URL, so the request never hits Anthropic's
|
|
65
|
-
* native API and the native computer-use tool block can't be sent.
|
|
66
|
-
*/
|
|
67
|
-
export function isAnthropicLikeModel(model) {
|
|
68
|
-
if (model.startsWith('anthropic/'))
|
|
69
|
-
return true;
|
|
70
|
-
// Bedrock: bedrock/anthropic.* OR bedrock/<region>.anthropic.*
|
|
71
|
-
// (region prefixes us./eu./apac./...)
|
|
72
|
-
if (/^bedrock\/(?:[a-z]{2,4}\.)?anthropic\./.test(model))
|
|
73
|
-
return true;
|
|
74
|
-
return false;
|
|
75
|
-
}
|
|
76
|
-
//# sourceMappingURL=errors.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/computer-use/errors.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,wBAAyB,SAAQ,KAAK;IACxC,IAAI,GAAG,gCAAyC,CAAA;IAChD,KAAK,CAAQ;IAEtB,YAAY,KAAa;QACvB,KAAK,CACH,mEAAmE,KAAK,KAAK;YAC7E,oFAAoF,CACrF,CAAA;QACD,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAA;QACtC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACpB,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IACrC,IAAI,GAAG,6BAAsC,CAAA;IAC7C,UAAU,CAAQ;IAE3B,YAAY,UAAkB;QAC5B,KAAK,CACH,0DAA0D,UAAU,IAAI;YACxE,2GAA2G,CAC5G,CAAA;QACD,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAA;QACnC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;IAC9B,CAAC;CACF;AAED,6DAA6D;AAE7D;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,IAAI,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAA;IAC/C,+DAA+D;IAC/D,sCAAsC;IACtC,IAAI,wCAAwC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACrE,OAAO,KAAK,CAAA;AACd,CAAC"}
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Playwright executor for {@link ComputerAction} (#A7 Phase 1).
|
|
3
|
-
*
|
|
4
|
-
* Translates Anthropic's `computer_20250124` action vocabulary into
|
|
5
|
-
* Playwright `Page` calls. Stateless apart from {@link ComputerExecutorState},
|
|
6
|
-
* which the caller threads through so `cursor_position` can answer.
|
|
7
|
-
*
|
|
8
|
-
* # Errors
|
|
9
|
-
*
|
|
10
|
-
* Playwright throws on failures (selector misses, timeouts, navigation
|
|
11
|
-
* during action). The executor catches and returns
|
|
12
|
-
* `{ type: 'error', text: <message> }` so the agent loop can hand the
|
|
13
|
-
* failure back to the model as a tool-result with `is_error: true`. The
|
|
14
|
-
* model decides whether to retry, recover, or give up — failing the
|
|
15
|
-
* whole agent run on a single missed click is too brittle.
|
|
16
|
-
*
|
|
17
|
-
* # Modifier keys & key chords
|
|
18
|
-
*
|
|
19
|
-
* Anthropic uses xdotool naming (`ctrl`, `cmd`, `Return`); Playwright
|
|
20
|
-
* uses its own (`Control`, `Meta`, `Enter`). {@link normalizeKey} maps
|
|
21
|
-
* the common ones; unmapped keys pass through verbatim and Playwright
|
|
22
|
-
* either accepts them (single chars, function keys) or throws — which
|
|
23
|
-
* surfaces as the standard error path above.
|
|
24
|
-
*
|
|
25
|
-
* # Scroll units
|
|
26
|
-
*
|
|
27
|
-
* Anthropic's `scroll_amount` is in mouse-wheel "clicks". A typical
|
|
28
|
-
* desktop wheel click is ~100px; the executor multiplies through
|
|
29
|
-
* {@link SCROLL_PIXELS_PER_CLICK}. Tunable later if customer feedback
|
|
30
|
-
* shows drift.
|
|
31
|
-
*/
|
|
32
|
-
import type { ComputerAction, ComputerActionResult, ComputerExecutorState, PageLike } from './actions.js';
|
|
33
|
-
/**
|
|
34
|
-
* Pixels per `scroll_amount` unit. Roughly matches a desktop mouse
|
|
35
|
-
* wheel click on most platforms.
|
|
36
|
-
*/
|
|
37
|
-
export declare const SCROLL_PIXELS_PER_CLICK = 100;
|
|
38
|
-
/**
|
|
39
|
-
* Number of intermediate move steps Playwright interpolates when the
|
|
40
|
-
* executor moves the mouse to a target. Higher = smoother (more hover
|
|
41
|
-
* events fire), but slower. 5 is a reasonable default — instant enough
|
|
42
|
-
* for most pages, slow enough that hover-driven UI (tooltips, menus)
|
|
43
|
-
* has a chance to react.
|
|
44
|
-
*/
|
|
45
|
-
export declare const MOUSE_MOVE_STEPS = 5;
|
|
46
|
-
/**
|
|
47
|
-
* Dispatch a single {@link ComputerAction} against a Playwright `Page`.
|
|
48
|
-
*
|
|
49
|
-
* Updates `state.cursor` after every action that targets a coordinate
|
|
50
|
-
* (move / click / drag / scroll). Caller owns `state` and threads it
|
|
51
|
-
* through every call within an agent run.
|
|
52
|
-
*
|
|
53
|
-
* Returns a {@link ComputerActionResult} suitable for forwarding to the
|
|
54
|
-
* model as a tool-result. Never throws — Playwright failures surface as
|
|
55
|
-
* `{ type: 'error', text }`.
|
|
56
|
-
*/
|
|
57
|
-
export declare function executeComputerAction(page: PageLike, action: ComputerAction, state: ComputerExecutorState): Promise<ComputerActionResult>;
|
|
58
|
-
/**
|
|
59
|
-
* Parse Anthropic's `+`-separated modifier text into Playwright key
|
|
60
|
-
* names. Empty / undefined input → empty array (no modifiers).
|
|
61
|
-
*/
|
|
62
|
-
export declare function parseModifiers(text: string | undefined): string[];
|
|
63
|
-
/**
|
|
64
|
-
* Map Anthropic / xdotool key names to Playwright key names.
|
|
65
|
-
* Unmapped keys (single chars, F-keys, anything we haven't covered)
|
|
66
|
-
* pass through verbatim — Playwright either accepts them or throws,
|
|
67
|
-
* which is the right failure mode.
|
|
68
|
-
*/
|
|
69
|
-
export declare function normalizeKey(key: string): string;
|
|
70
|
-
/**
|
|
71
|
-
* Normalize a key chord (`+`-separated). Playwright's
|
|
72
|
-
* `keyboard.press()` parses chords natively, so we just normalize each
|
|
73
|
-
* segment and rejoin.
|
|
74
|
-
*/
|
|
75
|
-
export declare function normalizeChord(chord: string): string;
|
|
76
|
-
//# sourceMappingURL=playwright.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.d.ts","sourceRoot":"","sources":["../../src/computer-use/playwright.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,KAAK,EACV,cAAc,EACd,oBAAoB,EACpB,qBAAqB,EACrB,QAAQ,EACT,MAAM,cAAc,CAAA;AAErB;;;GAGG;AACH,eAAO,MAAM,uBAAuB,MAAM,CAAA;AAE1C;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,IAAI,CAAA;AAEjC;;;;;;;;;;GAUG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,qBAAqB,GAC3B,OAAO,CAAC,oBAAoB,CAAC,CA4I/B;AAQD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAMjE;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CA8BhD;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKpD"}
|