@rudderjs/ai 1.4.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/README.md +484 -7
  2. package/boost/guidelines.md +62 -2
  3. package/boost/skills/ai-tools/SKILL.md +14 -5
  4. package/dist/agent.d.ts +66 -15
  5. package/dist/agent.d.ts.map +1 -1
  6. package/dist/agent.js +529 -58
  7. package/dist/agent.js.map +1 -1
  8. package/dist/budget/pricing.d.ts +124 -0
  9. package/dist/budget/pricing.d.ts.map +1 -0
  10. package/dist/budget/pricing.js +175 -0
  11. package/dist/budget/pricing.js.map +1 -0
  12. package/dist/budget/storage.d.ts +104 -0
  13. package/dist/budget/storage.d.ts.map +1 -0
  14. package/dist/budget/storage.js +0 -0
  15. package/dist/budget/storage.js.map +1 -0
  16. package/dist/budget/with-budget.d.ts +119 -0
  17. package/dist/budget/with-budget.d.ts.map +1 -0
  18. package/dist/budget/with-budget.js +175 -0
  19. package/dist/budget/with-budget.js.map +1 -0
  20. package/dist/budget-orm/index.d.ts +96 -0
  21. package/dist/budget-orm/index.d.ts.map +1 -0
  22. package/dist/budget-orm/index.js +177 -0
  23. package/dist/budget-orm/index.js.map +1 -0
  24. package/dist/commands/ai-eval.d.ts +93 -0
  25. package/dist/commands/ai-eval.d.ts.map +1 -0
  26. package/dist/commands/ai-eval.js +378 -0
  27. package/dist/commands/ai-eval.js.map +1 -0
  28. package/dist/computer-use/actions.d.ts +214 -0
  29. package/dist/computer-use/actions.d.ts.map +1 -0
  30. package/dist/computer-use/actions.js +48 -0
  31. package/dist/computer-use/actions.js.map +1 -0
  32. package/dist/computer-use/errors.d.ts +57 -0
  33. package/dist/computer-use/errors.d.ts.map +1 -0
  34. package/dist/computer-use/errors.js +76 -0
  35. package/dist/computer-use/errors.js.map +1 -0
  36. package/dist/computer-use/index.d.ts +53 -0
  37. package/dist/computer-use/index.d.ts.map +1 -0
  38. package/dist/computer-use/index.js +51 -0
  39. package/dist/computer-use/index.js.map +1 -0
  40. package/dist/computer-use/playwright.d.ts +76 -0
  41. package/dist/computer-use/playwright.d.ts.map +1 -0
  42. package/dist/computer-use/playwright.js +270 -0
  43. package/dist/computer-use/playwright.js.map +1 -0
  44. package/dist/computer-use/tool.d.ts +154 -0
  45. package/dist/computer-use/tool.d.ts.map +1 -0
  46. package/dist/computer-use/tool.js +210 -0
  47. package/dist/computer-use/tool.js.map +1 -0
  48. package/dist/eval/fixtures.d.ts +65 -0
  49. package/dist/eval/fixtures.d.ts.map +1 -0
  50. package/dist/eval/fixtures.js +110 -0
  51. package/dist/eval/fixtures.js.map +1 -0
  52. package/dist/eval/html-reporter.d.ts +25 -0
  53. package/dist/eval/html-reporter.d.ts.map +1 -0
  54. package/dist/eval/html-reporter.js +209 -0
  55. package/dist/eval/html-reporter.js.map +1 -0
  56. package/dist/eval/index.d.ts +271 -0
  57. package/dist/eval/index.d.ts.map +1 -0
  58. package/dist/eval/index.js +510 -0
  59. package/dist/eval/index.js.map +1 -0
  60. package/dist/eval/json-reporter.d.ts +43 -0
  61. package/dist/eval/json-reporter.d.ts.map +1 -0
  62. package/dist/eval/json-reporter.js +40 -0
  63. package/dist/eval/json-reporter.js.map +1 -0
  64. package/dist/fake.d.ts +36 -1
  65. package/dist/fake.d.ts.map +1 -1
  66. package/dist/fake.js +49 -2
  67. package/dist/fake.js.map +1 -1
  68. package/dist/file-search.d.ts +168 -0
  69. package/dist/file-search.d.ts.map +1 -0
  70. package/dist/file-search.js +158 -0
  71. package/dist/file-search.js.map +1 -0
  72. package/dist/handoff.d.ts +95 -0
  73. package/dist/handoff.d.ts.map +1 -0
  74. package/dist/handoff.js +78 -0
  75. package/dist/handoff.js.map +1 -0
  76. package/dist/index.d.ts +29 -5
  77. package/dist/index.d.ts.map +1 -1
  78. package/dist/index.js +22 -2
  79. package/dist/index.js.map +1 -1
  80. package/dist/mcp/client-tools.d.ts +39 -0
  81. package/dist/mcp/client-tools.d.ts.map +1 -0
  82. package/dist/mcp/client-tools.js +147 -0
  83. package/dist/mcp/client-tools.js.map +1 -0
  84. package/dist/mcp/index.d.ts +16 -0
  85. package/dist/mcp/index.d.ts.map +1 -0
  86. package/dist/mcp/index.js +15 -0
  87. package/dist/mcp/index.js.map +1 -0
  88. package/dist/mcp/server-from-agent.d.ts +24 -0
  89. package/dist/mcp/server-from-agent.d.ts.map +1 -0
  90. package/dist/mcp/server-from-agent.js +113 -0
  91. package/dist/mcp/server-from-agent.js.map +1 -0
  92. package/dist/mcp/types.d.ts +64 -0
  93. package/dist/mcp/types.d.ts.map +1 -0
  94. package/dist/mcp/types.js +6 -0
  95. package/dist/mcp/types.js.map +1 -0
  96. package/dist/memory-embedding/index.d.ts +121 -0
  97. package/dist/memory-embedding/index.d.ts.map +1 -0
  98. package/dist/memory-embedding/index.js +229 -0
  99. package/dist/memory-embedding/index.js.map +1 -0
  100. package/dist/memory-extract.d.ts +60 -0
  101. package/dist/memory-extract.d.ts.map +1 -0
  102. package/dist/memory-extract.js +163 -0
  103. package/dist/memory-extract.js.map +1 -0
  104. package/dist/memory-inject.d.ts +39 -0
  105. package/dist/memory-inject.d.ts.map +1 -0
  106. package/dist/memory-inject.js +135 -0
  107. package/dist/memory-inject.js.map +1 -0
  108. package/dist/memory-orm/index.d.ts +118 -0
  109. package/dist/memory-orm/index.d.ts.map +1 -0
  110. package/dist/memory-orm/index.js +187 -0
  111. package/dist/memory-orm/index.js.map +1 -0
  112. package/dist/memory.d.ts +55 -0
  113. package/dist/memory.d.ts.map +1 -0
  114. package/dist/memory.js +132 -0
  115. package/dist/memory.js.map +1 -0
  116. package/dist/observers.d.ts +22 -0
  117. package/dist/observers.d.ts.map +1 -1
  118. package/dist/observers.js.map +1 -1
  119. package/dist/provider-tools.d.ts +15 -1
  120. package/dist/provider-tools.d.ts.map +1 -1
  121. package/dist/provider-tools.js +21 -1
  122. package/dist/provider-tools.js.map +1 -1
  123. package/dist/providers/anthropic.d.ts +9 -1
  124. package/dist/providers/anthropic.d.ts.map +1 -1
  125. package/dist/providers/anthropic.js +66 -11
  126. package/dist/providers/anthropic.js.map +1 -1
  127. package/dist/providers/bedrock.d.ts +60 -0
  128. package/dist/providers/bedrock.d.ts.map +1 -0
  129. package/dist/providers/bedrock.js +167 -0
  130. package/dist/providers/bedrock.js.map +1 -0
  131. package/dist/providers/elevenlabs.d.ts +98 -0
  132. package/dist/providers/elevenlabs.d.ts.map +1 -0
  133. package/dist/providers/elevenlabs.js +229 -0
  134. package/dist/providers/elevenlabs.js.map +1 -0
  135. package/dist/providers/google.d.ts +83 -1
  136. package/dist/providers/google.d.ts.map +1 -1
  137. package/dist/providers/google.js +491 -8
  138. package/dist/providers/google.js.map +1 -1
  139. package/dist/providers/openai.d.ts +8 -1
  140. package/dist/providers/openai.d.ts.map +1 -1
  141. package/dist/providers/openai.js +215 -5
  142. package/dist/providers/openai.js.map +1 -1
  143. package/dist/providers/openrouter.d.ts +43 -0
  144. package/dist/providers/openrouter.d.ts.map +1 -0
  145. package/dist/providers/openrouter.js +21 -0
  146. package/dist/providers/openrouter.js.map +1 -0
  147. package/dist/providers/voyage.d.ts +91 -0
  148. package/dist/providers/voyage.d.ts.map +1 -0
  149. package/dist/providers/voyage.js +166 -0
  150. package/dist/providers/voyage.js.map +1 -0
  151. package/dist/queue-job.d.ts +69 -4
  152. package/dist/queue-job.d.ts.map +1 -1
  153. package/dist/queue-job.js +114 -11
  154. package/dist/queue-job.js.map +1 -1
  155. package/dist/registry.d.ts +3 -1
  156. package/dist/registry.d.ts.map +1 -1
  157. package/dist/registry.js +10 -0
  158. package/dist/registry.js.map +1 -1
  159. package/dist/server/provider.d.ts.map +1 -1
  160. package/dist/server/provider.js +38 -1
  161. package/dist/server/provider.js.map +1 -1
  162. package/dist/similarity-search.d.ts +163 -0
  163. package/dist/similarity-search.d.ts.map +1 -0
  164. package/dist/similarity-search.js +147 -0
  165. package/dist/similarity-search.js.map +1 -0
  166. package/dist/sub-agent-run-store.d.ts +40 -3
  167. package/dist/sub-agent-run-store.d.ts.map +1 -1
  168. package/dist/sub-agent-run-store.js.map +1 -1
  169. package/dist/tool.d.ts +59 -0
  170. package/dist/tool.d.ts.map +1 -1
  171. package/dist/tool.js +45 -4
  172. package/dist/tool.js.map +1 -1
  173. package/dist/types.d.ts +285 -1
  174. package/dist/types.d.ts.map +1 -1
  175. package/dist/vector-stores/index.d.ts +96 -0
  176. package/dist/vector-stores/index.d.ts.map +1 -0
  177. package/dist/vector-stores/index.js +153 -0
  178. package/dist/vector-stores/index.js.map +1 -0
  179. package/package.json +43 -4
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ai-eval.js","sourceRoot":"","sources":["../../src/commands/ai-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAA;AAC1C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAExF,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AAErD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AACrD,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAA;AACnF,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AA+DnC,2DAA2D;AAC3D,MAAM,UAAU,qBAAqB,CAAC,MAAc;IAClD,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAiB,EAAE,EAAE;QACpD,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAA;QACjD,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC,CAAC,CAAC,WAAW,CACZ,4GAA4G,CAC7G,CAAA;AACH,CAAC;AAED,4DAA4D;AAE5D,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;AAEvC;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,MAAM,IAAI,GAAkB,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;IAExD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAE,CAAA;QAClB,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAAC,SAAQ;QAAC,CAAC;QAEzD,sBAAsB;QACtB,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACzB,MAAM,IAAI,GAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,MAAM,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QAEpD,IAAI,IAAI,KAAK,QAAQ,EAAI,CAAC;YAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YAAC,SAAQ;QAAC,CAAC;QACvD,IAAI,IAAI,KAAK,QAAQ,EAAI,CAAC;YAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YAAC,SAAQ;QAAC,CAAC;QACvD,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YAAC,SAAQ;QAAC,CAAC;QACzD,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YAAC,SAAQ;QAAC,CAAC;QACzD,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,MAAM,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACnC,IAAI,CAAC,MAAM;gBAAE,CAAC,EAAE,CAAA,CAAG,wBAAwB;YAC3C,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,mBAAmB,CAAC,CAAA;YACrE,IAAI,IAAI,KAAK,QAAQ;gBAAE,IAAI,CAAC,IAAI,GAAG,KAAK,CAAA;YACxC,SAAQ;QACV,CAAC;QACD,iEAAiE;QACjE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,IAAI,UAAU,CAAC,CAAC,CAAC;QAAE,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9C,OAAO,IAAI,CAAA;AACb,CAAC;AAED,4DAA4D;AAE5D;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAmB,EAAE,OAAmB,EAAE;IACzE,MAAM,GAAG,GAAM,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAA;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAA;IAE5C,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,CAAC,KAAK,CAAC,0DAA0D,CAAC,CAAA;QACxE,OAAO,CAAC,CAAA;IACV,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,IAAI,iBAAiB,CAAC,EAAE,CAAC,IAAI,oBAAoB,CAAA;IAC1G,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,kBAAkB,CAAA;IACpD,MAAM,KAAK,GAAM,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IAE7C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,KAAK,CAAC,sCAAsC,OAAO,IAAI,CAAC,CAAA;QAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC7C,CAAC;IAED,MAAM,MAAM,GAAQ,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAA;IACxD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,kBAAkB,CAAC,GAAG,CAAC,CAAA;IAC/D,MAAM,OAAO,GAAgB,EAAE,CAAA;IAC/B,MAAM,WAAW,GAAkB,EAAE,CAAA;IACrC,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,sEAAsE;IACtE,gEAAgE;IAChE,kEAAkE;IAClE,IAAI,IAAI,GAAkB,IAAI,CAAA;IAC9B,IAAI,IAAI,CAAC,MAAM;QAAE,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;IAErC,IAAI,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,KAAuB,CAAA;YAC3B,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBAC3F,QAAQ,GAAG,CAAC,CAAA;gBACZ,IAAI,IAAI,CAAC,IAAI;oBAAE,MAAK;gBACpB,SAAQ;YACV,CAAC;YACD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,yCAAyC,CAAC,CAAA;gBAC5F,SAAQ;YACV,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBAAE,SAAQ;YAE1F,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YACnF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAA;YACxC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACxB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAA;YAClC,CAAC;iBAAM,CAAC;gBACN,aAAa,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC/D,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,QAAQ,GAAG,CAAC,CAAA;gBACZ,IAAI,IAAI,CAAC,IAAI;oBAAE,MAAK;YACtB,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,IAAI;YAAE,IAAI,CAAC,OAAO,EAAE,CAAA;IAC1B,CAAC;IAED,IAAI,IAAI,CAAC,IAAI;QAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACxC,IAAI,IAAI,CAAC,IAAI;QAAE,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,CAAC,CAAA;IACzE,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,QAAmB,EACnB,OAA0B,EAC1B,GAAmB,EACnB,MAAiD;IAEjD,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;IAC9E,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACnD,MAAM,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAA;QACzC,MAAM,CAAC,KAAK,CAAC,iCAAiC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,CAAA;IAC5E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,yCAAyC,GAAG,KAAK,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACrF,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,MAA4C,EAAE,MAAmB;IACjF,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;IACxD,OAAO,CAAC,CAAA;AACV,CAAC;AAED,SAAS,WAAW,CAAC,GAAY;IAC/B,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;AACzD,CAAC;AAUD;;;;;;;;;;;;GAYG;AACH,KAAK,UAAU,eAAe,CAC5B,KAAgB,EAChB,IAAoB,EACpB,GAAsB;IAEtB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAA;IAE9C,gEAAgE;IAChE,4CAA4C;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAwB,CAAA;IACnD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAA;YAC9B,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC,EAAE,CAAA;YACtC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;gBACxE,IAAI,OAAO;oBAAE,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;;oBAChD,GAAG,CAAC,MAAM,CAAC,KAAK,CACnB,4BAA4B,KAAK,CAAC,IAAI,IAAI,QAAQ,oCAAoC,CACvF,CAAA;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,KAAK,CAAC,IAAI,IAAI,QAAQ,KAAK,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACvG,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAY,EAAE;QACtD,MAAM,QAAQ,GAAM,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC,EAAE,CAAA;QACzC,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAA;QAC/C,MAAM,UAAU,GAAI,CAAC,CAAC,MAAM,CAAA;QAE5B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM;YACzB,CAAC,CAAC,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC;YACrE,CAAC,CAAC,WAAW,CAAA;QAEf,MAAM,MAAM,GAAW,IAAI,CAAC,MAAM;YAChC,CAAC,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC;YAClE,CAAC,CAAC,UAAU,CAAA;QAEd,MAAM,GAAG,GAAa;YACpB,KAAK,EAAG,CAAC,CAAC,KAAK;YACf,MAAM;YACN,KAAK,EAAG,OAAO;SAChB,CAAA;QACD,IAAI,CAAC,CAAC,IAAI;YAAgB,GAAG,CAAC,IAAI,GAAM,CAAC,CAAC,IAAI,CAAA;QAC9C,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS;YAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAA;QACpD,IAAI,CAAC,CAAC,IAAI,KAAO,SAAS;YAAE,GAAG,CAAC,IAAI,GAAM,CAAC,CAAC,IAAI,CAAA;QAChD,OAAO,GAAG,CAAA;IACZ,CAAC,CAAC,CAAA;IAEF,MAAM,OAAO,GAAsB;QACjC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK;QACvB,KAAK,EAAE,OAAO;KACf,CAAA;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,CAAC,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAA;IAC1E,OAAO,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;AACvC,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,iBAAiB,CACxB,IAAkB,EAClB,KAA+B,EAC/B,IAAoB;IAEpB,OAAO,GAAG,EAAE;QACV,IAAI,IAAI,IAAI,KAAK;YAAE,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QAClD,OAAO,IAAI,EAAE,CAAA;IACf,CAAC,CAAA;AACH,CAAC;AAED;;;;;;GAMG;AACH,SAAS,gBAAgB,CACvB,IAAgB,EAChB,KAAgB,EAChB,QAAgB,EAChB,KAAgB,EAChB,GAAyB;IAEzB,OAAO,KAAK,EAAE,QAAuB,EAAE,IAAI,EAAE,EAAE;QAC7C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE;gBAChE,KAAK;gBACL,KAAK,EAAE,iBAAiB,CAAC,QAAQ,CAAC;aACnC,CAAC,CAAA;YACF,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;QAChF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,KAAK,IAAI,QAAQ,KAAK,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC5F,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IAC7B,CAAC,CAAA;AACH,CAAC;AAED,4DAA4D;AAE5D;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,GAAW,EAAE,OAAe;IACnE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,CAAA;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IACvC,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,CAAA;IAChC,OAAO,GAAG,CAAC,IAAI,EAAE,CAAA;AACnB,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,YAAY,CAAC,OAAe;IACnC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACxC,IAAI,MAAe,CAAA;IACnB,IAAI,OAAe,CAAA;IACnB,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACpB,MAAM,GAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QACzD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;IAC5D,CAAC;SAAM,CAAC;QACN,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;QAC1C,MAAM,GAAI,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAC3D,OAAO,GAAG,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;IACnE,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,2CAA2C,OAAO,KAAK;YACvD,2CAA2C,CAC5C,CAAA;IACH,CAAC;IACD,OAAO;QACL,IAAI,EAAI,MAAM,IAAI,GAAG;QACrB,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;KACzB,CAAA;AACH,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,MAAc,EAAE,GAAa;IAC5D,IAAI,OAAO,CAAA;IACX,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAA;IACvD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAM;QAC5D,MAAM,GAAG,CAAA;IACX,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAQ;YACzE,MAAM,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAA;QAC5B,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACzD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACb,CAAC;IACH,CAAC;AACH,CAAC;AAED,4DAA4D;AAE5D,KAAK,UAAU,kBAAkB,CAAC,IAAY;IAC5C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAA4B,CAAA;IAC7E,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,CAA0B,CAAA;IAC3E,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACpF,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,4DAA4D;AAE5D;;;;;GAKG;AACH,KAAK,UAAU,iBAAiB;IAC9B,IAAI,CAAC;QACH,kEAAkE;QAClE,kEAAkE;QAClE,0BAA0B;QAC1B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAiD,CAAA;QAC3F,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU;YAAE,OAAO,IAAI,CAAA;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAA8C,IAAI,CAAC,CAAA;QAC1E,OAAO,GAAG,EAAE,IAAI,EAAE,OAAO,IAAI,IAAI,CAAA;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC"}
@@ -0,0 +1,214 @@
1
+ /**
2
+ * Action vocabulary for `@rudderjs/ai/computer-use` (#A7 Phase 1).
3
+ *
4
+ * Mirrors Anthropic's `computer_20250124` tool schema verbatim — see
5
+ * https://docs.anthropic.com/en/docs/build-with-claude/tool-use/computer-use-tool
6
+ *
7
+ * The model emits a {@link ComputerAction} as the JSON argument of a
8
+ * `computer` tool call; the executor (see `./playwright.ts`) dispatches
9
+ * the action against a Playwright `Page` and returns a
10
+ * {@link ComputerActionResult} the agent loop forwards back to the model.
11
+ *
12
+ * # Why mirror Anthropic exactly?
13
+ *
14
+ * Computer-use is `Anthropic-only` in v1 (see plan
15
+ * `docs/plans/2026-05-10-ai-computer-use.md`). Claude is fine-tuned on
16
+ * this exact action vocabulary; reusing the schema means:
17
+ *
18
+ * - Phase 2's `computerUseTool({ page })` factory maps cleanly to
19
+ * Anthropic's native `{ type: 'computer_20250124', name: 'computer',
20
+ * display_width_px, display_height_px }` block — zero translation
21
+ * layer.
22
+ * - When OpenAI or Google's native computer-use APIs mature enough to
23
+ * route through, only the provider adapter changes; the schema and
24
+ * executor stay put.
25
+ *
26
+ * # Coordinates
27
+ *
28
+ * `[x, y]` in viewport pixels. Origin is top-left. Anthropic suggests a
29
+ * 1280×800 viewport for Claude's training distribution; the executor
30
+ * does not validate bounds — out-of-viewport coordinates pass through
31
+ * to Playwright (which clips the cursor to viewport edges).
32
+ *
33
+ * # Modifier text on click / scroll actions
34
+ *
35
+ * Anthropic encodes "hold these modifier keys while clicking" as a `+`
36
+ * separated string on the `text` field — e.g.
37
+ * `{ action: 'left_click', coordinate: [400, 200], text: 'shift+ctrl' }`.
38
+ * The executor translates `cmd`/`super`/`meta` → `Meta`,
39
+ * `ctrl`/`control` → `Control`, etc. before passing to Playwright.
40
+ */
41
+ /** Viewport pixel coordinate, top-left origin. `[x, y]`. */
42
+ export type Coordinate = readonly [number, number];
43
+ /**
44
+ * Discriminated union of every action Anthropic's `computer_20250124`
45
+ * tool can emit. The executor dispatches on `action`; downstream code
46
+ * should treat unknown variants as a forward-compatibility hazard
47
+ * rather than a bug — Anthropic versions the schema with dated suffixes
48
+ * (e.g. a future `computer_20260101` may add new variants we'll route
49
+ * here once supported).
50
+ */
51
+ export type ComputerAction = {
52
+ action: 'screenshot';
53
+ } | {
54
+ action: 'cursor_position';
55
+ }
56
+ /** Pause for `duration` seconds. Useful between an action that triggers async UI and the screenshot that should observe the result. */
57
+ | {
58
+ action: 'wait';
59
+ duration: number;
60
+ } | {
61
+ action: 'mouse_move';
62
+ coordinate: Coordinate;
63
+ }
64
+ /** Single left-button click. `text` holds optional modifier keys, `+`-separated (e.g. `'shift+ctrl'`). */
65
+ | {
66
+ action: 'left_click';
67
+ coordinate: Coordinate;
68
+ text?: string;
69
+ } | {
70
+ action: 'right_click';
71
+ coordinate: Coordinate;
72
+ text?: string;
73
+ } | {
74
+ action: 'middle_click';
75
+ coordinate: Coordinate;
76
+ text?: string;
77
+ } | {
78
+ action: 'double_click';
79
+ coordinate: Coordinate;
80
+ text?: string;
81
+ } | {
82
+ action: 'triple_click';
83
+ coordinate: Coordinate;
84
+ text?: string;
85
+ }
86
+ /** Press the left mouse button. Pair with `left_mouse_up` to drag. `coordinate` (optional) moves first. */
87
+ | {
88
+ action: 'left_mouse_down';
89
+ coordinate?: Coordinate;
90
+ }
91
+ /** Release the left mouse button. `coordinate` (optional) moves first. */
92
+ | {
93
+ action: 'left_mouse_up';
94
+ coordinate?: Coordinate;
95
+ }
96
+ /** Type literal text. No modifier handling — use `key` for shortcuts. */
97
+ | {
98
+ action: 'type';
99
+ text: string;
100
+ }
101
+ /** Press a key chord. `text` is `+`-separated (e.g. `'ctrl+a'`, `'Return'`, `'cmd+shift+t'`). */
102
+ | {
103
+ action: 'key';
104
+ text: string;
105
+ }
106
+ /** Hold a single key for `duration` seconds. `text` is one key, not a chord. */
107
+ | {
108
+ action: 'hold_key';
109
+ text: string;
110
+ duration: number;
111
+ }
112
+ /** Scroll at `coordinate`. `scroll_amount` is in mouse-wheel "clicks" (~100px each). `text` holds optional modifiers. */
113
+ | {
114
+ action: 'scroll';
115
+ coordinate: Coordinate;
116
+ scroll_direction: 'up' | 'down' | 'left' | 'right';
117
+ scroll_amount: number;
118
+ text?: string;
119
+ };
120
+ /**
121
+ * Tool-result content the agent loop hands back to the model after each
122
+ * action. Shape matches Anthropic's tool-result block content vocabulary:
123
+ *
124
+ * - `image` carries a PNG screenshot. The agent loop wraps this as
125
+ * `{ type: 'image', source: { type: 'base64', media_type, data } }`
126
+ * when serializing to the API.
127
+ * - `text` carries a one-line confirmation (`"left-clicked at (400, 200)"`)
128
+ * or the response of an introspective action (`cursor_position`).
129
+ * - `error` carries the failure text from a Playwright throw. Caller
130
+ * typically maps this onto `{ is_error: true, content: [...] }` in
131
+ * the tool-result block so the model knows the action failed and can
132
+ * retry / recover.
133
+ */
134
+ export type ComputerActionResult = {
135
+ type: 'image';
136
+ media_type: 'image/png';
137
+ data: Uint8Array;
138
+ } | {
139
+ type: 'text';
140
+ text: string;
141
+ } | {
142
+ type: 'error';
143
+ text: string;
144
+ };
145
+ /**
146
+ * Per-run state the executor mutates. Tracks the last known cursor
147
+ * position so the `cursor_position` action can answer it (Playwright
148
+ * has no native API to read the synthesized mouse position).
149
+ *
150
+ * Create one with {@link makeExecutorState} per agent run; pass the
151
+ * same instance to every {@link executeComputerAction} call.
152
+ */
153
+ export interface ComputerExecutorState {
154
+ /** Last known cursor position in viewport pixels. Updated on move/click/drag. */
155
+ cursor: {
156
+ x: number;
157
+ y: number;
158
+ };
159
+ }
160
+ /**
161
+ * Construct a fresh {@link ComputerExecutorState}. Optional `initial`
162
+ * seeds the cursor (defaults to `(0, 0)`).
163
+ */
164
+ export declare function makeExecutorState(initial?: {
165
+ x: number;
166
+ y: number;
167
+ }): ComputerExecutorState;
168
+ /**
169
+ * Structural subset of Playwright's `Page` the executor calls. Defining
170
+ * this here means {@link executeComputerAction} types correctly without
171
+ * `@rudderjs/ai` taking a hard dependency on the `playwright` package
172
+ * (which carries a 300MB+ Chromium download). Apps install Playwright
173
+ * themselves and pass `page` in.
174
+ *
175
+ * Any object with this shape works — real Playwright `Page`,
176
+ * `puppeteer-core`'s `Page` (close enough), or a hand-rolled mock for
177
+ * tests. Phase 2's `computerUseTool({ page })` accepts the same type.
178
+ */
179
+ export interface PageLike {
180
+ mouse: PageMouseLike;
181
+ keyboard: PageKeyboardLike;
182
+ /** Take a screenshot of the current viewport. Returns PNG bytes by default. */
183
+ screenshot(options?: {
184
+ type?: 'png' | 'jpeg';
185
+ }): Promise<Uint8Array>;
186
+ }
187
+ export interface PageMouseLike {
188
+ move(x: number, y: number, options?: {
189
+ steps?: number;
190
+ }): Promise<void>;
191
+ click(x: number, y: number, options?: {
192
+ button?: 'left' | 'right' | 'middle';
193
+ clickCount?: number;
194
+ delay?: number;
195
+ }): Promise<void>;
196
+ down(options?: {
197
+ button?: 'left' | 'right' | 'middle';
198
+ }): Promise<void>;
199
+ up(options?: {
200
+ button?: 'left' | 'right' | 'middle';
201
+ }): Promise<void>;
202
+ wheel(deltaX: number, deltaY: number): Promise<void>;
203
+ }
204
+ export interface PageKeyboardLike {
205
+ type(text: string, options?: {
206
+ delay?: number;
207
+ }): Promise<void>;
208
+ press(key: string, options?: {
209
+ delay?: number;
210
+ }): Promise<void>;
211
+ down(key: string): Promise<void>;
212
+ up(key: string): Promise<void>;
213
+ }
214
+ //# sourceMappingURL=actions.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../../src/computer-use/actions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAEH,4DAA4D;AAC5D,MAAM,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;AAElD;;;;;;;GAOG;AACH,MAAM,MAAM,cAAc,GAEtB;IAAE,MAAM,EAAE,YAAY,CAAA;CAAE,GACxB;IAAE,MAAM,EAAE,iBAAiB,CAAA;CAAE;AAG/B,uIAAuI;GACrI;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAGpC;IAAE,MAAM,EAAE,YAAY,CAAC;IAAC,UAAU,EAAE,UAAU,CAAA;CAAE;AAClD,0GAA0G;GACxG;IAAE,MAAM,EAAE,YAAY,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/D;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAChE;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GACjE;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GACjE;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE;AACnE,2GAA2G;GACzG;IAAE,MAAM,EAAE,iBAAiB,CAAC;IAAC,UAAU,CAAC,EAAE,UAAU,CAAA;CAAE;AACxD,0EAA0E;GACxE;IAAE,MAAM,EAAE,eAAe,CAAC;IAAC,UAAU,CAAC,EAAE,UAAU,CAAA;CAAE;AAGtD,yEAAyE;GACvE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE;AAClC,iGAAiG;GAC/F;IAAE,MAAM,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE;AACjC,gFAAgF;GAC9E;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE;AAGxD,yHAAyH;GACvH;IACE,MAAM,EAAE,QAAQ,CAAA;IAChB,UAAU,EAAE,UAAU,CAAA;IACtB,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAA;IAClD,aAAa,EAAE,MAAM,CAAA;IACrB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd,CAAA;AAEL;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,oBAAoB,GAC5B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GAC5D;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACpC,iFAAiF;IACjF,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CACjC;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,qBAAqB,CAE3F;AAID;;;;;;;;;;GAUG;AACH,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,aAAa,CAAA;IACpB,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,+EAA+E;IAC/E,UAAU,CAAC,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;CACrE;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACvE,KAAK,CACH,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GACtF,OAAO,CAAC,IAAI,CAAC,CAAA;IAChB,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACvE,EAAE,CAAC,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACrE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CACrD;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/D,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/D,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChC,EAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC/B"}
@@ -0,0 +1,48 @@
1
+ /**
2
+ * Action vocabulary for `@rudderjs/ai/computer-use` (#A7 Phase 1).
3
+ *
4
+ * Mirrors Anthropic's `computer_20250124` tool schema verbatim — see
5
+ * https://docs.anthropic.com/en/docs/build-with-claude/tool-use/computer-use-tool
6
+ *
7
+ * The model emits a {@link ComputerAction} as the JSON argument of a
8
+ * `computer` tool call; the executor (see `./playwright.ts`) dispatches
9
+ * the action against a Playwright `Page` and returns a
10
+ * {@link ComputerActionResult} the agent loop forwards back to the model.
11
+ *
12
+ * # Why mirror Anthropic exactly?
13
+ *
14
+ * Computer-use is `Anthropic-only` in v1 (see plan
15
+ * `docs/plans/2026-05-10-ai-computer-use.md`). Claude is fine-tuned on
16
+ * this exact action vocabulary; reusing the schema means:
17
+ *
18
+ * - Phase 2's `computerUseTool({ page })` factory maps cleanly to
19
+ * Anthropic's native `{ type: 'computer_20250124', name: 'computer',
20
+ * display_width_px, display_height_px }` block — zero translation
21
+ * layer.
22
+ * - When OpenAI or Google's native computer-use APIs mature enough to
23
+ * route through, only the provider adapter changes; the schema and
24
+ * executor stay put.
25
+ *
26
+ * # Coordinates
27
+ *
28
+ * `[x, y]` in viewport pixels. Origin is top-left. Anthropic suggests a
29
+ * 1280×800 viewport for Claude's training distribution; the executor
30
+ * does not validate bounds — out-of-viewport coordinates pass through
31
+ * to Playwright (which clips the cursor to viewport edges).
32
+ *
33
+ * # Modifier text on click / scroll actions
34
+ *
35
+ * Anthropic encodes "hold these modifier keys while clicking" as a `+`
36
+ * separated string on the `text` field — e.g.
37
+ * `{ action: 'left_click', coordinate: [400, 200], text: 'shift+ctrl' }`.
38
+ * The executor translates `cmd`/`super`/`meta` → `Meta`,
39
+ * `ctrl`/`control` → `Control`, etc. before passing to Playwright.
40
+ */
41
+ /**
42
+ * Construct a fresh {@link ComputerExecutorState}. Optional `initial`
43
+ * seeds the cursor (defaults to `(0, 0)`).
44
+ */
45
+ export function makeExecutorState(initial) {
46
+ return { cursor: { x: initial?.x ?? 0, y: initial?.y ?? 0 } };
47
+ }
48
+ //# sourceMappingURL=actions.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"actions.js","sourceRoot":"","sources":["../../src/computer-use/actions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAqFH;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAkC;IAClE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,CAAA;AAC/D,CAAC"}
@@ -0,0 +1,57 @@
1
+ /**
2
+ * Error classes for `@rudderjs/ai/computer-use` (#A7 Phase 2).
3
+ *
4
+ * Both extend native `Error` so apps can `instanceof`-check from
5
+ * `try/catch`, error middleware, observers, etc.
6
+ */
7
+ /**
8
+ * Thrown by {@link computerUseTool} at construction time when the
9
+ * caller supplies a `model` that isn't an Anthropic-family model
10
+ * (matches `anthropic/*` or `bedrock/<region.>?anthropic.*`).
11
+ *
12
+ * Computer-use is Anthropic-only in v1 — see plan
13
+ * `docs/plans/2026-05-10-ai-computer-use.md`. Other providers either
14
+ * lack native computer-use entirely (Google, Cohere, …) or only have
15
+ * preview-quality versions that aren't worth shipping yet (OpenAI's
16
+ * `computer_use_preview`).
17
+ *
18
+ * Apps that wire `computerUseTool({ page, model: this.model() })`
19
+ * inside `Agent.tools()` get this error at agent boot — fail loud
20
+ * before the model gets a chance to hallucinate tool calls against a
21
+ * provider that can't execute them.
22
+ */
23
+ export declare class ComputerUseProviderError extends Error {
24
+ readonly code: "COMPUTER_USE_PROVIDER_MISMATCH";
25
+ readonly model: string;
26
+ constructor(model: string);
27
+ }
28
+ /**
29
+ * Thrown by {@link computerUseTool}'s execute when the per-run action
30
+ * counter exceeds {@link ComputerUseToolOptions.maxActions}.
31
+ *
32
+ * Bounds runaway agent loops (e.g. a model that keeps clicking the
33
+ * same broken button forever). Default cap is 50 — most real
34
+ * computer-use tasks finish well under that. Override via
35
+ * `computerUseTool({ page, maxActions: 100 })`.
36
+ */
37
+ export declare class ComputerUseLimitError extends Error {
38
+ readonly code: "COMPUTER_USE_LIMIT_EXCEEDED";
39
+ readonly maxActions: number;
40
+ constructor(maxActions: number);
41
+ }
42
+ /**
43
+ * Returns true when `model` is an Anthropic-family model id —
44
+ * `anthropic/*` or `bedrock/<region.>?anthropic.*` (covers cross-region
45
+ * inference profiles like `us.anthropic.*`, `eu.anthropic.*`,
46
+ * `apac.anthropic.*`).
47
+ *
48
+ * Used by {@link computerUseTool}'s upfront `model` check. Exported so
49
+ * apps can guard their own code path symmetrically.
50
+ *
51
+ * **Excludes** OpenRouter-routed Anthropic models
52
+ * (`openrouter/anthropic/*`) — OpenRouter goes through the OpenAI SDK
53
+ * with a different base URL, so the request never hits Anthropic's
54
+ * native API and the native computer-use tool block can't be sent.
55
+ */
56
+ export declare function isAnthropicLikeModel(model: string): boolean;
57
+ //# sourceMappingURL=errors.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/computer-use/errors.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,EAAG,gCAAgC,CAAS;IACzD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;gBAEV,KAAK,EAAE,MAAM;CAQ1B;AAED;;;;;;;;GAQG;AACH,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,QAAQ,CAAC,IAAI,EAAG,6BAA6B,CAAS;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;gBAEf,UAAU,EAAE,MAAM;CAQ/B;AAID;;;;;;;;;;;;;GAaG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAM3D"}
@@ -0,0 +1,76 @@
1
+ /**
2
+ * Error classes for `@rudderjs/ai/computer-use` (#A7 Phase 2).
3
+ *
4
+ * Both extend native `Error` so apps can `instanceof`-check from
5
+ * `try/catch`, error middleware, observers, etc.
6
+ */
7
+ /**
8
+ * Thrown by {@link computerUseTool} at construction time when the
9
+ * caller supplies a `model` that isn't an Anthropic-family model
10
+ * (matches `anthropic/*` or `bedrock/<region.>?anthropic.*`).
11
+ *
12
+ * Computer-use is Anthropic-only in v1 — see plan
13
+ * `docs/plans/2026-05-10-ai-computer-use.md`. Other providers either
14
+ * lack native computer-use entirely (Google, Cohere, …) or only have
15
+ * preview-quality versions that aren't worth shipping yet (OpenAI's
16
+ * `computer_use_preview`).
17
+ *
18
+ * Apps that wire `computerUseTool({ page, model: this.model() })`
19
+ * inside `Agent.tools()` get this error at agent boot — fail loud
20
+ * before the model gets a chance to hallucinate tool calls against a
21
+ * provider that can't execute them.
22
+ */
23
+ export class ComputerUseProviderError extends Error {
24
+ code = 'COMPUTER_USE_PROVIDER_MISMATCH';
25
+ model;
26
+ constructor(model) {
27
+ super(`[RudderJS AI] computerUseTool is Anthropic-only in v1; got model "${model}". ` +
28
+ `Use an "anthropic/*" or "bedrock/<region.>?anthropic.*" model, or remove the tool.`);
29
+ this.name = 'ComputerUseProviderError';
30
+ this.model = model;
31
+ }
32
+ }
33
+ /**
34
+ * Thrown by {@link computerUseTool}'s execute when the per-run action
35
+ * counter exceeds {@link ComputerUseToolOptions.maxActions}.
36
+ *
37
+ * Bounds runaway agent loops (e.g. a model that keeps clicking the
38
+ * same broken button forever). Default cap is 50 — most real
39
+ * computer-use tasks finish well under that. Override via
40
+ * `computerUseTool({ page, maxActions: 100 })`.
41
+ */
42
+ export class ComputerUseLimitError extends Error {
43
+ code = 'COMPUTER_USE_LIMIT_EXCEEDED';
44
+ maxActions;
45
+ constructor(maxActions) {
46
+ super(`[RudderJS AI] computerUseTool exceeded maxActions cap of ${maxActions}. ` +
47
+ `Bump the cap with computerUseTool({ page, maxActions: <n> }) if your agent legitimately needs more steps.`);
48
+ this.name = 'ComputerUseLimitError';
49
+ this.maxActions = maxActions;
50
+ }
51
+ }
52
+ // ─── Model classification ─────────────────────────────────
53
+ /**
54
+ * Returns true when `model` is an Anthropic-family model id —
55
+ * `anthropic/*` or `bedrock/<region.>?anthropic.*` (covers cross-region
56
+ * inference profiles like `us.anthropic.*`, `eu.anthropic.*`,
57
+ * `apac.anthropic.*`).
58
+ *
59
+ * Used by {@link computerUseTool}'s upfront `model` check. Exported so
60
+ * apps can guard their own code path symmetrically.
61
+ *
62
+ * **Excludes** OpenRouter-routed Anthropic models
63
+ * (`openrouter/anthropic/*`) — OpenRouter goes through the OpenAI SDK
64
+ * with a different base URL, so the request never hits Anthropic's
65
+ * native API and the native computer-use tool block can't be sent.
66
+ */
67
+ export function isAnthropicLikeModel(model) {
68
+ if (model.startsWith('anthropic/'))
69
+ return true;
70
+ // Bedrock: bedrock/anthropic.* OR bedrock/<region>.anthropic.*
71
+ // (region prefixes us./eu./apac./...)
72
+ if (/^bedrock\/(?:[a-z]{2,4}\.)?anthropic\./.test(model))
73
+ return true;
74
+ return false;
75
+ }
76
+ //# sourceMappingURL=errors.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/computer-use/errors.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,wBAAyB,SAAQ,KAAK;IACxC,IAAI,GAAG,gCAAyC,CAAA;IAChD,KAAK,CAAQ;IAEtB,YAAY,KAAa;QACvB,KAAK,CACH,qEAAqE,KAAK,KAAK;YAC/E,oFAAoF,CACrF,CAAA;QACD,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAA;QACtC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACpB,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IACrC,IAAI,GAAG,6BAAsC,CAAA;IAC7C,UAAU,CAAQ;IAE3B,YAAY,UAAkB;QAC5B,KAAK,CACH,4DAA4D,UAAU,IAAI;YAC1E,2GAA2G,CAC5G,CAAA;QACD,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAA;QACnC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;IAC9B,CAAC;CACF;AAED,6DAA6D;AAE7D;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,IAAI,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAA;IAC/C,+DAA+D;IAC/D,sCAAsC;IACtC,IAAI,wCAAwC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACrE,OAAO,KAAK,CAAA;AACd,CAAC"}
@@ -0,0 +1,53 @@
1
+ /**
2
+ * `@rudderjs/ai/computer-use` — Anthropic computer-use abstraction (#A7).
3
+ *
4
+ * Phase 1 (this entry) ships the action vocabulary + Playwright
5
+ * executor. Phase 2 will add `computerUseTool({ page })` — the agent
6
+ * tool factory that maps to Anthropic's native `computer_20250124`
7
+ * tool block and routes execution through the executor here.
8
+ *
9
+ * # Quick example (manual / phase-1 surface)
10
+ *
11
+ * ```ts
12
+ * import { chromium } from 'playwright'
13
+ * import { executeComputerAction, makeExecutorState } from '@rudderjs/ai/computer-use'
14
+ *
15
+ * const browser = await chromium.launch()
16
+ * const page = await browser.newPage()
17
+ * await page.setViewportSize({ width: 1280, height: 800 })
18
+ * await page.goto('https://example.com')
19
+ *
20
+ * const state = makeExecutorState()
21
+ * const screen = await executeComputerAction(page, { action: 'screenshot' }, state)
22
+ * if (screen.type === 'image') {
23
+ * // screen.data is a PNG Uint8Array
24
+ * }
25
+ *
26
+ * await executeComputerAction(page, { action: 'left_click', coordinate: [400, 200] }, state)
27
+ * ```
28
+ *
29
+ * Phase 2 will collapse this to:
30
+ *
31
+ * ```ts
32
+ * import { computerUseTool } from '@rudderjs/ai/computer-use'
33
+ *
34
+ * class BrowserAgent extends Agent {
35
+ * model = 'anthropic/claude-opus-4-7'
36
+ * tools() { return [computerUseTool({ page })] }
37
+ * }
38
+ * ```
39
+ *
40
+ * # Anthropic-only in v1
41
+ *
42
+ * The action vocabulary mirrors Anthropic's `computer_20250124` schema
43
+ * verbatim. Phase 2's tool factory throws `ComputerUseProviderError` at
44
+ * agent boot for non-Anthropic models — see plan
45
+ * `docs/plans/2026-05-10-ai-computer-use.md`.
46
+ */
47
+ export type { ComputerAction, ComputerActionResult, ComputerExecutorState, Coordinate, PageLike, PageMouseLike, PageKeyboardLike, } from './actions.js';
48
+ export { makeExecutorState } from './actions.js';
49
+ export { executeComputerAction, parseModifiers, normalizeKey, normalizeChord, SCROLL_PIXELS_PER_CLICK, MOUSE_MOVE_STEPS, } from './playwright.js';
50
+ export type { ComputerUseTool, ComputerUseToolOptions, } from './tool.js';
51
+ export { computerUseTool, isComputerUseTool, COMPUTER_USE_MARKER, COMPUTER_USE_TOOL_NAME, } from './tool.js';
52
+ export { ComputerUseLimitError, ComputerUseProviderError, isAnthropicLikeModel, } from './errors.js';
53
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/computer-use/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAEH,YAAY,EACV,cAAc,EACd,oBAAoB,EACpB,qBAAqB,EACrB,UAAU,EACV,QAAQ,EACR,aAAa,EACb,gBAAgB,GACjB,MAAM,cAAc,CAAA;AAErB,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAEhD,OAAO,EACL,qBAAqB,EACrB,cAAc,EACd,YAAY,EACZ,cAAc,EACd,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,iBAAiB,CAAA;AAIxB,YAAY,EACV,eAAe,EACf,sBAAsB,GACvB,MAAM,WAAW,CAAA;AAElB,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,mBAAmB,EACnB,sBAAsB,GACvB,MAAM,WAAW,CAAA;AAElB,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,GACrB,MAAM,aAAa,CAAA"}
@@ -0,0 +1,51 @@
1
+ /**
2
+ * `@rudderjs/ai/computer-use` — Anthropic computer-use abstraction (#A7).
3
+ *
4
+ * Phase 1 (this entry) ships the action vocabulary + Playwright
5
+ * executor. Phase 2 will add `computerUseTool({ page })` — the agent
6
+ * tool factory that maps to Anthropic's native `computer_20250124`
7
+ * tool block and routes execution through the executor here.
8
+ *
9
+ * # Quick example (manual / phase-1 surface)
10
+ *
11
+ * ```ts
12
+ * import { chromium } from 'playwright'
13
+ * import { executeComputerAction, makeExecutorState } from '@rudderjs/ai/computer-use'
14
+ *
15
+ * const browser = await chromium.launch()
16
+ * const page = await browser.newPage()
17
+ * await page.setViewportSize({ width: 1280, height: 800 })
18
+ * await page.goto('https://example.com')
19
+ *
20
+ * const state = makeExecutorState()
21
+ * const screen = await executeComputerAction(page, { action: 'screenshot' }, state)
22
+ * if (screen.type === 'image') {
23
+ * // screen.data is a PNG Uint8Array
24
+ * }
25
+ *
26
+ * await executeComputerAction(page, { action: 'left_click', coordinate: [400, 200] }, state)
27
+ * ```
28
+ *
29
+ * Phase 2 will collapse this to:
30
+ *
31
+ * ```ts
32
+ * import { computerUseTool } from '@rudderjs/ai/computer-use'
33
+ *
34
+ * class BrowserAgent extends Agent {
35
+ * model = 'anthropic/claude-opus-4-7'
36
+ * tools() { return [computerUseTool({ page })] }
37
+ * }
38
+ * ```
39
+ *
40
+ * # Anthropic-only in v1
41
+ *
42
+ * The action vocabulary mirrors Anthropic's `computer_20250124` schema
43
+ * verbatim. Phase 2's tool factory throws `ComputerUseProviderError` at
44
+ * agent boot for non-Anthropic models — see plan
45
+ * `docs/plans/2026-05-10-ai-computer-use.md`.
46
+ */
47
+ export { makeExecutorState } from './actions.js';
48
+ export { executeComputerAction, parseModifiers, normalizeKey, normalizeChord, SCROLL_PIXELS_PER_CLICK, MOUSE_MOVE_STEPS, } from './playwright.js';
49
+ export { computerUseTool, isComputerUseTool, COMPUTER_USE_MARKER, COMPUTER_USE_TOOL_NAME, } from './tool.js';
50
+ export { ComputerUseLimitError, ComputerUseProviderError, isAnthropicLikeModel, } from './errors.js';
51
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/computer-use/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAYH,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAEhD,OAAO,EACL,qBAAqB,EACrB,cAAc,EACd,YAAY,EACZ,cAAc,EACd,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,iBAAiB,CAAA;AASxB,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,mBAAmB,EACnB,sBAAsB,GACvB,MAAM,WAAW,CAAA;AAElB,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,GACrB,MAAM,aAAa,CAAA"}
@@ -0,0 +1,76 @@
1
+ /**
2
+ * Playwright executor for {@link ComputerAction} (#A7 Phase 1).
3
+ *
4
+ * Translates Anthropic's `computer_20250124` action vocabulary into
5
+ * Playwright `Page` calls. Stateless apart from {@link ComputerExecutorState},
6
+ * which the caller threads through so `cursor_position` can answer.
7
+ *
8
+ * # Errors
9
+ *
10
+ * Playwright throws on failures (selector misses, timeouts, navigation
11
+ * during action). The executor catches and returns
12
+ * `{ type: 'error', text: <message> }` so the agent loop can hand the
13
+ * failure back to the model as a tool-result with `is_error: true`. The
14
+ * model decides whether to retry, recover, or give up — failing the
15
+ * whole agent run on a single missed click is too brittle.
16
+ *
17
+ * # Modifier keys & key chords
18
+ *
19
+ * Anthropic uses xdotool naming (`ctrl`, `cmd`, `Return`); Playwright
20
+ * uses its own (`Control`, `Meta`, `Enter`). {@link normalizeKey} maps
21
+ * the common ones; unmapped keys pass through verbatim and Playwright
22
+ * either accepts them (single chars, function keys) or throws — which
23
+ * surfaces as the standard error path above.
24
+ *
25
+ * # Scroll units
26
+ *
27
+ * Anthropic's `scroll_amount` is in mouse-wheel "clicks". A typical
28
+ * desktop wheel click is ~100px; the executor multiplies through
29
+ * {@link SCROLL_PIXELS_PER_CLICK}. Tunable later if customer feedback
30
+ * shows drift.
31
+ */
32
+ import type { ComputerAction, ComputerActionResult, ComputerExecutorState, PageLike } from './actions.js';
33
+ /**
34
+ * Pixels per `scroll_amount` unit. Roughly matches a desktop mouse
35
+ * wheel click on most platforms.
36
+ */
37
+ export declare const SCROLL_PIXELS_PER_CLICK = 100;
38
+ /**
39
+ * Number of intermediate move steps Playwright interpolates when the
40
+ * executor moves the mouse to a target. Higher = smoother (more hover
41
+ * events fire), but slower. 5 is a reasonable default — instant enough
42
+ * for most pages, slow enough that hover-driven UI (tooltips, menus)
43
+ * has a chance to react.
44
+ */
45
+ export declare const MOUSE_MOVE_STEPS = 5;
46
+ /**
47
+ * Dispatch a single {@link ComputerAction} against a Playwright `Page`.
48
+ *
49
+ * Updates `state.cursor` after every action that targets a coordinate
50
+ * (move / click / drag / scroll). Caller owns `state` and threads it
51
+ * through every call within an agent run.
52
+ *
53
+ * Returns a {@link ComputerActionResult} suitable for forwarding to the
54
+ * model as a tool-result. Never throws — Playwright failures surface as
55
+ * `{ type: 'error', text }`.
56
+ */
57
+ export declare function executeComputerAction(page: PageLike, action: ComputerAction, state: ComputerExecutorState): Promise<ComputerActionResult>;
58
+ /**
59
+ * Parse Anthropic's `+`-separated modifier text into Playwright key
60
+ * names. Empty / undefined input → empty array (no modifiers).
61
+ */
62
+ export declare function parseModifiers(text: string | undefined): string[];
63
+ /**
64
+ * Map Anthropic / xdotool key names to Playwright key names.
65
+ * Unmapped keys (single chars, F-keys, anything we haven't covered)
66
+ * pass through verbatim — Playwright either accepts them or throws,
67
+ * which is the right failure mode.
68
+ */
69
+ export declare function normalizeKey(key: string): string;
70
+ /**
71
+ * Normalize a key chord (`+`-separated). Playwright's
72
+ * `keyboard.press()` parses chords natively, so we just normalize each
73
+ * segment and rejoin.
74
+ */
75
+ export declare function normalizeChord(chord: string): string;
76
+ //# sourceMappingURL=playwright.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"playwright.d.ts","sourceRoot":"","sources":["../../src/computer-use/playwright.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,KAAK,EACV,cAAc,EACd,oBAAoB,EACpB,qBAAqB,EACrB,QAAQ,EACT,MAAM,cAAc,CAAA;AAErB;;;GAGG;AACH,eAAO,MAAM,uBAAuB,MAAM,CAAA;AAE1C;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,IAAI,CAAA;AAEjC;;;;;;;;;;GAUG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,qBAAqB,GAC3B,OAAO,CAAC,oBAAoB,CAAC,CA4I/B;AAQD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAMjE;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CA8BhD;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKpD"}