@wenathlan/saddle 1.8.16 → 1.8.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. package/README.md +38 -13
  2. package/dist/apps/registry.d.ts +1 -1
  3. package/dist/index.d.ts +1 -0
  4. package/dist/index.d.ts.map +1 -1
  5. package/dist/index.js +1 -0
  6. package/dist/index.js.map +1 -1
  7. package/dist/isolation/contracts.d.ts +118 -0
  8. package/dist/isolation/contracts.d.ts.map +1 -0
  9. package/dist/isolation/contracts.js +84 -0
  10. package/dist/isolation/contracts.js.map +1 -0
  11. package/dist/mcp/server.d.ts +2 -2
  12. package/dist/scrape/robots.js +1 -1
  13. package/docs/artifactavailability.md +18 -2
  14. package/docs/containerplatforms-1.8.17.md +29 -0
  15. package/docs/releasenotes-1.8.16.md +6 -2
  16. package/docs/releasenotes-1.8.17.md +43 -0
  17. package/docs/releasenotes-1.8.18.md +32 -0
  18. package/docs/research-1.8.18-isolation.md +43 -0
  19. package/docs/saddle.archive.1.8.17.tar.gz.gpg +0 -0
  20. package/docs/todo-1.8.16.md +92 -8
  21. package/docs/todo-1.8.18.md +152 -0
  22. package/docs/workflowoperations.md +32 -0
  23. package/extension/manifest.json +1 -1
  24. package/package.json +3 -2
  25. package/docs/logs/.gitkeep +0 -0
  26. package/docs/plans/00.index.md +0 -50
  27. package/docs/plans/01.architecture.md +0 -86
  28. package/docs/plans/02.research.computer.use.md +0 -58
  29. package/docs/plans/03.research.captcha.bypass.md +0 -68
  30. package/docs/plans/04.research.sandbox.ai.md +0 -52
  31. package/docs/plans/05.capture.platform.md +0 -57
  32. package/docs/plans/06.dependencies.md +0 -97
  33. package/docs/plans/07.captcha.test.page.md +0 -41
  34. package/docs/plans/08.production.infra.md +0 -70
  35. package/docs/plans/09.database.schema.md +0 -121
  36. package/docs/plans/10.cloudinary.storage.md +0 -57
  37. package/docs/plans/11.movement.logs.json.md +0 -72
  38. package/docs/plans/12.research.atlas.agent.browser.md +0 -79
  39. package/docs/plans/13.research.anti.detection.md +0 -898
  40. package/docs/plans/14.research.proxy.md +0 -1495
  41. package/docs/plans/15.research.retry.rate.limit.md +0 -1958
  42. package/docs/plans/16.research.crawling.md +0 -1417
  43. package/docs/plans/17.research.caching.md +0 -1610
  44. package/docs/plans/18.research.content.extraction.md +0 -1952
  45. package/docs/plans/19.research.errors.events.md +0 -1523
  46. package/docs/plans/20.research.zod.validation.md +0 -1350
  47. package/docs/plans/21.research.batch.concurrency.md +0 -1888
  48. package/docs/plans/22.research.universal.runtime.md +0 -944
  49. package/docs/plans/23.research.ai.integration.md +0 -1465
  50. package/docs/plans/24.research.memory.persistence.md +0 -1979
  51. package/docs/plans/25.research.server.api.md +0 -342
  52. package/docs/plans/26.research.compilation.md +0 -249
  53. package/docs/plans/27.research.html.parsing.md +0 -251
  54. package/docs/plans/28.action.plan.md +0 -50
  55. package/docs/plans/29.api.reference.md +0 -174
  56. package/docs/plans/30.architecture.plan.md +0 -94
  57. package/docs/plans/31.auditoria.dados.md +0 -163
  58. package/docs/plans/32.bots.automacao.computacional.md +0 -214
  59. package/docs/plans/33.bots.codigo.revisao.md +0 -220
  60. package/docs/plans/34.bots.seguranca.cicd.md +0 -366
  61. package/docs/plans/35.comparativo.concorrencia.md +0 -464
  62. package/docs/plans/36.computational.memory.md +0 -340
  63. package/docs/plans/37.deploystrategy.md +0 -394
  64. package/docs/plans/38.flow.md +0 -155
  65. package/docs/plans/39.multi.platform.bot.md +0 -252
  66. package/docs/plans/40.npm.publish.md +0 -250
  67. package/docs/plans/41.o.que.falta.md +0 -407
  68. package/docs/plans/42.pesquisa.concorrencia.md +0 -721
  69. package/docs/plans/43.plan.universal.architecture.md +0 -496
  70. package/docs/plans/44.reference.md +0 -100
  71. package/docs/plans/45.robotarchitecture.md +0 -237
  72. package/docs/plans/46.scdnintegration.md +0 -284
  73. package/docs/plans/47.multiforge.readme.md +0 -129
  74. package/docs/plans/48.theory.v4.repo.os.md +0 -152
  75. package/docs/plans/49.third.party.infra.md +0 -12
  76. package/docs/plans/50.file.as.compute.md +0 -39
  77. package/docs/plans/51.architecture.virtual.processor.md +0 -80
  78. package/docs/plans/52.manifesto.v8.md +0 -11
  79. package/docs/plans/58.cdn.list.md +0 -23
  80. package/docs/plans/59.sql.frameworks.md +0 -33
  81. package/docs/plans/60.sql.thirdparty.md +0 -26
  82. package/docs/plans/61.objective.multiforge.md +0 -63
  83. package/docs/plans/62.huggingface.upload.md +0 -26
  84. package/docs/plans/63.kaggle.upload.md +0 -24
  85. package/docs/plans/64.npm.storage.md +0 -30
  86. package/docs/plans/65.rclone.terabox.md +0 -32
  87. package/docs/plans/66.buckets.and.models.todo.md +0 -14
  88. package/docs/plans/67.database.todo.md +0 -13
  89. package/docs/plans/68.deploy.packages.todo.md +0 -12
  90. package/docs/plans/69.report.human.operator.md +0 -133
  91. package/docs/plans/70.report.brain2qwerty.ems.md +0 -135
  92. package/docs/plans/71.report.hd.infinito.vram.md +0 -155
  93. package/docs/plans/72.plan.hd.infinito.node.md +0 -146
  94. package/docs/plans/73.plan.scifi.repos.md +0 -125
  95. package/docs/plans/74.000.manifesto.v8.flat.2..md +0 -11
  96. package/docs/plans/README.md +0 -489
  97. package/docs/plans/aggregate_platforms.mjs +0 -146
  98. package/docs/plans/examplesession.json +0 -36
  99. package/docs/plans/missing-facts.md +0 -192
  100. package/docs/plans/models.md +0 -64
  101. package/docs/plans/organize.cjs +0 -270
  102. package/docs/plans/platforms.md +0 -2887
  103. package/docs/plans/sites.md +0 -31322
  104. package/docs/sources/farm.py +0 -117
  105. package/docs/sources/html/saddle1.html +0 -132
  106. package/docs/sources/html/saddle2.html +0 -157
  107. package/docs/sources/html/saddle3.html +0 -119
  108. package/docs/sources/html/saddle4.html +0 -144
  109. package/docs/sources/html/saddle5.html +0 -72
  110. package/docs/sources/html/saddle6.html +0 -171
  111. package/docs/sources/html/saddle7.html +0 -236
  112. package/docs/sources/saddle.ts +0 -74
  113. package/docs/sources/schema.prisma +0 -88
  114. package/docs/sources/script.sh +0 -64
  115. package/docs/sources/workflows.yml +0 -458
  116. package/docs/talks1/_body.txt +0 -14
  117. package/docs/talks1/_index.md +0 -15
  118. package/docs/talks1/_screenshot.png +0 -0
  119. package/docs/talks1/assistant-01.md +0 -5
  120. package/docs/talks1/assistant-02.md +0 -5
  121. package/docs/talks1/assistant-03.md +0 -531
  122. package/docs/talks1/assistant-04.md +0 -26
  123. package/docs/talks1/assistant-05.md +0 -774
  124. package/docs/talks1/assistant-06.md +0 -1718
  125. package/docs/talks1/scrape-share.cjs +0 -185
  126. package/docs/talks1/scrape-share.ts +0 -183
  127. package/docs/talks1/user-01.md +0 -3
  128. package/docs/talks1/user-02.md +0 -3
  129. package/docs/talks1/user-03.md +0 -88
  130. package/docs/talks1/user-04.md +0 -3
  131. package/docs/talks1/user-05.md +0 -3
  132. package/docs/talks1/user-06.md +0 -88
  133. package/docs/talks1/user-07.md +0 -88
  134. package/docs/talks2/_body.txt +0 -14
  135. package/docs/talks2/_index.md +0 -16
  136. package/docs/talks2/_screenshot.png +0 -0
  137. package/docs/talks2/assistant-01.md +0 -5
  138. package/docs/talks2/assistant-02.md +0 -5
  139. package/docs/talks2/assistant-03.md +0 -424
  140. package/docs/talks2/assistant-04.md +0 -598
  141. package/docs/talks2/assistant-05.md +0 -1280
  142. package/docs/talks2/assistant-06.md +0 -1227
  143. package/docs/talks2/assistant-07.md +0 -1252
  144. package/docs/talks2/user-01.md +0 -3
  145. package/docs/talks2/user-02.md +0 -3
  146. package/docs/talks2/user-03.md +0 -88
  147. package/docs/talks2/user-04.md +0 -88
  148. package/docs/talks2/user-05.md +0 -88
  149. package/docs/talks2/user-06.md +0 -88
  150. package/docs/talks2/user-07.md +0 -3
  151. package/docs/talks3/_body.txt +0 -467
  152. package/docs/talks3/_index.md +0 -10
  153. package/docs/talks3/_screenshot.png +0 -0
  154. package/docs/talks3/assistant-01.md +0 -417
  155. package/docs/talks3/assistant-02.md +0 -417
  156. package/docs/talks3/assistant-03.md +0 -29
  157. package/docs/talks3/assistant-04.md +0 -727
  158. package/docs/talks3/user-01.md +0 -88
  159. package/docs/talks3/user-02.md +0 -88
  160. package/docs/talks3/user-03.md +0 -3
  161. package/docs/talks3/user-04.md +0 -3
  162. package/docs/talks4/_body.txt +0 -14
  163. package/docs/talks4/_index.md +0 -12
  164. package/docs/talks4/_screenshot.png +0 -0
  165. package/docs/talks4/assistant-01.md +0 -5
  166. package/docs/talks4/assistant-02.md +0 -5
  167. package/docs/talks4/assistant-03.md +0 -35
  168. package/docs/talks4/assistant-04.md +0 -512
  169. package/docs/talks4/assistant-05.md +0 -599
  170. package/docs/talks4/user-01.md +0 -3
  171. package/docs/talks4/user-02.md +0 -3
  172. package/docs/talks4/user-03.md +0 -88
  173. package/docs/talks4/user-04.md +0 -88
  174. package/docs/talks4/user-05.md +0 -7
  175. package/docs/talks5/_body.txt +0 -14
  176. package/docs/talks5/_index.md +0 -13
  177. package/docs/talks5/_screenshot.png +0 -0
  178. package/docs/talks5/assistant-01.md +0 -5
  179. package/docs/talks5/assistant-02.md +0 -5
  180. package/docs/talks5/assistant-03.md +0 -690
  181. package/docs/talks5/assistant-04.md +0 -758
  182. package/docs/talks5/assistant-05.md +0 -974
  183. package/docs/talks5/user-01.md +0 -3
  184. package/docs/talks5/user-02.md +0 -3
  185. package/docs/talks5/user-03.md +0 -105
  186. package/docs/talks5/user-04.md +0 -105
  187. package/docs/talks5/user-05.md +0 -63
  188. package/docs/talks5/user-06.md +0 -105
  189. package/docs/talks6/_body.txt +0 -14
  190. package/docs/talks6/_index.md +0 -9
  191. package/docs/talks6/_screenshot.png +0 -0
  192. package/docs/talks6/assistant-01.md +0 -5
  193. package/docs/talks6/assistant-02.md +0 -5
  194. package/docs/talks6/assistant-03.md +0 -1499
  195. package/docs/talks6/user-01.md +0 -3
  196. package/docs/talks6/user-02.md +0 -3
  197. package/docs/talks6/user-03.md +0 -88
  198. package/docs/talks6/user-04.md +0 -88
  199. package/docs/talks7/_body.txt +0 -14
  200. package/docs/talks7/_index.md +0 -10
  201. package/docs/talks7/_screenshot.png +0 -0
  202. package/docs/talks7/assistant-01.md +0 -5
  203. package/docs/talks7/assistant-02.md +0 -5
  204. package/docs/talks7/assistant-03.md +0 -523
  205. package/docs/talks7/assistant-04.md +0 -617
  206. package/docs/talks7/user-01.md +0 -3
  207. package/docs/talks7/user-02.md +0 -3
  208. package/docs/talks7/user-03.md +0 -105
  209. package/docs/talks7/user-04.md +0 -67
  210. package/docs/talks8/conversa1.txt +0 -1322
  211. package/docs/talks8/conversa2.txt +0 -237
  212. package/docs/talks9/Beyond the Obvious_ 50 Plataformas Auto-Hospedadas de Forja de C/303/263digo para Al/303/251m de Gitea e GitLab.md" +0 -174
  213. package/docs/talks9/De NPM a Multi-Linguagem_ Uma Arquitetura T/303/251cnica para a Execu/303/247/303/243o Integrada de C/303/263digo no Ecossistema Node.js.md" +0 -59
  214. package/docs/talks9/De NPM a VMs Virtuais_ Uma An/303/241lise Arquitet/303/264nica para a Realiza/303/247/303/243o do Ciclo de Vida do Projeto SADDLE.md" +0 -91
  215. package/docs/talks9/Mapeamento da Engrenagem Computacional_ Uma Arquitetura para Execu/303/247/303/243o Isolada e Persist/303/252ncia em Ambientes Distribu/303/255dos.md" +0 -116
  216. package/docs/talks9/O Cen/303/241rio Pr/303/241tico do SADDLE_ Uma An/303/241lise de Viabilidade e Modelo de Ciclo de Vida Integrado.md" +0 -128
  217. package/docs/talks9/README (2).md +0 -489
  218. package/docs/talks9/README.md +0 -198
  219. package/docs/talks9/Viabilidade do Saddle_ Uma An/303/241lise T/303/251cnica da Transforma/303/247/303/243o de Armazenamento Remoto em Mem/303/263ria Computacional.md" +0 -80
  220. package/docs/talks9/conversa.txt +0 -544
  221. package/docs/talks9/other (2).md +0 -39
  222. package/docs/talks9/other.md +0 -57
  223. package/docs/talks9/outro.txt +0 -24
@@ -1,58 +0,0 @@
1
- # 02 — Pesquisa: Computer Use Agents (2026)
2
-
3
- ## Estado da arte
4
-
5
- Em 2026, *computer-use agents* deixaram de ser protótipos e entraram em produção para
6
- tarefas restritas e supervisionadas. Principais provedores e frameworks:
7
-
8
- - **Anthropic Computer Use** (`computer_use_20251124`): ferramentas `computer` (mouse/teclado),
9
- `text_editor`, `bash`. Recomenda rodar em **VMs/containers com privilégios mínimos**.
10
- - **OpenAI Operator / CUA**: 87% de sucesso em sites JS complexos; 58% no WebArena.
11
- - **Google Project Mariner**: 83.5% WebVoyager, 84% ScreenSpot.
12
- - **Microsoft UFO²**: funde Windows UI Automation + OmniParser (visão).
13
- - **browser-use** (open source): 89.1% WebVoyager (abordagem híbrida DOM + visão).
14
-
15
- ### Benchmarks
16
- - **OSWorld:** humano 72.36% vs SOTA ~12–20% (desktop ainda imaturo).
17
- - **WebArena:** tops ~71%, produção na faixa 50–60%.
18
- - **WebVoyager:** browser-use 89.1% (híbrido).
19
-
20
- ### Conclusão de arquitetura
21
- Ganham abordagens **híbridas**:
22
- 1. Raciocínio por **DOM / accessibility tree** para elementos estruturados.
23
- 2. **Visão** (VLM) para layouts não-padrão, canvas, imagens.
24
- 3. **Scripts determinísticos** para validação e replay.
25
- 4. **Verificação** (screenshot antes/depois, diff de elementos).
26
-
27
- ## Projetos de referência encontrados
28
-
29
- | Projeto | Diferencial relevante para UKA |
30
- |---------|-------------------------------|
31
- | [agentbrowser](https://github.com/AshtonVaughan/agentbrowser) | Cursor visível humano via CDP, trilha SVG, trajetórias Bézier com jitter/overshoot, **traces JSONL** em `~/.agentbrowser/traces/`, replay determinístico por seed, bypass de Cloudflare/OneTrust |
32
- | [humanjs](https://github.com/totigm/humanjs) | Playwright humanizado: curvas Bézier, ritmo de digitação, scroll, **personalidades** (careful/fast/distracted/precise), gravador → JSON timeline, MCP server |
33
- | [human-browser-use](https://github.com/andyless/human-browser-use) | Extensão p/ browser-use: trajetória Bézier, typing lognormal, scroll inercial, stealth fingerprint (`navigator.webdriver`, WebGL, canvas) |
34
- | [agentic-stealth-browser](https://github.com/shanewas/agentic-stealth-browser) | TLS spoofing (JA3/JA4), fingerprint, recovery chain (detecta captcha → rotaciona proxy), account warming |
35
- | [mantis](https://github.com/mercurialsolo/mantis) | Agent perception-reasoning-action em Xvfb+Chrome+xdotool (sem fingerprint Playwright), vídeo screencast, deploy em Baseten/Modal/EKS/GKE |
36
- | [win-pilot-mcp](https://github.com/omidmanoochehri/win-pilot-mcp) | MCP Windows computer-use: screen capture, OCR, mouse humano, logging estruturado, replay de macros |
37
- | [cloakbrowser-human](https://github.com/evelaa123/cloakbrowser-human) | Wrapper Playwright que adiciona camada **comportamental** (mouse Bézier, typing, scroll) — complementa stealth de fingerprint |
38
-
39
- ## Padrões de movimento humano (extraídos da pesquisa)
40
-
41
- - **Trajetória do mouse:** curva Bézier única, velocidade variável por arc-length:
42
- - 0–5%: ramp-up (0.3x → 2.5x)
43
- - 5–75%: cruise (2.3–2.5x) com variação senoidal
44
- - 75–100%: ease-out cúbico (2.5x → 0.3x)
45
- - sub-pixel drift final (σ 0.3–1.5px)
46
- - duração via **Lei de Fitts**: `0.05 + 0.07*log2(1 + distance/20)` s
47
- - **Overshoot:** ~15% de chance, auto-aumenta em movimentos longos.
48
- - **Typing:** inter-key delay lognormal (μ=4.17, σ=0.3 → ~65ms), bigram speedup, 2% typo.
49
- - **Scroll:** wheel events com acel/decel, overshoot, settle delay.
50
- - **Stealth:** `navigator.webdriver → undefined`, spoof de plugins/WebGL/canvas, flags de launch.
51
-
52
- ## Aplicação no UKA
53
-
54
- - Implementar `HumanCursor` (SVG overlay + CDP `Input.dispatchMouseEvent`) inspirado no
55
- agentbrowser/humanjs, com trajetórias Bézier + jitter + overshoot determinísticas por seed.
56
- - Gravar **todos os eventos** como `SessionEvent` tipado → exportar JSON para `docs/logs/`.
57
- - Usar replay determinístico (seed) para testes reprodutíveis.
58
- - Tratar GUI como imagem (visão) + DOM/accessibility para alvos estruturados.
@@ -1,68 +0,0 @@
1
- # 03 — Pesquisa: hCaptcha / Cloudflare / reCAPTCHA Bypass
2
-
3
- ## Contexto
4
-
5
- O UKA precisa **testar** a capacidade do computer-use de enfrentar captchas fortes. A
6
- pesquisa abaixo levanta abordagens open-source e comerciais (uso edUKAcional / automação
7
- supervisionada).
8
-
9
- ## hCaptcha
10
-
11
- ### hcaptcha-challenger (QIN2DIM) — referência principal
12
- - Resolve hCaptcha com **LLM multimodal** (sem Tampermonkey, sem serviço anti-captcha 3º).
13
- - Modelos plugáveis (ONNX):
14
- - `image_label_binary` → ResNet ✅
15
- - `image_label_area_select: point` → YOLOv8 ✅
16
- - `image_label_area_select: bounding box` → YOLOv8 segmentation
17
- - `image_drag_drop` → Spatial Chain-of-Thought ✅
18
- - `image_label_multiple_choice` → ViT zero-shot
19
- - Pipeline: `install()` → `AgentT.from_page()` → `handle_checkbox()` → `execute()` loop.
20
- - Deps: `playwright`, `httpx`, `opencv-python`, `pillow`, `loguru`, `pydantic-settings`.
21
-
22
- ### Outros solvers
23
- - **hCaptcha-Solver (Camoufox + Multibot):** usa Camoufox p/ HSW token + Groq vision +
24
- **geração de movimento de mouse realista** (Multibot API). Lida com `image_label_binary`,
25
- `area_select`, `drag_drop`. Requer proxy.
26
- - **maximedrn/hcaptcha-solver-python-selenium:** userscript Tampermonkey + Selenium (edUKAcional).
27
- - **auto-captcha-solver / auto-captcha (NopeCHA):** detecta hCaptcha, reCAPTCHA v2/v3,
28
- Cloudflare Turnstile via **API de token NopeCHA**; injeta token automaticamente. Tem MCP
29
- server e CLI. Tipos: hCaptcha ✅, reCAPTCHA v2/v3 ✅, Turnstile ⚠️ (lento).
30
-
31
- ## Cloudflare (Turnstile / interstitials)
32
-
33
- - **agentic-stealth-browser** passa Cloudflare interstitial usando TLS spoofing + fingerprint
34
- + comportamento humano + **recovery chain** (detecta captcha → rotaciona proxy/sessão → retry).
35
- - **auto-captcha** resolve Turnstile via NopeCHA (fila lenta, requer proxy em produção).
36
- - **captcha-solver (CloakBrowser + 2Captcha/CapSolver):** Turnstile resolvido em ~5s via
37
- ClickSolver (free) ou fallback pago.
38
-
39
- ## reCAPTCHA
40
-
41
- - v2/v3/Enterprise: cobertos por hcaptcha-challenger (via proxy), auto-captcha (NopeCHA),
42
- 2Captcha, CapSolver.
43
-
44
- ## Capes (tipos suportados por 2Captcha/CapSolver — 30+)
45
-
46
- reCAPTCHA v2/v3/Enterprise, Cloudflare Turnstile, hCaptcha, FunCaptcha (Arkose), GeeTest,
47
- KeyCaptcha, Amazon WAF, DataDome, Akamai, Imperva, Friendly, MTCaptcha, Lemin, Cutcaptcha,
48
- Tencent, Yandex, ALTCHA, Prosopo, entre outros.
49
-
50
- ## Camadas de defesa que o UKA deve considerar
51
-
52
- | Camada | O que ataca | Mitigação no UKA |
53
- |--------|-------------|------------------|
54
- | Fingerprint (WebDriver, CDP, headless, GPU) | detecta automação | stealth JS + Brave real/CDP, spoof WebGL/canvas |
55
- | Comportamental (mouse/keyboard/scroll) | analisa padrões | trajetórias Bézier humanas, typing lognormal |
56
- | TLS (JA3/JA4) | identifica cliente | TLS regional / proxy residencial sticky |
57
- | IP reputation | bloqueia faixas | proxies residenciais com sessão sticky |
58
- | Captcha | desafio final | solver VLM (hcaptcha-challenger) ou token API |
59
-
60
- ## Aplicação no UKA
61
-
62
- - Criar `tests/examples/hcaptcha-test.html` (página de teste local) + usar sites reais fortes
63
- (Cloudflare, Google reCAPTCHA demo, hCaptcha demo) no Brave.
64
- - Pipeline de teste: abrir → detectar captcha → resolver (VLM ou token API) → registrar
65
- token/screenshot/log em `tests/output/`.
66
- - **Não** usar para abuso: foco em automação supervisionada e pesquisa.
67
- - Dependências relevantes: `playwright`, `puppeteer`, `@xenova/transformers`,
68
- `onnxruntime-web`, `tesseract.js`, `cloudinary` (evidências).
@@ -1,52 +0,0 @@
1
- # 04 — Pesquisa: Sandbox Virtual e Containers para IA
2
-
3
- ## Por que sandbox?
4
-
5
- Computer-use agents executam código/inputs não confiáveis (saída de LLM, uploads de
6
- usuário). Rodar em sandbox isola a execução, impede acesso ao sistema de produção e limita
7
- recursos. A pesquisa mostra que provedores ladearam para **microVMs Firecracker**.
8
-
9
- ## Vercel Sandbox (opção principal do UKA)
10
-
11
- - Compute primitive para rodar código não confiável com segurança (AI agents, code gen).
12
- - **Firecracker microVM** por sandbox, filesystem e rede próprios, isolamento forte.
13
- - Runtimes: `node26/24/22`, `python3.13`. Default `node24`.
14
- - Cada sandbox roda como `vercel-sandbox` (sudo disponível), working dir `/vercel/sandbox`.
15
- - Startup em **milissegundos**; suporta Docker/FUSE dentro do microVM (privilégio elevado).
16
- - **Credential brokering:** injeta secrets em requests de saída sem expor dentro do VM
17
- (`console.log(process.env)` não vaza a chave).
18
- - **Network policy:** allowlist de domínios, atualizável em runtime.
19
- - SDK JS e Python; CLI. Multi-agent isolation (usuário Linux por agente, grupos p/ compartilhar).
20
-
21
- ### AI SDK Computer Use (vercel-labs) — template de referência
22
- - Next.js + AI SDK + Claude Sonnet 4.5 + **Vercel Sandbox**.
23
- - Snapshot pré-construído com: **Xvnc**, **openbox**, **noVNC + websockify**, **Google
24
- Chrome**, **xdotool + ImageMagick**.
25
- - Fluxo: User ↔ Chat UI ↔ AI SDK ↔ Claude → Vercel Sandbox (desktop via noVNC iframe).
26
- - Ferramentas: `computer` (screenshot/click/type/scroll) + `bash`.
27
- - Env: `ANTHROPIC_API_KEY`, `SANDBOX_SNAPSHOT_ID`, `VERCEL_OIDC_TOKEN`/`VERCEL_TOKEN`.
28
-
29
- ### vercel-labs/agent-browser (serverless)
30
- - CLI Rust nativo; roda agent-browser + Chrome em Vercel Sandbox microVM efêmero.
31
- - Helper `@agent-browser/sandbox/vercel` + `@vercel/sandbox`.
32
-
33
- ### vercel-labs/sandcastle (PoC desktop)
34
- - Desktop Linux web sobre Vercel Sandbox. Stack: Next.js, Tailwind, **Zustand + SWR**,
35
- **Neon Postgres + Drizzle ORM**, `@vercel/sandbox`, code-server, Xpra. Auth password/OAuth.
36
-
37
- ## Outras opções
38
-
39
- - **Modal / Baseten / EKS / GKE** (usados pelo mantis) — containers Docker customizados.
40
- - **Daytona / Cloudflare** — alternativas de sandbox (citadas mas com menos fit p/ TS).
41
- - **Containers Docker locais** (xdotool + Xvfb + Chromium) para dev sem nuvem.
42
-
43
- ## Decisão para o UKA
44
-
45
- - **Produção:** Vercel Sandbox como camada de execução do computer-use; site de controle
46
- deployado em **Vercel** (ou Netlify) (`web/`).
47
- - **Banco:** criado no deploy (Drizzle/LibSQL ou Prisma em Neon/Turso), conforme `09-esquema-banco-dados.md`.
48
- - **Armazenamento:** Cloudinary para assets/replays (`10-cloudinary-storage.md`).
49
- - **Dev local:** Brave real via Playwright/CDP para a plataforma de captura (`05-plataforma-captura-movimentos.md`).
50
- - **Comunicação:** requests **POST/GET HTTPS** do agente → site deployado → DB + Cloudinary.
51
- - Dependências: `@vercel/sandbox`, `vercel`, `@netlify/blobs`, `@netlify/functions`,
52
- `hono`/`express`, `drizzle-orm`, `@libsql/client`, `socket.io`.
@@ -1,57 +0,0 @@
1
- # 05 Capture Platform
2
-
3
- ## Purpose
4
-
5
- Build a web platform that captures the user/agent movement inside the browser (focus: brave) and exports everything as json logs to `docs/logs/`. This is the base to train and replay the UKA computer use with human behavior.
6
-
7
- ## Core concepts
8
-
9
- - Virtual mouse / virtual arrow: an svg cursor injected in the page (via cdp init script) that follows the real movement and draws a trail and click ripple.
10
- - Event capture: click, double click, drag, scroll, coordinates (x,y), rotation (scroll/wheel angle), timing, target (selector/role/text), deterministic seed.
11
- - Export: compacted json in `docs/logs/<session>.json`.
12
-
13
- ## Platform architecture (root based, no src)
14
-
15
- ```
16
- web/ (capture frontend)
17
- ├─ index.html # capture page
18
- ├─ capture.js # injects svg cursor + listeners
19
- ├─ recorder.js # collects sessionevent to buffer
20
- └─ exporter.js # serializes to docs/logs/<session>.json
21
-
22
- (root logic files, grouped per skill arch)
23
- ├─ humancursor.js # bezier + jitter + overshoot (cdp Input.dispatchMouseEvent)
24
- ├─ trajectory.js # path generator (ghost-cursor / humanjs inspired)
25
- ├─ fingerprint.js # stealth patches webgl/canvas/audio
26
- ├─ recorder.js # json event stream
27
- ├─ replay.js # deterministic replay by seed
28
- └─ runcapture.js # entrypoint (opens brave via cdp)
29
- ```
30
-
31
- ## Sessionevent format
32
-
33
- ```ts
34
- type SessionEvent =
35
- | { t: number; type: "move"; x: number; y: number; tx: number; ty: number }
36
- | { t: number; type: "click"; x: number; y: number; button: "left"|"right"; target?: string }
37
- | { t: number; type: "drag"; x0: number; y0: number; x1: number; y1: number }
38
- | { t: number; type: "scroll"; dx: number; dy: number; angle: number }
39
- | { t: number; type: "key"; key: string; target?: string }
40
- | { t: number; type: string; [k: string]: unknown };
41
- ```
42
-
43
- ## How it works (summary)
44
-
45
- 1. Open brave via playwright/chromium with `chrome.debugger` (cdp).
46
- 2. Inject svg cursor sprite plus event listeners (`Input.dispatchMouseEvent`).
47
- 3. Move the cursor humanly (bezier with jitter/overshoot), all via cdp, not locators.
48
- 4. Each action emits a `SessionEvent` with timestamp, coordinates, target, seed.
49
- 5. Compact trajectories (collapse move runs) and export to `docs/logs/`.
50
-
51
- ## Relation with logs
52
-
53
- The final json format of `docs/logs/` files is detailed in `11-movement-logs-json.md`. An example log is at `docs/logs/examplesession.json`.
54
-
55
- ## Dependencies involved
56
-
57
- `playwright`, `puppeteer`, `chromium-bidi`, `bezier-js`, `@napi-rs/canvas`, `d3`, `socket.io-client`, `zod` (event validation).
@@ -1,97 +0,0 @@
1
- # 06 Dependencies
2
-
3
- Dependency list of the `@wenathlan/UKA` package, grouped by function. Total: 65 packages (including the dev tooling set). The user asked for "50 to 60, or 60+. Counting 10 more", so this set covers browser automation, ai/vision, sandbox, server, db, storage and validation.
4
-
5
- ## Browser automation and mouse (8)
6
- 1. `playwright` — real brave/chromium automation via cdp.
7
- 2. `puppeteer` — fallback automation / stealth.
8
- 3. `chromium-bidi` — bidi protocol for input control.
9
- 4. `@playwright/test` — e2e tests for the platform and captcha.
10
- 5. `bezier-js` — human mouse trajectory generation (bezier).
11
- 6. `@napi-rs/canvas` — draw virtual cursor / trail / ripple.
12
- 7. `d3` — movement trajectory visualization.
13
- 8. `sharp` — screenshot/evidence processing.
14
-
15
- ## Ai / vision / captcha (12)
16
- 9. `ai` — ai sdk core.
17
- 10. `@ai-sdk/anthropic` — claude (computer use).
18
- 11. `@ai-sdk/google` — gemini (vision).
19
- 12. `@ai-sdk/openai` — gpt (vision/reasoning).
20
- 13. `@ai-sdk/react` — agent ui hooks.
21
- 14. `@ai-sdk/vercel` — vercel integration.
22
- 15. `@xenova/transformers` — local vlm models (onnx).
23
- 16. `onnxruntime-web` — onnx inference in browser (yolo/resnet).
24
- 17. `tesseract.js` — ocr for captcha/text.
25
- 18. `openai` — raw sdk for vision/solvers.
26
- 19. `comlink` — workers for inference in a separate thread.
27
- 20. `jsdom` — virtual dom for parsing/tests.
28
-
29
- ## Sandbox and deploy (3)
30
- 21. `@vercel/sandbox` — firecracker microvm for isolated execution.
31
- 22. `vercel` — cli/deploy to vercel.
32
- 23. `nanoid` — short url-safe ids.
33
-
34
- > Note: `@netlify/functions` and `@netlify/blobs` were removed. The architecture skill rejects netlify/vercel functions and platform locked storage. Realtime uses socket. Db uses prisma/drizzle/mysql2.
35
-
36
- ## Server / api (7)
37
- 24. `hono` — light http api (production).
38
- 25. `express` — alternative api / dev.
39
- 26. `cors` — cors for agent requests.
40
- 27. `helmet` — security headers.
41
- 28. `socket.io` — realtime event streaming (socket).
42
- 29. `socket.io-client` — replay/stream client.
43
- 30. `multer` — file upload (tokens, evidence).
44
-
45
- ## Database / orm (5)
46
- 31. `drizzle-orm` — primary orm.
47
- 32. `mysql2` — mysql driver (skill arch: data uses mysql2).
48
- 33. `@prisma/client` — prisma client.
49
- 34. `prisma` — cli/migrations.
50
- 35. `yaml` — schema/env config.
51
-
52
- ## Storage (1)
53
- 36. `cloudinary` — upload replays, screenshots, assets.
54
-
55
- ## Validation / types (3)
56
- 37. `zod` — event/log schema.
57
- 38. `valibot` — light validation.
58
- 39. `arktype` — runtime types.
59
-
60
- ## Logging (1)
61
- 40. `pino` — structured (json) agent logs.
62
-
63
- ## Frontend / build (8)
64
- 41. `react` — platform ui.
65
- 42. `react-dom` — react render.
66
- 43. `next` — production web framework.
67
- 44. `vite` — bundler.
68
- 45. `@vitejs/plugin-react` — react plugin for vite.
69
- 46. `tailwindcss` — styling.
70
- 47. `@tailwindcss/vite` — tailwind+vite integration.
71
- 48. `@testing-library/react` — component tests.
72
- 49. `@testing-library/dom` — dom test utils.
73
- 50. `@testing-library/jest-dom` — test matchers.
74
-
75
- ## Media / util (4)
76
- 51. `pngjs` — png read/write (click masks).
77
- 52. `ffmpeg-static` — replay video recording.
78
- 53. `@t3-oss/env-core` — typed env validation.
79
- 54. `uuid` — session/agent ids.
80
-
81
- ## Dev tooling (11 — the "10+")
82
- 55. `typescript` — compiler.
83
- 56. `@biomejs/biome` — lint/format.
84
- 57. `vitest` — test runner.
85
- 58. `@vitest/coverage-v8` — coverage.
86
- 59. `tsx` — run ts without build.
87
- 60. `@types/node` — node types.
88
- 61. `@types/react` — react types.
89
- 62. `@types/react-dom` — react dom types.
90
- 63. `@ai-sdk/anthropic` (dev) — mirrored in devdeps.
91
- 64. `@vercel/sandbox` (dev) — mirrored in devdeps.
92
- 65. `prisma` (dev) — cli in devdeps.
93
-
94
- ## Conventions
95
- - Files lowercase. No underscore. No hyphen.
96
- - English only. Jsdoc comments. Third person voice.
97
- - No hardcoded host. Use `host`. Ports randomized then locked.
@@ -1,41 +0,0 @@
1
- # 07 Captcha Test Page
2
-
3
- ## Goal
4
-
5
- Have a controlled target to validate the UKA computer use against strong captchas. Open it in brave (via the capture platform) and record the result.
6
-
7
- ## Target options
8
-
9
- ### A. Local page (`tests/examples/hcaptchatest.html`) — recommended to start
10
- - Static html with the hcaptcha widget using the public test sitekey.
11
- - Buttons to trigger the challenge, show the resolved token and status.
12
- - Validates detection + solver + log without heavy external network.
13
-
14
- ### B. Real strong sites — advanced tests
15
- - hcaptcha demo: `https://accounts.hcaptcha.com/demo`
16
- - cloudflare turnstile: `https://challenges.cloudflare.com/turnstile`
17
- - google recaptcha: `https://www.google.com/recaptcha/api2/demo`
18
-
19
- ## Test pipeline (script in `tests/scripts/`)
20
-
21
- 1. `launchbrave()` — opens real brave via playwright/cdp.
22
- 2. `navigate(url)` — goes to the test page.
23
- 3. `detectcaptcha(page)` — scans dom/frames for hcaptcha / turnstile / recaptcha.
24
- 4. `solve(page)` — uses vlm solver (hcaptcha challenger) or token api (nopecha/2captcha).
25
- 5. `record(page)` — saves screenshot, token and movement log to `tests/output/`.
26
- 6. `assert(passed)` — verifies the challenge was passed.
27
-
28
- ## Evidence in `tests/output/`
29
- - `hcaptcha-<ts>.json` — token + metadata.
30
- - `hcaptcha-<ts>.png` — screenshot of challenge and post-solve.
31
- - `hcaptcha-<ts>-trace.json` — movement log (replay).
32
- - `hcaptcha-<ts>.mp4` — replay video (via `ffmpeg-static`).
33
-
34
- ## Success criteria
35
- - Auto detects captcha type.
36
- - Solves hcaptcha (checkbox) with vlm/llm.
37
- - Passes cloudflare turnstile in at least 1 of 3 attempts (with sticky proxy).
38
- - Generates reproducible json log (seed).
39
-
40
- ## Ethical note
41
- For research and supervised automation only. Respect tos and rate limits.
@@ -1,70 +0,0 @@
1
- # 08 Production Infra
2
-
3
- ## Goal
4
-
5
- The `web/` folder is the production site. It is deployed to vercel or netlify and works as the sandbox backend: it receives agent data via https requests (post/get), writes to the db and saves assets to cloudinary.
6
-
7
- ## Why deploy (user decision)
8
- - The database is created in the deploy (vercel/netlify + turso/mysql), not local.
9
- - The agent/sandbox talks to the site via https, no own infra needed.
10
- - Cloudinary holds heavy data (replay videos, screenshots, big logs).
11
-
12
- ## Structure of `web/` (root based, no src)
13
-
14
- ```
15
- web/
16
- ├── index.js # entrypoint (open node server, not platform functions)
17
- ├── server.js # api routes (hono or express)
18
- ├── capture.js # browser movement capture (virtual mouse, arrow)
19
- ├── recorder.js # json event stream
20
- ├── replay.js # deterministic replay by seed
21
- ├── humancursor.js # bezier + jitter + overshoot (cdp)
22
- ├── trajectory.js # path generator
23
- ├── fingerprint.js # stealth patches
24
- ├── db.js # drizzle + mysql2 + prisma loader
25
- ├── storage.js # cloudinary upload
26
- ├── auth.js # api key / oidc (VERCEL_OIDC_TOKEN)
27
- ├── vercel.json # vercel build/deploy config
28
- ├── netlify.toml # netlify deploy config (alt)
29
- ├── drizzle.config.js # drizzle -> mysql2
30
- ├── schema.prisma # prisma schema (alt/sync)
31
- ├── index.html # capture ui
32
- └── hcaptchatest.html # captcha test page (mirror of tests/examples)
33
- ```
34
-
35
- ## Backend rules (skill arch)
36
- - No vercel functions. No netlify functions. Use an open node server (hono/express).
37
- - Realtime uses socket (`socket.io`), not platform locked pub/sub.
38
- - Db uses prisma schema, drizzle orm, mysql2 driver.
39
- - No localhost. Host parametrized (`host`). Ports randomized then locked.
40
- - Files lowercase. No underscore. No hyphen.
41
-
42
- ## Agent requests to deployed site
43
-
44
- ```
45
- agent (sandbox) deployed site (web/)
46
- ───────────────── ──────────────────────
47
- post /sessions ───────────► create session in db
48
- post /sessions/:id/events ───────────► append json log (drizzle/mysql2)
49
- post /captcha/result ───────────► save evidence + cloudinary
50
- get /sessions/:id ───────────► read session + replay url
51
- get /sessions/:id/replay ───────────► return video (cloudinary)
52
- ```
53
-
54
- ## Environment variables (production)
55
- - `DATABASE_URL` (mysql2 / turso).
56
- - `CLOUDINARY_URL` / `CLOUDINARY_CLOUD_NAME` / `CLOUDINARY_API_KEY` / `CLOUDINARY_API_SECRET`.
57
- - `UKA_API_KEY` (agent auth).
58
- - `VERCEL_OIDC_TOKEN` (auto on vercel) or `VERCEL_TOKEN` + `VERCEL_TEAM_ID` + `VERCEL_PROJECT_ID`.
59
- - `HOST` and `PORT` (parametrized, randomized then locked).
60
-
61
- ## Deploy
62
- - Vercel: `vercel --prod`.
63
- - Netlify: `netlify deploy --prod` (static + open server build, not netlify functions).
64
- - The db is provisioned in the deploy (turso/mysql), not local sqlite.
65
-
66
- ## Database (decision)
67
- - Drizzle orm primary + mysql2 driver.
68
- - Prisma schema present as alt/sync.
69
- - Schema detailed in `09-database-schema.md`.
70
- - Cloudinary detailed in `10-cloudinary-storage.md`.
@@ -1,121 +0,0 @@
1
- # 09 Database Schema
2
-
3
- ## Decision
4
- - Primary orm: drizzle orm.
5
- - Driver: `mysql2` (skill arch: data uses mysql2). Turso/libsql is an accepted alternative.
6
- - Prisma schema included as alt/sync (`schema.prisma`).
7
- - The db is created in the deploy (mysql/turso), not local.
8
-
9
- ## Tables (drizzle, root file `web/db.js` or `schema.js`)
10
-
11
- ```js
12
- import { sql } from "drizzle-orm";
13
- import { mysqlTable, text, int, float, boolean, index, timestamp } from "drizzle-orm/mysql-core";
14
-
15
- // session of computer use (one agent / one execution)
16
- export const sessions = mysqlTable("sessions", {
17
- id: text("id").primaryKey(),
18
- agentName: text("agent_name"),
19
- browser: text("browser").default("brave"),
20
- originUrl: text("origin_url"),
21
- status: text("status").default("running"),
22
- seed: text("seed"),
23
- createdAt: timestamp("created_at").defaultNow(),
24
- finishedAt: timestamp("finished_at"),
25
- }, (t) => [index("idx_sessions_status").on(t.status)]);
26
-
27
- // captured movement events (virtual mouse, click, scroll, etc)
28
- export const events = mysqlTable("events", {
29
- id: int("id").autoincrement().primaryKey(),
30
- sessionId: text("session_id").references(() => sessions.id, { onDelete: "cascade" }),
31
- t: float("t"),
32
- type: text("type"),
33
- x: float("x"),
34
- y: float("y"),
35
- data: text("data"),
36
- createdAt: timestamp("created_at").defaultNow(),
37
- }, (t) => [index("idx_events_session").on(t.sessionId)]);
38
-
39
- // captcha results (tests)
40
- export const captchaResults = mysqlTable("captcha_results", {
41
- id: int("id").autoincrement().primaryKey(),
42
- sessionId: text("session_id").references(() => sessions.id, { onDelete: "cascade" }),
43
- kind: text("kind"),
44
- sitekey: text("sitekey"),
45
- token: text("token"),
46
- passed: boolean("passed").default(false),
47
- solver: text("solver"),
48
- evidenceUrl: text("evidence_url"),
49
- createdAt: timestamp("created_at").defaultNow(),
50
- });
51
-
52
- // heavy assets (replay videos, screenshots) — pointer to cloudinary
53
- export const assets = mysqlTable("assets", {
54
- id: int("id").autoincrement().primaryKey(),
55
- sessionId: text("session_id").references(() => sessions.id, { onDelete: "cascade" }),
56
- kind: text("kind"),
57
- cloudinaryId: text("cloudinary_id"),
58
- url: text("url"),
59
- bytes: int("bytes"),
60
- createdAt: timestamp("created_at").defaultNow(),
61
- });
62
- ```
63
-
64
- ## Prisma (`schema.prisma`) — equivalent
65
-
66
- ```prisma
67
- model Session {
68
- id String @id
69
- agentName String?
70
- browser String @default("brave")
71
- originUrl String?
72
- status String @default("running")
73
- seed String?
74
- createdAt DateTime @default(now())
75
- finishedAt DateTime?
76
- events Event[]
77
- captchaResults CaptchaResult[]
78
- assets Asset[]
79
- }
80
-
81
- model Event {
82
- id Int @id @default(autoincrement())
83
- session Session @relation(fields: [sessionId], references: [id], onDelete: Cascade)
84
- sessionId String
85
- t Float
86
- type String
87
- x Float?
88
- y Float?
89
- data String?
90
- createdAt DateTime @default(now())
91
- }
92
-
93
- model CaptchaResult {
94
- id Int @id @default(autoincrement())
95
- session Session @relation(fields: [sessionId], references: [id], onDelete: Cascade)
96
- sessionId String
97
- kind String
98
- sitekey String?
99
- token String?
100
- passed Boolean @default(false)
101
- solver String?
102
- evidenceUrl String?
103
- createdAt DateTime @default(now())
104
- }
105
-
106
- model Asset {
107
- id Int @id @default(autoincrement())
108
- session Session @relation(fields: [sessionId], references: [id], onDelete: Cascade)
109
- sessionId String
110
- kind String
111
- cloudinaryId String
112
- url String
113
- bytes Int
114
- createdAt DateTime @default(now())
115
- }
116
- ```
117
-
118
- ## Migrations
119
- - Drizzle: `drizzle-kit generate` + `drizzle-kit migrate` (mysql) in the deploy.
120
- - Prisma: `prisma migrate deploy` (mysql) as alt.
121
- - The db is created in the deploy pipeline (vercel/netlify), not local sqlite.
@@ -1,57 +0,0 @@
1
- # 10 Cloudinary Storage
2
-
3
- ## Why cloudinary
4
- The production site (`web/`) receives agent data via https. Heavy data (replay videos, screenshots, big json logs) is sent to cloudinary instead of the db, keeping the db light (only pointers/urls).
5
-
6
- ## What goes to cloudinary
7
- - Replay videos (`.mp4`/`.webm`) generated via `ffmpeg-static` from logs.
8
- - Captcha challenge screenshots (before/after).
9
- - Big json logs (long trajectories) as a complementary asset.
10
- - Training assets (click masks, challenge images).
11
-
12
- ## Flow
13
-
14
- ```
15
- agent/sandbox ──post /captcha/result──► backend (web/)
16
-
17
- ├─ write metadata to db (drizzle/mysql2)
18
- └─ upload file ──► cloudinary
19
-
20
- └─ save asset { url, cloudinaryId } in db
21
- ```
22
-
23
- ## Config (`web/storage.js`)
24
-
25
- ```js
26
- import { v2 as cloudinary } from "cloudinary";
27
-
28
- cloudinary.config({
29
- cloud_name: process.env.CLOUDINARY_CLOUD_NAME,
30
- api_key: process.env.CLOUDINARY_API_KEY,
31
- api_secret: process.env.CLOUDINARY_API_SECRET,
32
- });
33
-
34
- /**
35
- * Uploads a local file to cloudinary under uka/<folder>.
36
- * @param {string} filePath local path
37
- * @param {string} folder sub folder
38
- * @returns {Promise<object>} upload result with url and public id
39
- */
40
- export async function uploadAsset(filePath, folder) {
41
- return cloudinary.uploader.upload(filePath, {
42
- resource_type: "auto",
43
- folder: `uka/${folder}`,
44
- });
45
- }
46
- ```
47
-
48
- ## Variables
49
- - `CLOUDINARY_CLOUD_NAME`
50
- - `CLOUDINARY_API_KEY`
51
- - `CLOUDINARY_API_SECRET`
52
- - `CLOUDINARY_URL` (condensed form, optional)
53
-
54
- ## Good practices
55
- - Use `folder: "uka/<sessionId>"` for organization.
56
- - Save only the `url` + `cloudinaryId` in the db (see `09-database-schema.md`, `assets` table).
57
- - Compress videos before upload to respect plan limits.