@wenathlan/saddle 1.8.16 → 1.8.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -13
- package/dist/apps/registry.d.ts +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/isolation/contracts.d.ts +118 -0
- package/dist/isolation/contracts.d.ts.map +1 -0
- package/dist/isolation/contracts.js +84 -0
- package/dist/isolation/contracts.js.map +1 -0
- package/dist/mcp/server.d.ts +2 -2
- package/dist/scrape/robots.js +1 -1
- package/docs/artifactavailability.md +18 -2
- package/docs/containerplatforms-1.8.17.md +29 -0
- package/docs/releasenotes-1.8.16.md +6 -2
- package/docs/releasenotes-1.8.17.md +43 -0
- package/docs/releasenotes-1.8.18.md +32 -0
- package/docs/research-1.8.18-isolation.md +43 -0
- package/docs/saddle.archive.1.8.17.tar.gz.gpg +0 -0
- package/docs/todo-1.8.16.md +92 -8
- package/docs/todo-1.8.18.md +152 -0
- package/docs/workflowoperations.md +32 -0
- package/extension/manifest.json +1 -1
- package/package.json +3 -2
- package/docs/logs/.gitkeep +0 -0
- package/docs/plans/00.index.md +0 -50
- package/docs/plans/01.architecture.md +0 -86
- package/docs/plans/02.research.computer.use.md +0 -58
- package/docs/plans/03.research.captcha.bypass.md +0 -68
- package/docs/plans/04.research.sandbox.ai.md +0 -52
- package/docs/plans/05.capture.platform.md +0 -57
- package/docs/plans/06.dependencies.md +0 -97
- package/docs/plans/07.captcha.test.page.md +0 -41
- package/docs/plans/08.production.infra.md +0 -70
- package/docs/plans/09.database.schema.md +0 -121
- package/docs/plans/10.cloudinary.storage.md +0 -57
- package/docs/plans/11.movement.logs.json.md +0 -72
- package/docs/plans/12.research.atlas.agent.browser.md +0 -79
- package/docs/plans/13.research.anti.detection.md +0 -898
- package/docs/plans/14.research.proxy.md +0 -1495
- package/docs/plans/15.research.retry.rate.limit.md +0 -1958
- package/docs/plans/16.research.crawling.md +0 -1417
- package/docs/plans/17.research.caching.md +0 -1610
- package/docs/plans/18.research.content.extraction.md +0 -1952
- package/docs/plans/19.research.errors.events.md +0 -1523
- package/docs/plans/20.research.zod.validation.md +0 -1350
- package/docs/plans/21.research.batch.concurrency.md +0 -1888
- package/docs/plans/22.research.universal.runtime.md +0 -944
- package/docs/plans/23.research.ai.integration.md +0 -1465
- package/docs/plans/24.research.memory.persistence.md +0 -1979
- package/docs/plans/25.research.server.api.md +0 -342
- package/docs/plans/26.research.compilation.md +0 -249
- package/docs/plans/27.research.html.parsing.md +0 -251
- package/docs/plans/28.action.plan.md +0 -50
- package/docs/plans/29.api.reference.md +0 -174
- package/docs/plans/30.architecture.plan.md +0 -94
- package/docs/plans/31.auditoria.dados.md +0 -163
- package/docs/plans/32.bots.automacao.computacional.md +0 -214
- package/docs/plans/33.bots.codigo.revisao.md +0 -220
- package/docs/plans/34.bots.seguranca.cicd.md +0 -366
- package/docs/plans/35.comparativo.concorrencia.md +0 -464
- package/docs/plans/36.computational.memory.md +0 -340
- package/docs/plans/37.deploystrategy.md +0 -394
- package/docs/plans/38.flow.md +0 -155
- package/docs/plans/39.multi.platform.bot.md +0 -252
- package/docs/plans/40.npm.publish.md +0 -250
- package/docs/plans/41.o.que.falta.md +0 -407
- package/docs/plans/42.pesquisa.concorrencia.md +0 -721
- package/docs/plans/43.plan.universal.architecture.md +0 -496
- package/docs/plans/44.reference.md +0 -100
- package/docs/plans/45.robotarchitecture.md +0 -237
- package/docs/plans/46.scdnintegration.md +0 -284
- package/docs/plans/47.multiforge.readme.md +0 -129
- package/docs/plans/48.theory.v4.repo.os.md +0 -152
- package/docs/plans/49.third.party.infra.md +0 -12
- package/docs/plans/50.file.as.compute.md +0 -39
- package/docs/plans/51.architecture.virtual.processor.md +0 -80
- package/docs/plans/52.manifesto.v8.md +0 -11
- package/docs/plans/58.cdn.list.md +0 -23
- package/docs/plans/59.sql.frameworks.md +0 -33
- package/docs/plans/60.sql.thirdparty.md +0 -26
- package/docs/plans/61.objective.multiforge.md +0 -63
- package/docs/plans/62.huggingface.upload.md +0 -26
- package/docs/plans/63.kaggle.upload.md +0 -24
- package/docs/plans/64.npm.storage.md +0 -30
- package/docs/plans/65.rclone.terabox.md +0 -32
- package/docs/plans/66.buckets.and.models.todo.md +0 -14
- package/docs/plans/67.database.todo.md +0 -13
- package/docs/plans/68.deploy.packages.todo.md +0 -12
- package/docs/plans/69.report.human.operator.md +0 -133
- package/docs/plans/70.report.brain2qwerty.ems.md +0 -135
- package/docs/plans/71.report.hd.infinito.vram.md +0 -155
- package/docs/plans/72.plan.hd.infinito.node.md +0 -146
- package/docs/plans/73.plan.scifi.repos.md +0 -125
- package/docs/plans/74.000.manifesto.v8.flat.2..md +0 -11
- package/docs/plans/README.md +0 -489
- package/docs/plans/aggregate_platforms.mjs +0 -146
- package/docs/plans/examplesession.json +0 -36
- package/docs/plans/missing-facts.md +0 -192
- package/docs/plans/models.md +0 -64
- package/docs/plans/organize.cjs +0 -270
- package/docs/plans/platforms.md +0 -2887
- package/docs/plans/sites.md +0 -31322
- package/docs/sources/farm.py +0 -117
- package/docs/sources/html/saddle1.html +0 -132
- package/docs/sources/html/saddle2.html +0 -157
- package/docs/sources/html/saddle3.html +0 -119
- package/docs/sources/html/saddle4.html +0 -144
- package/docs/sources/html/saddle5.html +0 -72
- package/docs/sources/html/saddle6.html +0 -171
- package/docs/sources/html/saddle7.html +0 -236
- package/docs/sources/saddle.ts +0 -74
- package/docs/sources/schema.prisma +0 -88
- package/docs/sources/script.sh +0 -64
- package/docs/sources/workflows.yml +0 -458
- package/docs/talks1/_body.txt +0 -14
- package/docs/talks1/_index.md +0 -15
- package/docs/talks1/_screenshot.png +0 -0
- package/docs/talks1/assistant-01.md +0 -5
- package/docs/talks1/assistant-02.md +0 -5
- package/docs/talks1/assistant-03.md +0 -531
- package/docs/talks1/assistant-04.md +0 -26
- package/docs/talks1/assistant-05.md +0 -774
- package/docs/talks1/assistant-06.md +0 -1718
- package/docs/talks1/scrape-share.cjs +0 -185
- package/docs/talks1/scrape-share.ts +0 -183
- package/docs/talks1/user-01.md +0 -3
- package/docs/talks1/user-02.md +0 -3
- package/docs/talks1/user-03.md +0 -88
- package/docs/talks1/user-04.md +0 -3
- package/docs/talks1/user-05.md +0 -3
- package/docs/talks1/user-06.md +0 -88
- package/docs/talks1/user-07.md +0 -88
- package/docs/talks2/_body.txt +0 -14
- package/docs/talks2/_index.md +0 -16
- package/docs/talks2/_screenshot.png +0 -0
- package/docs/talks2/assistant-01.md +0 -5
- package/docs/talks2/assistant-02.md +0 -5
- package/docs/talks2/assistant-03.md +0 -424
- package/docs/talks2/assistant-04.md +0 -598
- package/docs/talks2/assistant-05.md +0 -1280
- package/docs/talks2/assistant-06.md +0 -1227
- package/docs/talks2/assistant-07.md +0 -1252
- package/docs/talks2/user-01.md +0 -3
- package/docs/talks2/user-02.md +0 -3
- package/docs/talks2/user-03.md +0 -88
- package/docs/talks2/user-04.md +0 -88
- package/docs/talks2/user-05.md +0 -88
- package/docs/talks2/user-06.md +0 -88
- package/docs/talks2/user-07.md +0 -3
- package/docs/talks3/_body.txt +0 -467
- package/docs/talks3/_index.md +0 -10
- package/docs/talks3/_screenshot.png +0 -0
- package/docs/talks3/assistant-01.md +0 -417
- package/docs/talks3/assistant-02.md +0 -417
- package/docs/talks3/assistant-03.md +0 -29
- package/docs/talks3/assistant-04.md +0 -727
- package/docs/talks3/user-01.md +0 -88
- package/docs/talks3/user-02.md +0 -88
- package/docs/talks3/user-03.md +0 -3
- package/docs/talks3/user-04.md +0 -3
- package/docs/talks4/_body.txt +0 -14
- package/docs/talks4/_index.md +0 -12
- package/docs/talks4/_screenshot.png +0 -0
- package/docs/talks4/assistant-01.md +0 -5
- package/docs/talks4/assistant-02.md +0 -5
- package/docs/talks4/assistant-03.md +0 -35
- package/docs/talks4/assistant-04.md +0 -512
- package/docs/talks4/assistant-05.md +0 -599
- package/docs/talks4/user-01.md +0 -3
- package/docs/talks4/user-02.md +0 -3
- package/docs/talks4/user-03.md +0 -88
- package/docs/talks4/user-04.md +0 -88
- package/docs/talks4/user-05.md +0 -7
- package/docs/talks5/_body.txt +0 -14
- package/docs/talks5/_index.md +0 -13
- package/docs/talks5/_screenshot.png +0 -0
- package/docs/talks5/assistant-01.md +0 -5
- package/docs/talks5/assistant-02.md +0 -5
- package/docs/talks5/assistant-03.md +0 -690
- package/docs/talks5/assistant-04.md +0 -758
- package/docs/talks5/assistant-05.md +0 -974
- package/docs/talks5/user-01.md +0 -3
- package/docs/talks5/user-02.md +0 -3
- package/docs/talks5/user-03.md +0 -105
- package/docs/talks5/user-04.md +0 -105
- package/docs/talks5/user-05.md +0 -63
- package/docs/talks5/user-06.md +0 -105
- package/docs/talks6/_body.txt +0 -14
- package/docs/talks6/_index.md +0 -9
- package/docs/talks6/_screenshot.png +0 -0
- package/docs/talks6/assistant-01.md +0 -5
- package/docs/talks6/assistant-02.md +0 -5
- package/docs/talks6/assistant-03.md +0 -1499
- package/docs/talks6/user-01.md +0 -3
- package/docs/talks6/user-02.md +0 -3
- package/docs/talks6/user-03.md +0 -88
- package/docs/talks6/user-04.md +0 -88
- package/docs/talks7/_body.txt +0 -14
- package/docs/talks7/_index.md +0 -10
- package/docs/talks7/_screenshot.png +0 -0
- package/docs/talks7/assistant-01.md +0 -5
- package/docs/talks7/assistant-02.md +0 -5
- package/docs/talks7/assistant-03.md +0 -523
- package/docs/talks7/assistant-04.md +0 -617
- package/docs/talks7/user-01.md +0 -3
- package/docs/talks7/user-02.md +0 -3
- package/docs/talks7/user-03.md +0 -105
- package/docs/talks7/user-04.md +0 -67
- package/docs/talks8/conversa1.txt +0 -1322
- package/docs/talks8/conversa2.txt +0 -237
- package/docs/talks9/Beyond the Obvious_ 50 Plataformas Auto-Hospedadas de Forja de C/303/263digo para Al/303/251m de Gitea e GitLab.md" +0 -174
- package/docs/talks9/De NPM a Multi-Linguagem_ Uma Arquitetura T/303/251cnica para a Execu/303/247/303/243o Integrada de C/303/263digo no Ecossistema Node.js.md" +0 -59
- package/docs/talks9/De NPM a VMs Virtuais_ Uma An/303/241lise Arquitet/303/264nica para a Realiza/303/247/303/243o do Ciclo de Vida do Projeto SADDLE.md" +0 -91
- package/docs/talks9/Mapeamento da Engrenagem Computacional_ Uma Arquitetura para Execu/303/247/303/243o Isolada e Persist/303/252ncia em Ambientes Distribu/303/255dos.md" +0 -116
- package/docs/talks9/O Cen/303/241rio Pr/303/241tico do SADDLE_ Uma An/303/241lise de Viabilidade e Modelo de Ciclo de Vida Integrado.md" +0 -128
- package/docs/talks9/README (2).md +0 -489
- package/docs/talks9/README.md +0 -198
- package/docs/talks9/Viabilidade do Saddle_ Uma An/303/241lise T/303/251cnica da Transforma/303/247/303/243o de Armazenamento Remoto em Mem/303/263ria Computacional.md" +0 -80
- package/docs/talks9/conversa.txt +0 -544
- package/docs/talks9/other (2).md +0 -39
- package/docs/talks9/other.md +0 -57
- package/docs/talks9/outro.txt +0 -24
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
# 02 — Pesquisa: Computer Use Agents (2026)
|
|
2
|
-
|
|
3
|
-
## Estado da arte
|
|
4
|
-
|
|
5
|
-
Em 2026, *computer-use agents* deixaram de ser protótipos e entraram em produção para
|
|
6
|
-
tarefas restritas e supervisionadas. Principais provedores e frameworks:
|
|
7
|
-
|
|
8
|
-
- **Anthropic Computer Use** (`computer_use_20251124`): ferramentas `computer` (mouse/teclado),
|
|
9
|
-
`text_editor`, `bash`. Recomenda rodar em **VMs/containers com privilégios mínimos**.
|
|
10
|
-
- **OpenAI Operator / CUA**: 87% de sucesso em sites JS complexos; 58% no WebArena.
|
|
11
|
-
- **Google Project Mariner**: 83.5% WebVoyager, 84% ScreenSpot.
|
|
12
|
-
- **Microsoft UFO²**: funde Windows UI Automation + OmniParser (visão).
|
|
13
|
-
- **browser-use** (open source): 89.1% WebVoyager (abordagem híbrida DOM + visão).
|
|
14
|
-
|
|
15
|
-
### Benchmarks
|
|
16
|
-
- **OSWorld:** humano 72.36% vs SOTA ~12–20% (desktop ainda imaturo).
|
|
17
|
-
- **WebArena:** tops ~71%, produção na faixa 50–60%.
|
|
18
|
-
- **WebVoyager:** browser-use 89.1% (híbrido).
|
|
19
|
-
|
|
20
|
-
### Conclusão de arquitetura
|
|
21
|
-
Ganham abordagens **híbridas**:
|
|
22
|
-
1. Raciocínio por **DOM / accessibility tree** para elementos estruturados.
|
|
23
|
-
2. **Visão** (VLM) para layouts não-padrão, canvas, imagens.
|
|
24
|
-
3. **Scripts determinísticos** para validação e replay.
|
|
25
|
-
4. **Verificação** (screenshot antes/depois, diff de elementos).
|
|
26
|
-
|
|
27
|
-
## Projetos de referência encontrados
|
|
28
|
-
|
|
29
|
-
| Projeto | Diferencial relevante para UKA |
|
|
30
|
-
|---------|-------------------------------|
|
|
31
|
-
| [agentbrowser](https://github.com/AshtonVaughan/agentbrowser) | Cursor visível humano via CDP, trilha SVG, trajetórias Bézier com jitter/overshoot, **traces JSONL** em `~/.agentbrowser/traces/`, replay determinístico por seed, bypass de Cloudflare/OneTrust |
|
|
32
|
-
| [humanjs](https://github.com/totigm/humanjs) | Playwright humanizado: curvas Bézier, ritmo de digitação, scroll, **personalidades** (careful/fast/distracted/precise), gravador → JSON timeline, MCP server |
|
|
33
|
-
| [human-browser-use](https://github.com/andyless/human-browser-use) | Extensão p/ browser-use: trajetória Bézier, typing lognormal, scroll inercial, stealth fingerprint (`navigator.webdriver`, WebGL, canvas) |
|
|
34
|
-
| [agentic-stealth-browser](https://github.com/shanewas/agentic-stealth-browser) | TLS spoofing (JA3/JA4), fingerprint, recovery chain (detecta captcha → rotaciona proxy), account warming |
|
|
35
|
-
| [mantis](https://github.com/mercurialsolo/mantis) | Agent perception-reasoning-action em Xvfb+Chrome+xdotool (sem fingerprint Playwright), vídeo screencast, deploy em Baseten/Modal/EKS/GKE |
|
|
36
|
-
| [win-pilot-mcp](https://github.com/omidmanoochehri/win-pilot-mcp) | MCP Windows computer-use: screen capture, OCR, mouse humano, logging estruturado, replay de macros |
|
|
37
|
-
| [cloakbrowser-human](https://github.com/evelaa123/cloakbrowser-human) | Wrapper Playwright que adiciona camada **comportamental** (mouse Bézier, typing, scroll) — complementa stealth de fingerprint |
|
|
38
|
-
|
|
39
|
-
## Padrões de movimento humano (extraídos da pesquisa)
|
|
40
|
-
|
|
41
|
-
- **Trajetória do mouse:** curva Bézier única, velocidade variável por arc-length:
|
|
42
|
-
- 0–5%: ramp-up (0.3x → 2.5x)
|
|
43
|
-
- 5–75%: cruise (2.3–2.5x) com variação senoidal
|
|
44
|
-
- 75–100%: ease-out cúbico (2.5x → 0.3x)
|
|
45
|
-
- sub-pixel drift final (σ 0.3–1.5px)
|
|
46
|
-
- duração via **Lei de Fitts**: `0.05 + 0.07*log2(1 + distance/20)` s
|
|
47
|
-
- **Overshoot:** ~15% de chance, auto-aumenta em movimentos longos.
|
|
48
|
-
- **Typing:** inter-key delay lognormal (μ=4.17, σ=0.3 → ~65ms), bigram speedup, 2% typo.
|
|
49
|
-
- **Scroll:** wheel events com acel/decel, overshoot, settle delay.
|
|
50
|
-
- **Stealth:** `navigator.webdriver → undefined`, spoof de plugins/WebGL/canvas, flags de launch.
|
|
51
|
-
|
|
52
|
-
## Aplicação no UKA
|
|
53
|
-
|
|
54
|
-
- Implementar `HumanCursor` (SVG overlay + CDP `Input.dispatchMouseEvent`) inspirado no
|
|
55
|
-
agentbrowser/humanjs, com trajetórias Bézier + jitter + overshoot determinísticas por seed.
|
|
56
|
-
- Gravar **todos os eventos** como `SessionEvent` tipado → exportar JSON para `docs/logs/`.
|
|
57
|
-
- Usar replay determinístico (seed) para testes reprodutíveis.
|
|
58
|
-
- Tratar GUI como imagem (visão) + DOM/accessibility para alvos estruturados.
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
# 03 — Pesquisa: hCaptcha / Cloudflare / reCAPTCHA Bypass
|
|
2
|
-
|
|
3
|
-
## Contexto
|
|
4
|
-
|
|
5
|
-
O UKA precisa **testar** a capacidade do computer-use de enfrentar captchas fortes. A
|
|
6
|
-
pesquisa abaixo levanta abordagens open-source e comerciais (uso edUKAcional / automação
|
|
7
|
-
supervisionada).
|
|
8
|
-
|
|
9
|
-
## hCaptcha
|
|
10
|
-
|
|
11
|
-
### hcaptcha-challenger (QIN2DIM) — referência principal
|
|
12
|
-
- Resolve hCaptcha com **LLM multimodal** (sem Tampermonkey, sem serviço anti-captcha 3º).
|
|
13
|
-
- Modelos plugáveis (ONNX):
|
|
14
|
-
- `image_label_binary` → ResNet ✅
|
|
15
|
-
- `image_label_area_select: point` → YOLOv8 ✅
|
|
16
|
-
- `image_label_area_select: bounding box` → YOLOv8 segmentation
|
|
17
|
-
- `image_drag_drop` → Spatial Chain-of-Thought ✅
|
|
18
|
-
- `image_label_multiple_choice` → ViT zero-shot
|
|
19
|
-
- Pipeline: `install()` → `AgentT.from_page()` → `handle_checkbox()` → `execute()` loop.
|
|
20
|
-
- Deps: `playwright`, `httpx`, `opencv-python`, `pillow`, `loguru`, `pydantic-settings`.
|
|
21
|
-
|
|
22
|
-
### Outros solvers
|
|
23
|
-
- **hCaptcha-Solver (Camoufox + Multibot):** usa Camoufox p/ HSW token + Groq vision +
|
|
24
|
-
**geração de movimento de mouse realista** (Multibot API). Lida com `image_label_binary`,
|
|
25
|
-
`area_select`, `drag_drop`. Requer proxy.
|
|
26
|
-
- **maximedrn/hcaptcha-solver-python-selenium:** userscript Tampermonkey + Selenium (edUKAcional).
|
|
27
|
-
- **auto-captcha-solver / auto-captcha (NopeCHA):** detecta hCaptcha, reCAPTCHA v2/v3,
|
|
28
|
-
Cloudflare Turnstile via **API de token NopeCHA**; injeta token automaticamente. Tem MCP
|
|
29
|
-
server e CLI. Tipos: hCaptcha ✅, reCAPTCHA v2/v3 ✅, Turnstile ⚠️ (lento).
|
|
30
|
-
|
|
31
|
-
## Cloudflare (Turnstile / interstitials)
|
|
32
|
-
|
|
33
|
-
- **agentic-stealth-browser** passa Cloudflare interstitial usando TLS spoofing + fingerprint
|
|
34
|
-
+ comportamento humano + **recovery chain** (detecta captcha → rotaciona proxy/sessão → retry).
|
|
35
|
-
- **auto-captcha** resolve Turnstile via NopeCHA (fila lenta, requer proxy em produção).
|
|
36
|
-
- **captcha-solver (CloakBrowser + 2Captcha/CapSolver):** Turnstile resolvido em ~5s via
|
|
37
|
-
ClickSolver (free) ou fallback pago.
|
|
38
|
-
|
|
39
|
-
## reCAPTCHA
|
|
40
|
-
|
|
41
|
-
- v2/v3/Enterprise: cobertos por hcaptcha-challenger (via proxy), auto-captcha (NopeCHA),
|
|
42
|
-
2Captcha, CapSolver.
|
|
43
|
-
|
|
44
|
-
## Capes (tipos suportados por 2Captcha/CapSolver — 30+)
|
|
45
|
-
|
|
46
|
-
reCAPTCHA v2/v3/Enterprise, Cloudflare Turnstile, hCaptcha, FunCaptcha (Arkose), GeeTest,
|
|
47
|
-
KeyCaptcha, Amazon WAF, DataDome, Akamai, Imperva, Friendly, MTCaptcha, Lemin, Cutcaptcha,
|
|
48
|
-
Tencent, Yandex, ALTCHA, Prosopo, entre outros.
|
|
49
|
-
|
|
50
|
-
## Camadas de defesa que o UKA deve considerar
|
|
51
|
-
|
|
52
|
-
| Camada | O que ataca | Mitigação no UKA |
|
|
53
|
-
|--------|-------------|------------------|
|
|
54
|
-
| Fingerprint (WebDriver, CDP, headless, GPU) | detecta automação | stealth JS + Brave real/CDP, spoof WebGL/canvas |
|
|
55
|
-
| Comportamental (mouse/keyboard/scroll) | analisa padrões | trajetórias Bézier humanas, typing lognormal |
|
|
56
|
-
| TLS (JA3/JA4) | identifica cliente | TLS regional / proxy residencial sticky |
|
|
57
|
-
| IP reputation | bloqueia faixas | proxies residenciais com sessão sticky |
|
|
58
|
-
| Captcha | desafio final | solver VLM (hcaptcha-challenger) ou token API |
|
|
59
|
-
|
|
60
|
-
## Aplicação no UKA
|
|
61
|
-
|
|
62
|
-
- Criar `tests/examples/hcaptcha-test.html` (página de teste local) + usar sites reais fortes
|
|
63
|
-
(Cloudflare, Google reCAPTCHA demo, hCaptcha demo) no Brave.
|
|
64
|
-
- Pipeline de teste: abrir → detectar captcha → resolver (VLM ou token API) → registrar
|
|
65
|
-
token/screenshot/log em `tests/output/`.
|
|
66
|
-
- **Não** usar para abuso: foco em automação supervisionada e pesquisa.
|
|
67
|
-
- Dependências relevantes: `playwright`, `puppeteer`, `@xenova/transformers`,
|
|
68
|
-
`onnxruntime-web`, `tesseract.js`, `cloudinary` (evidências).
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
# 04 — Pesquisa: Sandbox Virtual e Containers para IA
|
|
2
|
-
|
|
3
|
-
## Por que sandbox?
|
|
4
|
-
|
|
5
|
-
Computer-use agents executam código/inputs não confiáveis (saída de LLM, uploads de
|
|
6
|
-
usuário). Rodar em sandbox isola a execução, impede acesso ao sistema de produção e limita
|
|
7
|
-
recursos. A pesquisa mostra que provedores ladearam para **microVMs Firecracker**.
|
|
8
|
-
|
|
9
|
-
## Vercel Sandbox (opção principal do UKA)
|
|
10
|
-
|
|
11
|
-
- Compute primitive para rodar código não confiável com segurança (AI agents, code gen).
|
|
12
|
-
- **Firecracker microVM** por sandbox, filesystem e rede próprios, isolamento forte.
|
|
13
|
-
- Runtimes: `node26/24/22`, `python3.13`. Default `node24`.
|
|
14
|
-
- Cada sandbox roda como `vercel-sandbox` (sudo disponível), working dir `/vercel/sandbox`.
|
|
15
|
-
- Startup em **milissegundos**; suporta Docker/FUSE dentro do microVM (privilégio elevado).
|
|
16
|
-
- **Credential brokering:** injeta secrets em requests de saída sem expor dentro do VM
|
|
17
|
-
(`console.log(process.env)` não vaza a chave).
|
|
18
|
-
- **Network policy:** allowlist de domínios, atualizável em runtime.
|
|
19
|
-
- SDK JS e Python; CLI. Multi-agent isolation (usuário Linux por agente, grupos p/ compartilhar).
|
|
20
|
-
|
|
21
|
-
### AI SDK Computer Use (vercel-labs) — template de referência
|
|
22
|
-
- Next.js + AI SDK + Claude Sonnet 4.5 + **Vercel Sandbox**.
|
|
23
|
-
- Snapshot pré-construído com: **Xvnc**, **openbox**, **noVNC + websockify**, **Google
|
|
24
|
-
Chrome**, **xdotool + ImageMagick**.
|
|
25
|
-
- Fluxo: User ↔ Chat UI ↔ AI SDK ↔ Claude → Vercel Sandbox (desktop via noVNC iframe).
|
|
26
|
-
- Ferramentas: `computer` (screenshot/click/type/scroll) + `bash`.
|
|
27
|
-
- Env: `ANTHROPIC_API_KEY`, `SANDBOX_SNAPSHOT_ID`, `VERCEL_OIDC_TOKEN`/`VERCEL_TOKEN`.
|
|
28
|
-
|
|
29
|
-
### vercel-labs/agent-browser (serverless)
|
|
30
|
-
- CLI Rust nativo; roda agent-browser + Chrome em Vercel Sandbox microVM efêmero.
|
|
31
|
-
- Helper `@agent-browser/sandbox/vercel` + `@vercel/sandbox`.
|
|
32
|
-
|
|
33
|
-
### vercel-labs/sandcastle (PoC desktop)
|
|
34
|
-
- Desktop Linux web sobre Vercel Sandbox. Stack: Next.js, Tailwind, **Zustand + SWR**,
|
|
35
|
-
**Neon Postgres + Drizzle ORM**, `@vercel/sandbox`, code-server, Xpra. Auth password/OAuth.
|
|
36
|
-
|
|
37
|
-
## Outras opções
|
|
38
|
-
|
|
39
|
-
- **Modal / Baseten / EKS / GKE** (usados pelo mantis) — containers Docker customizados.
|
|
40
|
-
- **Daytona / Cloudflare** — alternativas de sandbox (citadas mas com menos fit p/ TS).
|
|
41
|
-
- **Containers Docker locais** (xdotool + Xvfb + Chromium) para dev sem nuvem.
|
|
42
|
-
|
|
43
|
-
## Decisão para o UKA
|
|
44
|
-
|
|
45
|
-
- **Produção:** Vercel Sandbox como camada de execução do computer-use; site de controle
|
|
46
|
-
deployado em **Vercel** (ou Netlify) (`web/`).
|
|
47
|
-
- **Banco:** criado no deploy (Drizzle/LibSQL ou Prisma em Neon/Turso), conforme `09-esquema-banco-dados.md`.
|
|
48
|
-
- **Armazenamento:** Cloudinary para assets/replays (`10-cloudinary-storage.md`).
|
|
49
|
-
- **Dev local:** Brave real via Playwright/CDP para a plataforma de captura (`05-plataforma-captura-movimentos.md`).
|
|
50
|
-
- **Comunicação:** requests **POST/GET HTTPS** do agente → site deployado → DB + Cloudinary.
|
|
51
|
-
- Dependências: `@vercel/sandbox`, `vercel`, `@netlify/blobs`, `@netlify/functions`,
|
|
52
|
-
`hono`/`express`, `drizzle-orm`, `@libsql/client`, `socket.io`.
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
# 05 Capture Platform
|
|
2
|
-
|
|
3
|
-
## Purpose
|
|
4
|
-
|
|
5
|
-
Build a web platform that captures the user/agent movement inside the browser (focus: brave) and exports everything as json logs to `docs/logs/`. This is the base to train and replay the UKA computer use with human behavior.
|
|
6
|
-
|
|
7
|
-
## Core concepts
|
|
8
|
-
|
|
9
|
-
- Virtual mouse / virtual arrow: an svg cursor injected in the page (via cdp init script) that follows the real movement and draws a trail and click ripple.
|
|
10
|
-
- Event capture: click, double click, drag, scroll, coordinates (x,y), rotation (scroll/wheel angle), timing, target (selector/role/text), deterministic seed.
|
|
11
|
-
- Export: compacted json in `docs/logs/<session>.json`.
|
|
12
|
-
|
|
13
|
-
## Platform architecture (root based, no src)
|
|
14
|
-
|
|
15
|
-
```
|
|
16
|
-
web/ (capture frontend)
|
|
17
|
-
├─ index.html # capture page
|
|
18
|
-
├─ capture.js # injects svg cursor + listeners
|
|
19
|
-
├─ recorder.js # collects sessionevent to buffer
|
|
20
|
-
└─ exporter.js # serializes to docs/logs/<session>.json
|
|
21
|
-
|
|
22
|
-
(root logic files, grouped per skill arch)
|
|
23
|
-
├─ humancursor.js # bezier + jitter + overshoot (cdp Input.dispatchMouseEvent)
|
|
24
|
-
├─ trajectory.js # path generator (ghost-cursor / humanjs inspired)
|
|
25
|
-
├─ fingerprint.js # stealth patches webgl/canvas/audio
|
|
26
|
-
├─ recorder.js # json event stream
|
|
27
|
-
├─ replay.js # deterministic replay by seed
|
|
28
|
-
└─ runcapture.js # entrypoint (opens brave via cdp)
|
|
29
|
-
```
|
|
30
|
-
|
|
31
|
-
## Sessionevent format
|
|
32
|
-
|
|
33
|
-
```ts
|
|
34
|
-
type SessionEvent =
|
|
35
|
-
| { t: number; type: "move"; x: number; y: number; tx: number; ty: number }
|
|
36
|
-
| { t: number; type: "click"; x: number; y: number; button: "left"|"right"; target?: string }
|
|
37
|
-
| { t: number; type: "drag"; x0: number; y0: number; x1: number; y1: number }
|
|
38
|
-
| { t: number; type: "scroll"; dx: number; dy: number; angle: number }
|
|
39
|
-
| { t: number; type: "key"; key: string; target?: string }
|
|
40
|
-
| { t: number; type: string; [k: string]: unknown };
|
|
41
|
-
```
|
|
42
|
-
|
|
43
|
-
## How it works (summary)
|
|
44
|
-
|
|
45
|
-
1. Open brave via playwright/chromium with `chrome.debugger` (cdp).
|
|
46
|
-
2. Inject svg cursor sprite plus event listeners (`Input.dispatchMouseEvent`).
|
|
47
|
-
3. Move the cursor humanly (bezier with jitter/overshoot), all via cdp, not locators.
|
|
48
|
-
4. Each action emits a `SessionEvent` with timestamp, coordinates, target, seed.
|
|
49
|
-
5. Compact trajectories (collapse move runs) and export to `docs/logs/`.
|
|
50
|
-
|
|
51
|
-
## Relation with logs
|
|
52
|
-
|
|
53
|
-
The final json format of `docs/logs/` files is detailed in `11-movement-logs-json.md`. An example log is at `docs/logs/examplesession.json`.
|
|
54
|
-
|
|
55
|
-
## Dependencies involved
|
|
56
|
-
|
|
57
|
-
`playwright`, `puppeteer`, `chromium-bidi`, `bezier-js`, `@napi-rs/canvas`, `d3`, `socket.io-client`, `zod` (event validation).
|
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
# 06 Dependencies
|
|
2
|
-
|
|
3
|
-
Dependency list of the `@wenathlan/UKA` package, grouped by function. Total: 65 packages (including the dev tooling set). The user asked for "50 to 60, or 60+. Counting 10 more", so this set covers browser automation, ai/vision, sandbox, server, db, storage and validation.
|
|
4
|
-
|
|
5
|
-
## Browser automation and mouse (8)
|
|
6
|
-
1. `playwright` — real brave/chromium automation via cdp.
|
|
7
|
-
2. `puppeteer` — fallback automation / stealth.
|
|
8
|
-
3. `chromium-bidi` — bidi protocol for input control.
|
|
9
|
-
4. `@playwright/test` — e2e tests for the platform and captcha.
|
|
10
|
-
5. `bezier-js` — human mouse trajectory generation (bezier).
|
|
11
|
-
6. `@napi-rs/canvas` — draw virtual cursor / trail / ripple.
|
|
12
|
-
7. `d3` — movement trajectory visualization.
|
|
13
|
-
8. `sharp` — screenshot/evidence processing.
|
|
14
|
-
|
|
15
|
-
## Ai / vision / captcha (12)
|
|
16
|
-
9. `ai` — ai sdk core.
|
|
17
|
-
10. `@ai-sdk/anthropic` — claude (computer use).
|
|
18
|
-
11. `@ai-sdk/google` — gemini (vision).
|
|
19
|
-
12. `@ai-sdk/openai` — gpt (vision/reasoning).
|
|
20
|
-
13. `@ai-sdk/react` — agent ui hooks.
|
|
21
|
-
14. `@ai-sdk/vercel` — vercel integration.
|
|
22
|
-
15. `@xenova/transformers` — local vlm models (onnx).
|
|
23
|
-
16. `onnxruntime-web` — onnx inference in browser (yolo/resnet).
|
|
24
|
-
17. `tesseract.js` — ocr for captcha/text.
|
|
25
|
-
18. `openai` — raw sdk for vision/solvers.
|
|
26
|
-
19. `comlink` — workers for inference in a separate thread.
|
|
27
|
-
20. `jsdom` — virtual dom for parsing/tests.
|
|
28
|
-
|
|
29
|
-
## Sandbox and deploy (3)
|
|
30
|
-
21. `@vercel/sandbox` — firecracker microvm for isolated execution.
|
|
31
|
-
22. `vercel` — cli/deploy to vercel.
|
|
32
|
-
23. `nanoid` — short url-safe ids.
|
|
33
|
-
|
|
34
|
-
> Note: `@netlify/functions` and `@netlify/blobs` were removed. The architecture skill rejects netlify/vercel functions and platform locked storage. Realtime uses socket. Db uses prisma/drizzle/mysql2.
|
|
35
|
-
|
|
36
|
-
## Server / api (7)
|
|
37
|
-
24. `hono` — light http api (production).
|
|
38
|
-
25. `express` — alternative api / dev.
|
|
39
|
-
26. `cors` — cors for agent requests.
|
|
40
|
-
27. `helmet` — security headers.
|
|
41
|
-
28. `socket.io` — realtime event streaming (socket).
|
|
42
|
-
29. `socket.io-client` — replay/stream client.
|
|
43
|
-
30. `multer` — file upload (tokens, evidence).
|
|
44
|
-
|
|
45
|
-
## Database / orm (5)
|
|
46
|
-
31. `drizzle-orm` — primary orm.
|
|
47
|
-
32. `mysql2` — mysql driver (skill arch: data uses mysql2).
|
|
48
|
-
33. `@prisma/client` — prisma client.
|
|
49
|
-
34. `prisma` — cli/migrations.
|
|
50
|
-
35. `yaml` — schema/env config.
|
|
51
|
-
|
|
52
|
-
## Storage (1)
|
|
53
|
-
36. `cloudinary` — upload replays, screenshots, assets.
|
|
54
|
-
|
|
55
|
-
## Validation / types (3)
|
|
56
|
-
37. `zod` — event/log schema.
|
|
57
|
-
38. `valibot` — light validation.
|
|
58
|
-
39. `arktype` — runtime types.
|
|
59
|
-
|
|
60
|
-
## Logging (1)
|
|
61
|
-
40. `pino` — structured (json) agent logs.
|
|
62
|
-
|
|
63
|
-
## Frontend / build (8)
|
|
64
|
-
41. `react` — platform ui.
|
|
65
|
-
42. `react-dom` — react render.
|
|
66
|
-
43. `next` — production web framework.
|
|
67
|
-
44. `vite` — bundler.
|
|
68
|
-
45. `@vitejs/plugin-react` — react plugin for vite.
|
|
69
|
-
46. `tailwindcss` — styling.
|
|
70
|
-
47. `@tailwindcss/vite` — tailwind+vite integration.
|
|
71
|
-
48. `@testing-library/react` — component tests.
|
|
72
|
-
49. `@testing-library/dom` — dom test utils.
|
|
73
|
-
50. `@testing-library/jest-dom` — test matchers.
|
|
74
|
-
|
|
75
|
-
## Media / util (4)
|
|
76
|
-
51. `pngjs` — png read/write (click masks).
|
|
77
|
-
52. `ffmpeg-static` — replay video recording.
|
|
78
|
-
53. `@t3-oss/env-core` — typed env validation.
|
|
79
|
-
54. `uuid` — session/agent ids.
|
|
80
|
-
|
|
81
|
-
## Dev tooling (11 — the "10+")
|
|
82
|
-
55. `typescript` — compiler.
|
|
83
|
-
56. `@biomejs/biome` — lint/format.
|
|
84
|
-
57. `vitest` — test runner.
|
|
85
|
-
58. `@vitest/coverage-v8` — coverage.
|
|
86
|
-
59. `tsx` — run ts without build.
|
|
87
|
-
60. `@types/node` — node types.
|
|
88
|
-
61. `@types/react` — react types.
|
|
89
|
-
62. `@types/react-dom` — react dom types.
|
|
90
|
-
63. `@ai-sdk/anthropic` (dev) — mirrored in devdeps.
|
|
91
|
-
64. `@vercel/sandbox` (dev) — mirrored in devdeps.
|
|
92
|
-
65. `prisma` (dev) — cli in devdeps.
|
|
93
|
-
|
|
94
|
-
## Conventions
|
|
95
|
-
- Files lowercase. No underscore. No hyphen.
|
|
96
|
-
- English only. Jsdoc comments. Third person voice.
|
|
97
|
-
- No hardcoded host. Use `host`. Ports randomized then locked.
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
# 07 Captcha Test Page
|
|
2
|
-
|
|
3
|
-
## Goal
|
|
4
|
-
|
|
5
|
-
Have a controlled target to validate the UKA computer use against strong captchas. Open it in brave (via the capture platform) and record the result.
|
|
6
|
-
|
|
7
|
-
## Target options
|
|
8
|
-
|
|
9
|
-
### A. Local page (`tests/examples/hcaptchatest.html`) — recommended to start
|
|
10
|
-
- Static html with the hcaptcha widget using the public test sitekey.
|
|
11
|
-
- Buttons to trigger the challenge, show the resolved token and status.
|
|
12
|
-
- Validates detection + solver + log without heavy external network.
|
|
13
|
-
|
|
14
|
-
### B. Real strong sites — advanced tests
|
|
15
|
-
- hcaptcha demo: `https://accounts.hcaptcha.com/demo`
|
|
16
|
-
- cloudflare turnstile: `https://challenges.cloudflare.com/turnstile`
|
|
17
|
-
- google recaptcha: `https://www.google.com/recaptcha/api2/demo`
|
|
18
|
-
|
|
19
|
-
## Test pipeline (script in `tests/scripts/`)
|
|
20
|
-
|
|
21
|
-
1. `launchbrave()` — opens real brave via playwright/cdp.
|
|
22
|
-
2. `navigate(url)` — goes to the test page.
|
|
23
|
-
3. `detectcaptcha(page)` — scans dom/frames for hcaptcha / turnstile / recaptcha.
|
|
24
|
-
4. `solve(page)` — uses vlm solver (hcaptcha challenger) or token api (nopecha/2captcha).
|
|
25
|
-
5. `record(page)` — saves screenshot, token and movement log to `tests/output/`.
|
|
26
|
-
6. `assert(passed)` — verifies the challenge was passed.
|
|
27
|
-
|
|
28
|
-
## Evidence in `tests/output/`
|
|
29
|
-
- `hcaptcha-<ts>.json` — token + metadata.
|
|
30
|
-
- `hcaptcha-<ts>.png` — screenshot of challenge and post-solve.
|
|
31
|
-
- `hcaptcha-<ts>-trace.json` — movement log (replay).
|
|
32
|
-
- `hcaptcha-<ts>.mp4` — replay video (via `ffmpeg-static`).
|
|
33
|
-
|
|
34
|
-
## Success criteria
|
|
35
|
-
- Auto detects captcha type.
|
|
36
|
-
- Solves hcaptcha (checkbox) with vlm/llm.
|
|
37
|
-
- Passes cloudflare turnstile in at least 1 of 3 attempts (with sticky proxy).
|
|
38
|
-
- Generates reproducible json log (seed).
|
|
39
|
-
|
|
40
|
-
## Ethical note
|
|
41
|
-
For research and supervised automation only. Respect tos and rate limits.
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
# 08 Production Infra
|
|
2
|
-
|
|
3
|
-
## Goal
|
|
4
|
-
|
|
5
|
-
The `web/` folder is the production site. It is deployed to vercel or netlify and works as the sandbox backend: it receives agent data via https requests (post/get), writes to the db and saves assets to cloudinary.
|
|
6
|
-
|
|
7
|
-
## Why deploy (user decision)
|
|
8
|
-
- The database is created in the deploy (vercel/netlify + turso/mysql), not local.
|
|
9
|
-
- The agent/sandbox talks to the site via https, no own infra needed.
|
|
10
|
-
- Cloudinary holds heavy data (replay videos, screenshots, big logs).
|
|
11
|
-
|
|
12
|
-
## Structure of `web/` (root based, no src)
|
|
13
|
-
|
|
14
|
-
```
|
|
15
|
-
web/
|
|
16
|
-
├── index.js # entrypoint (open node server, not platform functions)
|
|
17
|
-
├── server.js # api routes (hono or express)
|
|
18
|
-
├── capture.js # browser movement capture (virtual mouse, arrow)
|
|
19
|
-
├── recorder.js # json event stream
|
|
20
|
-
├── replay.js # deterministic replay by seed
|
|
21
|
-
├── humancursor.js # bezier + jitter + overshoot (cdp)
|
|
22
|
-
├── trajectory.js # path generator
|
|
23
|
-
├── fingerprint.js # stealth patches
|
|
24
|
-
├── db.js # drizzle + mysql2 + prisma loader
|
|
25
|
-
├── storage.js # cloudinary upload
|
|
26
|
-
├── auth.js # api key / oidc (VERCEL_OIDC_TOKEN)
|
|
27
|
-
├── vercel.json # vercel build/deploy config
|
|
28
|
-
├── netlify.toml # netlify deploy config (alt)
|
|
29
|
-
├── drizzle.config.js # drizzle -> mysql2
|
|
30
|
-
├── schema.prisma # prisma schema (alt/sync)
|
|
31
|
-
├── index.html # capture ui
|
|
32
|
-
└── hcaptchatest.html # captcha test page (mirror of tests/examples)
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
## Backend rules (skill arch)
|
|
36
|
-
- No vercel functions. No netlify functions. Use an open node server (hono/express).
|
|
37
|
-
- Realtime uses socket (`socket.io`), not platform locked pub/sub.
|
|
38
|
-
- Db uses prisma schema, drizzle orm, mysql2 driver.
|
|
39
|
-
- No localhost. Host parametrized (`host`). Ports randomized then locked.
|
|
40
|
-
- Files lowercase. No underscore. No hyphen.
|
|
41
|
-
|
|
42
|
-
## Agent requests to deployed site
|
|
43
|
-
|
|
44
|
-
```
|
|
45
|
-
agent (sandbox) deployed site (web/)
|
|
46
|
-
───────────────── ──────────────────────
|
|
47
|
-
post /sessions ───────────► create session in db
|
|
48
|
-
post /sessions/:id/events ───────────► append json log (drizzle/mysql2)
|
|
49
|
-
post /captcha/result ───────────► save evidence + cloudinary
|
|
50
|
-
get /sessions/:id ───────────► read session + replay url
|
|
51
|
-
get /sessions/:id/replay ───────────► return video (cloudinary)
|
|
52
|
-
```
|
|
53
|
-
|
|
54
|
-
## Environment variables (production)
|
|
55
|
-
- `DATABASE_URL` (mysql2 / turso).
|
|
56
|
-
- `CLOUDINARY_URL` / `CLOUDINARY_CLOUD_NAME` / `CLOUDINARY_API_KEY` / `CLOUDINARY_API_SECRET`.
|
|
57
|
-
- `UKA_API_KEY` (agent auth).
|
|
58
|
-
- `VERCEL_OIDC_TOKEN` (auto on vercel) or `VERCEL_TOKEN` + `VERCEL_TEAM_ID` + `VERCEL_PROJECT_ID`.
|
|
59
|
-
- `HOST` and `PORT` (parametrized, randomized then locked).
|
|
60
|
-
|
|
61
|
-
## Deploy
|
|
62
|
-
- Vercel: `vercel --prod`.
|
|
63
|
-
- Netlify: `netlify deploy --prod` (static + open server build, not netlify functions).
|
|
64
|
-
- The db is provisioned in the deploy (turso/mysql), not local sqlite.
|
|
65
|
-
|
|
66
|
-
## Database (decision)
|
|
67
|
-
- Drizzle orm primary + mysql2 driver.
|
|
68
|
-
- Prisma schema present as alt/sync.
|
|
69
|
-
- Schema detailed in `09-database-schema.md`.
|
|
70
|
-
- Cloudinary detailed in `10-cloudinary-storage.md`.
|
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
# 09 Database Schema
|
|
2
|
-
|
|
3
|
-
## Decision
|
|
4
|
-
- Primary orm: drizzle orm.
|
|
5
|
-
- Driver: `mysql2` (skill arch: data uses mysql2). Turso/libsql is an accepted alternative.
|
|
6
|
-
- Prisma schema included as alt/sync (`schema.prisma`).
|
|
7
|
-
- The db is created in the deploy (mysql/turso), not local.
|
|
8
|
-
|
|
9
|
-
## Tables (drizzle, root file `web/db.js` or `schema.js`)
|
|
10
|
-
|
|
11
|
-
```js
|
|
12
|
-
import { sql } from "drizzle-orm";
|
|
13
|
-
import { mysqlTable, text, int, float, boolean, index, timestamp } from "drizzle-orm/mysql-core";
|
|
14
|
-
|
|
15
|
-
// session of computer use (one agent / one execution)
|
|
16
|
-
export const sessions = mysqlTable("sessions", {
|
|
17
|
-
id: text("id").primaryKey(),
|
|
18
|
-
agentName: text("agent_name"),
|
|
19
|
-
browser: text("browser").default("brave"),
|
|
20
|
-
originUrl: text("origin_url"),
|
|
21
|
-
status: text("status").default("running"),
|
|
22
|
-
seed: text("seed"),
|
|
23
|
-
createdAt: timestamp("created_at").defaultNow(),
|
|
24
|
-
finishedAt: timestamp("finished_at"),
|
|
25
|
-
}, (t) => [index("idx_sessions_status").on(t.status)]);
|
|
26
|
-
|
|
27
|
-
// captured movement events (virtual mouse, click, scroll, etc)
|
|
28
|
-
export const events = mysqlTable("events", {
|
|
29
|
-
id: int("id").autoincrement().primaryKey(),
|
|
30
|
-
sessionId: text("session_id").references(() => sessions.id, { onDelete: "cascade" }),
|
|
31
|
-
t: float("t"),
|
|
32
|
-
type: text("type"),
|
|
33
|
-
x: float("x"),
|
|
34
|
-
y: float("y"),
|
|
35
|
-
data: text("data"),
|
|
36
|
-
createdAt: timestamp("created_at").defaultNow(),
|
|
37
|
-
}, (t) => [index("idx_events_session").on(t.sessionId)]);
|
|
38
|
-
|
|
39
|
-
// captcha results (tests)
|
|
40
|
-
export const captchaResults = mysqlTable("captcha_results", {
|
|
41
|
-
id: int("id").autoincrement().primaryKey(),
|
|
42
|
-
sessionId: text("session_id").references(() => sessions.id, { onDelete: "cascade" }),
|
|
43
|
-
kind: text("kind"),
|
|
44
|
-
sitekey: text("sitekey"),
|
|
45
|
-
token: text("token"),
|
|
46
|
-
passed: boolean("passed").default(false),
|
|
47
|
-
solver: text("solver"),
|
|
48
|
-
evidenceUrl: text("evidence_url"),
|
|
49
|
-
createdAt: timestamp("created_at").defaultNow(),
|
|
50
|
-
});
|
|
51
|
-
|
|
52
|
-
// heavy assets (replay videos, screenshots) — pointer to cloudinary
|
|
53
|
-
export const assets = mysqlTable("assets", {
|
|
54
|
-
id: int("id").autoincrement().primaryKey(),
|
|
55
|
-
sessionId: text("session_id").references(() => sessions.id, { onDelete: "cascade" }),
|
|
56
|
-
kind: text("kind"),
|
|
57
|
-
cloudinaryId: text("cloudinary_id"),
|
|
58
|
-
url: text("url"),
|
|
59
|
-
bytes: int("bytes"),
|
|
60
|
-
createdAt: timestamp("created_at").defaultNow(),
|
|
61
|
-
});
|
|
62
|
-
```
|
|
63
|
-
|
|
64
|
-
## Prisma (`schema.prisma`) — equivalent
|
|
65
|
-
|
|
66
|
-
```prisma
|
|
67
|
-
model Session {
|
|
68
|
-
id String @id
|
|
69
|
-
agentName String?
|
|
70
|
-
browser String @default("brave")
|
|
71
|
-
originUrl String?
|
|
72
|
-
status String @default("running")
|
|
73
|
-
seed String?
|
|
74
|
-
createdAt DateTime @default(now())
|
|
75
|
-
finishedAt DateTime?
|
|
76
|
-
events Event[]
|
|
77
|
-
captchaResults CaptchaResult[]
|
|
78
|
-
assets Asset[]
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
model Event {
|
|
82
|
-
id Int @id @default(autoincrement())
|
|
83
|
-
session Session @relation(fields: [sessionId], references: [id], onDelete: Cascade)
|
|
84
|
-
sessionId String
|
|
85
|
-
t Float
|
|
86
|
-
type String
|
|
87
|
-
x Float?
|
|
88
|
-
y Float?
|
|
89
|
-
data String?
|
|
90
|
-
createdAt DateTime @default(now())
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
model CaptchaResult {
|
|
94
|
-
id Int @id @default(autoincrement())
|
|
95
|
-
session Session @relation(fields: [sessionId], references: [id], onDelete: Cascade)
|
|
96
|
-
sessionId String
|
|
97
|
-
kind String
|
|
98
|
-
sitekey String?
|
|
99
|
-
token String?
|
|
100
|
-
passed Boolean @default(false)
|
|
101
|
-
solver String?
|
|
102
|
-
evidenceUrl String?
|
|
103
|
-
createdAt DateTime @default(now())
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
model Asset {
|
|
107
|
-
id Int @id @default(autoincrement())
|
|
108
|
-
session Session @relation(fields: [sessionId], references: [id], onDelete: Cascade)
|
|
109
|
-
sessionId String
|
|
110
|
-
kind String
|
|
111
|
-
cloudinaryId String
|
|
112
|
-
url String
|
|
113
|
-
bytes Int
|
|
114
|
-
createdAt DateTime @default(now())
|
|
115
|
-
}
|
|
116
|
-
```
|
|
117
|
-
|
|
118
|
-
## Migrations
|
|
119
|
-
- Drizzle: `drizzle-kit generate` + `drizzle-kit migrate` (mysql) in the deploy.
|
|
120
|
-
- Prisma: `prisma migrate deploy` (mysql) as alt.
|
|
121
|
-
- The db is created in the deploy pipeline (vercel/netlify), not local sqlite.
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
# 10 Cloudinary Storage
|
|
2
|
-
|
|
3
|
-
## Why cloudinary
|
|
4
|
-
The production site (`web/`) receives agent data via https. Heavy data (replay videos, screenshots, big json logs) is sent to cloudinary instead of the db, keeping the db light (only pointers/urls).
|
|
5
|
-
|
|
6
|
-
## What goes to cloudinary
|
|
7
|
-
- Replay videos (`.mp4`/`.webm`) generated via `ffmpeg-static` from logs.
|
|
8
|
-
- Captcha challenge screenshots (before/after).
|
|
9
|
-
- Big json logs (long trajectories) as a complementary asset.
|
|
10
|
-
- Training assets (click masks, challenge images).
|
|
11
|
-
|
|
12
|
-
## Flow
|
|
13
|
-
|
|
14
|
-
```
|
|
15
|
-
agent/sandbox ──post /captcha/result──► backend (web/)
|
|
16
|
-
│
|
|
17
|
-
├─ write metadata to db (drizzle/mysql2)
|
|
18
|
-
└─ upload file ──► cloudinary
|
|
19
|
-
│
|
|
20
|
-
└─ save asset { url, cloudinaryId } in db
|
|
21
|
-
```
|
|
22
|
-
|
|
23
|
-
## Config (`web/storage.js`)
|
|
24
|
-
|
|
25
|
-
```js
|
|
26
|
-
import { v2 as cloudinary } from "cloudinary";
|
|
27
|
-
|
|
28
|
-
cloudinary.config({
|
|
29
|
-
cloud_name: process.env.CLOUDINARY_CLOUD_NAME,
|
|
30
|
-
api_key: process.env.CLOUDINARY_API_KEY,
|
|
31
|
-
api_secret: process.env.CLOUDINARY_API_SECRET,
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* Uploads a local file to cloudinary under uka/<folder>.
|
|
36
|
-
* @param {string} filePath local path
|
|
37
|
-
* @param {string} folder sub folder
|
|
38
|
-
* @returns {Promise<object>} upload result with url and public id
|
|
39
|
-
*/
|
|
40
|
-
export async function uploadAsset(filePath, folder) {
|
|
41
|
-
return cloudinary.uploader.upload(filePath, {
|
|
42
|
-
resource_type: "auto",
|
|
43
|
-
folder: `uka/${folder}`,
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
|
-
```
|
|
47
|
-
|
|
48
|
-
## Variables
|
|
49
|
-
- `CLOUDINARY_CLOUD_NAME`
|
|
50
|
-
- `CLOUDINARY_API_KEY`
|
|
51
|
-
- `CLOUDINARY_API_SECRET`
|
|
52
|
-
- `CLOUDINARY_URL` (condensed form, optional)
|
|
53
|
-
|
|
54
|
-
## Good practices
|
|
55
|
-
- Use `folder: "uka/<sessionId>"` for organization.
|
|
56
|
-
- Save only the `url` + `cloudinaryId` in the db (see `09-database-schema.md`, `assets` table).
|
|
57
|
-
- Compress videos before upload to respect plan limits.
|