@wenathlan/saddle 1.8.2 → 1.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/browser/playwright.js +22 -0
- package/docs/.gitkeep +0 -0
- package/docs/gapmatrix.md +3 -3
- package/docs/libraryapi.md +1 -0
- package/docs/logs/.gitkeep +0 -0
- package/docs/packageaudit185.md +23 -0
- package/docs/plans/00.index.md +50 -0
- package/docs/plans/01.architecture.md +86 -0
- package/docs/plans/02.research.computer.use.md +58 -0
- package/docs/plans/03.research.captcha.bypass.md +68 -0
- package/docs/plans/04.research.sandbox.ai.md +52 -0
- package/docs/plans/05.capture.platform.md +57 -0
- package/docs/plans/06.dependencies.md +97 -0
- package/docs/plans/07.captcha.test.page.md +41 -0
- package/docs/plans/08.production.infra.md +70 -0
- package/docs/plans/09.database.schema.md +121 -0
- package/docs/plans/10.cloudinary.storage.md +57 -0
- package/docs/plans/11.movement.logs.json.md +72 -0
- package/docs/plans/12.research.atlas.agent.browser.md +79 -0
- package/docs/plans/13.research.anti.detection.md +898 -0
- package/docs/plans/14.research.proxy.md +1495 -0
- package/docs/plans/15.research.retry.rate.limit.md +1958 -0
- package/docs/plans/16.research.crawling.md +1417 -0
- package/docs/plans/17.research.caching.md +1610 -0
- package/docs/plans/18.research.content.extraction.md +1952 -0
- package/docs/plans/19.research.errors.events.md +1523 -0
- package/docs/plans/20.research.zod.validation.md +1350 -0
- package/docs/plans/21.research.batch.concurrency.md +1888 -0
- package/docs/plans/22.research.universal.runtime.md +944 -0
- package/docs/plans/23.research.ai.integration.md +1465 -0
- package/docs/plans/24.research.memory.persistence.md +1979 -0
- package/docs/plans/25.research.server.api.md +342 -0
- package/docs/plans/26.research.compilation.md +249 -0
- package/docs/plans/27.research.html.parsing.md +251 -0
- package/docs/plans/28.action.plan.md +50 -0
- package/docs/plans/29.api.reference.md +174 -0
- package/docs/plans/30.architecture.plan.md +94 -0
- package/docs/plans/31.auditoria.dados.md +163 -0
- package/docs/plans/32.bots.automacao.computacional.md +214 -0
- package/docs/plans/33.bots.codigo.revisao.md +220 -0
- package/docs/plans/34.bots.seguranca.cicd.md +366 -0
- package/docs/plans/35.comparativo.concorrencia.md +464 -0
- package/docs/plans/36.computational.memory.md +340 -0
- package/docs/plans/37.deploystrategy.md +394 -0
- package/docs/plans/38.flow.md +155 -0
- package/docs/plans/39.multi.platform.bot.md +252 -0
- package/docs/plans/40.npm.publish.md +250 -0
- package/docs/plans/41.o.que.falta.md +407 -0
- package/docs/plans/42.pesquisa.concorrencia.md +721 -0
- package/docs/plans/43.plan.universal.architecture.md +496 -0
- package/docs/plans/44.reference.md +100 -0
- package/docs/plans/45.robotarchitecture.md +237 -0
- package/docs/plans/46.scdnintegration.md +284 -0
- package/docs/plans/47.multiforge.readme.md +129 -0
- package/docs/plans/48.theory.v4.repo.os.md +152 -0
- package/docs/plans/49.third.party.infra.md +12 -0
- package/docs/plans/50.file.as.compute.md +39 -0
- package/docs/plans/51.architecture.virtual.processor.md +80 -0
- package/docs/plans/52.manifesto.v8.md +11 -0
- package/docs/plans/58.cdn.list.md +23 -0
- package/docs/plans/59.sql.frameworks.md +33 -0
- package/docs/plans/60.sql.thirdparty.md +26 -0
- package/docs/plans/61.objective.multiforge.md +63 -0
- package/docs/plans/62.huggingface.upload.md +26 -0
- package/docs/plans/63.kaggle.upload.md +24 -0
- package/docs/plans/64.npm.storage.md +30 -0
- package/docs/plans/65.rclone.terabox.md +32 -0
- package/docs/plans/66.buckets.and.models.todo.md +14 -0
- package/docs/plans/67.database.todo.md +13 -0
- package/docs/plans/68.deploy.packages.todo.md +12 -0
- package/docs/plans/69.report.human.operator.md +133 -0
- package/docs/plans/70.report.brain2qwerty.ems.md +135 -0
- package/docs/plans/71.report.hd.infinito.vram.md +155 -0
- package/docs/plans/72.plan.hd.infinito.node.md +146 -0
- package/docs/plans/73.plan.scifi.repos.md +125 -0
- package/docs/plans/74.000.manifesto.v8.flat.2..md +11 -0
- package/docs/plans/README.md +489 -0
- package/docs/plans/aggregate_platforms.mjs +146 -0
- package/docs/plans/examplesession.json +36 -0
- package/docs/plans/missing-facts.md +192 -0
- package/docs/plans/models.md +64 -0
- package/docs/plans/organize.cjs +270 -0
- package/docs/plans/platforms.md +2887 -0
- package/docs/plans/sites.md +31322 -0
- package/docs/platformpipelineaudit.md +18 -0
- package/docs/platformpipelines.md +13 -0
- package/docs/registryresearch.md +4 -0
- package/docs/release.md +4 -4
- package/docs/release182notes.md +2 -0
- package/docs/release184notes.md +7 -0
- package/docs/release185notes.md +7 -0
- package/docs/sources/farm.py +117 -0
- package/docs/sources/html/saddle1.html +132 -0
- package/docs/sources/html/saddle2.html +157 -0
- package/docs/sources/html/saddle3.html +119 -0
- package/docs/sources/html/saddle4.html +144 -0
- package/docs/sources/html/saddle5.html +72 -0
- package/docs/sources/html/saddle6.html +171 -0
- package/docs/sources/html/saddle7.html +236 -0
- package/docs/sources/saddle.ts +74 -0
- package/docs/sources/schema.prisma +88 -0
- package/docs/sources/script.sh +64 -0
- package/docs/sources/workflows.yml +458 -0
- package/docs/talks1/_body.txt +14 -0
- package/docs/talks1/_index.md +15 -0
- package/docs/talks1/_screenshot.png +0 -0
- package/docs/talks1/assistant-01.md +5 -0
- package/docs/talks1/assistant-02.md +5 -0
- package/docs/talks1/assistant-03.md +531 -0
- package/docs/talks1/assistant-04.md +26 -0
- package/docs/talks1/assistant-05.md +774 -0
- package/docs/talks1/assistant-06.md +1718 -0
- package/docs/talks1/scrape-share.cjs +185 -0
- package/docs/talks1/scrape-share.ts +183 -0
- package/docs/talks1/user-01.md +3 -0
- package/docs/talks1/user-02.md +3 -0
- package/docs/talks1/user-03.md +88 -0
- package/docs/talks1/user-04.md +3 -0
- package/docs/talks1/user-05.md +3 -0
- package/docs/talks1/user-06.md +88 -0
- package/docs/talks1/user-07.md +88 -0
- package/docs/talks2/_body.txt +14 -0
- package/docs/talks2/_index.md +16 -0
- package/docs/talks2/_screenshot.png +0 -0
- package/docs/talks2/assistant-01.md +5 -0
- package/docs/talks2/assistant-02.md +5 -0
- package/docs/talks2/assistant-03.md +424 -0
- package/docs/talks2/assistant-04.md +598 -0
- package/docs/talks2/assistant-05.md +1280 -0
- package/docs/talks2/assistant-06.md +1227 -0
- package/docs/talks2/assistant-07.md +1252 -0
- package/docs/talks2/user-01.md +3 -0
- package/docs/talks2/user-02.md +3 -0
- package/docs/talks2/user-03.md +88 -0
- package/docs/talks2/user-04.md +88 -0
- package/docs/talks2/user-05.md +88 -0
- package/docs/talks2/user-06.md +88 -0
- package/docs/talks2/user-07.md +3 -0
- package/docs/talks3/_body.txt +467 -0
- package/docs/talks3/_index.md +10 -0
- package/docs/talks3/_screenshot.png +0 -0
- package/docs/talks3/assistant-01.md +417 -0
- package/docs/talks3/assistant-02.md +417 -0
- package/docs/talks3/assistant-03.md +29 -0
- package/docs/talks3/assistant-04.md +727 -0
- package/docs/talks3/user-01.md +88 -0
- package/docs/talks3/user-02.md +88 -0
- package/docs/talks3/user-03.md +3 -0
- package/docs/talks3/user-04.md +3 -0
- package/docs/talks4/_body.txt +14 -0
- package/docs/talks4/_index.md +12 -0
- package/docs/talks4/_screenshot.png +0 -0
- package/docs/talks4/assistant-01.md +5 -0
- package/docs/talks4/assistant-02.md +5 -0
- package/docs/talks4/assistant-03.md +35 -0
- package/docs/talks4/assistant-04.md +512 -0
- package/docs/talks4/assistant-05.md +599 -0
- package/docs/talks4/user-01.md +3 -0
- package/docs/talks4/user-02.md +3 -0
- package/docs/talks4/user-03.md +88 -0
- package/docs/talks4/user-04.md +88 -0
- package/docs/talks4/user-05.md +7 -0
- package/docs/talks5/_body.txt +14 -0
- package/docs/talks5/_index.md +13 -0
- package/docs/talks5/_screenshot.png +0 -0
- package/docs/talks5/assistant-01.md +5 -0
- package/docs/talks5/assistant-02.md +5 -0
- package/docs/talks5/assistant-03.md +690 -0
- package/docs/talks5/assistant-04.md +758 -0
- package/docs/talks5/assistant-05.md +974 -0
- package/docs/talks5/user-01.md +3 -0
- package/docs/talks5/user-02.md +3 -0
- package/docs/talks5/user-03.md +105 -0
- package/docs/talks5/user-04.md +105 -0
- package/docs/talks5/user-05.md +63 -0
- package/docs/talks5/user-06.md +105 -0
- package/docs/talks6/_body.txt +14 -0
- package/docs/talks6/_index.md +9 -0
- package/docs/talks6/_screenshot.png +0 -0
- package/docs/talks6/assistant-01.md +5 -0
- package/docs/talks6/assistant-02.md +5 -0
- package/docs/talks6/assistant-03.md +1499 -0
- package/docs/talks6/user-01.md +3 -0
- package/docs/talks6/user-02.md +3 -0
- package/docs/talks6/user-03.md +88 -0
- package/docs/talks6/user-04.md +88 -0
- package/docs/talks7/_body.txt +14 -0
- package/docs/talks7/_index.md +10 -0
- package/docs/talks7/_screenshot.png +0 -0
- package/docs/talks7/assistant-01.md +5 -0
- package/docs/talks7/assistant-02.md +5 -0
- package/docs/talks7/assistant-03.md +523 -0
- package/docs/talks7/assistant-04.md +617 -0
- package/docs/talks7/user-01.md +3 -0
- package/docs/talks7/user-02.md +3 -0
- package/docs/talks7/user-03.md +105 -0
- package/docs/talks7/user-04.md +67 -0
- package/docs/talks8/conversa1.txt +1322 -0
- package/docs/talks8/conversa2.txt +237 -0
- package/docs/talks9/Beyond the Obvious_ 50 Plataformas Auto-Hospedadas de Forja de C/303/263digo para Al/303/251m de Gitea e GitLab.md" +174 -0
- package/docs/talks9/De NPM a Multi-Linguagem_ Uma Arquitetura T/303/251cnica para a Execu/303/247/303/243o Integrada de C/303/263digo no Ecossistema Node.js.md" +59 -0
- package/docs/talks9/De NPM a VMs Virtuais_ Uma An/303/241lise Arquitet/303/264nica para a Realiza/303/247/303/243o do Ciclo de Vida do Projeto SADDLE.md" +91 -0
- package/docs/talks9/Mapeamento da Engrenagem Computacional_ Uma Arquitetura para Execu/303/247/303/243o Isolada e Persist/303/252ncia em Ambientes Distribu/303/255dos.md" +116 -0
- package/docs/talks9/O Cen/303/241rio Pr/303/241tico do SADDLE_ Uma An/303/241lise de Viabilidade e Modelo de Ciclo de Vida Integrado.md" +128 -0
- package/docs/talks9/README (2).md +489 -0
- package/docs/talks9/README.md +198 -0
- package/docs/talks9/Viabilidade do Saddle_ Uma An/303/241lise T/303/251cnica da Transforma/303/247/303/243o de Armazenamento Remoto em Mem/303/263ria Computacional.md" +80 -0
- package/docs/talks9/conversa.txt +544 -0
- package/docs/talks9/other (2).md +39 -0
- package/docs/talks9/other.md +57 -0
- package/docs/talks9/outro.txt +24 -0
- package/extension/README.md +3 -3
- package/extension/serviceworker.js +63 -4
- package/extension/worker.js +3 -1
- package/package.json +12 -2
- package/packager/manifest.js +1 -1
- package/scrape/agent.ts +122 -0
- package/scrape/batch.ts +79 -0
- package/scrape/biome.json +76 -0
- package/scrape/browser.ts +222 -0
- package/scrape/cache.ts +84 -0
- package/scrape/chunking.ts +193 -0
- package/scrape/cli.ts +105 -0
- package/scrape/crawler.ts +115 -0
- package/scrape/dev-server.ts +94 -0
- package/scrape/errors.ts +132 -0
- package/scrape/events.ts +26 -0
- package/scrape/extract.ts +165 -0
- package/scrape/fetch.ts +105 -0
- package/scrape/formats.ts +85 -0
- package/scrape/headers.ts +71 -0
- package/scrape/index.ts +92 -0
- package/scrape/jsdom.d.ts +6 -0
- package/scrape/llms-txt.ts +84 -0
- package/scrape/middleware.ts +90 -0
- package/scrape/package-lock.json +9397 -0
- package/scrape/package.json +1420 -0
- package/scrape/pool.ts +95 -0
- package/scrape/port.ts +18 -0
- package/scrape/proxy.ts +103 -0
- package/scrape/rate-limiter.ts +95 -0
- package/scrape/renderer.ts +194 -0
- package/scrape/retry.ts +64 -0
- package/scrape/robots.ts +137 -0
- package/scrape/scrape.ts +123 -0
- package/scrape/serialize.ts +310 -0
- package/scrape/server.ts +137 -0
- package/scrape/session.ts +109 -0
- package/scrape/sitemap.ts +131 -0
- package/scrape/tokens.ts +45 -0
- package/scrape/tsconfig.json +28 -0
- package/scrape/types.ts +214 -0
- package/scrape/utils.ts +77 -0
- package/scrape/vite.config.ts +55 -0
- package/scrape/vitest.config.ts +17 -0
- package/workflow/templates.js +4 -4
|
@@ -0,0 +1,944 @@
|
|
|
1
|
+
# Pesquisa: Universal JavaScript Cross-Runtime (2026)
|
|
2
|
+
|
|
3
|
+
> Como escrever código JavaScript/TypeScript que roda em Node.js, Deno, Bun, browsers e edge runtimes sem dependências de runtime específico.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 1. Fundamento: WinterTC Minimum Common Web Platform API (ECMA-429)
|
|
8
|
+
|
|
9
|
+
O **WinterTC** (anteriormente WinterCG) definiu o **Minimum Common Web Platform API** — um conjunto mínimo de APIs que todos os runtimes devem suportar. O objetivo é que o mesmo código funcione em qualquer lugar.
|
|
10
|
+
|
|
11
|
+
### APIs incluídas no mínimo comum
|
|
12
|
+
|
|
13
|
+
| API | Descrição | Status |
|
|
14
|
+
|-----|-----------|--------|
|
|
15
|
+
| `fetch()` | HTTP client | ✅ Universal |
|
|
16
|
+
| `Request` | Request object | ✅ Universal |
|
|
17
|
+
| `Response` | Response object | ✅ Universal |
|
|
18
|
+
| `Headers` | HTTP headers | ✅ Universal |
|
|
19
|
+
| `URL` | URL parsing | ✅ Universal |
|
|
20
|
+
| `URLPattern` | URL matching | ✅ Universal (quase todos) |
|
|
21
|
+
| `crypto.subtle` | Web Crypto API | ✅ Universal |
|
|
22
|
+
| `ReadableStream` | Streams | ✅ Universal |
|
|
23
|
+
| `WritableStream` | Streams | ✅ Universal |
|
|
24
|
+
| `TransformStream` | Streams | ✅ Universal |
|
|
25
|
+
| `TextEncoder` | UTF-8 encoding | ✅ Universal |
|
|
26
|
+
| `TextDecoder` | UTF-8 decoding | ✅ Universal |
|
|
27
|
+
| `structuredClone` | Deep cloning | ✅ Universal |
|
|
28
|
+
| `setTimeout` / `setInterval` | Timers | ✅ Universal |
|
|
29
|
+
| `queueMicrotask` | Microtasks | ✅ Universal |
|
|
30
|
+
| `AbortController` | Abort signals | ✅ Universal |
|
|
31
|
+
| `console` | Logging | ✅ Universal |
|
|
32
|
+
| `atob` / `btoa` | Base64 | ✅ Universal |
|
|
33
|
+
| `performance.now()` | Timing | ✅ Universal |
|
|
34
|
+
|
|
35
|
+
### APIs que NÃO são universais (evitar)
|
|
36
|
+
|
|
37
|
+
| API | Disponível em | Evitar se |
|
|
38
|
+
|-----|---------------|-----------|
|
|
39
|
+
| `fs` | Node/Bun/Deno | Precisa de polyfill ou adapter |
|
|
40
|
+
| `process` | Node/Bun/Deno | Não existe em browsers |
|
|
41
|
+
| `Buffer` | Node/Bun/Deno | Usar `ArrayBuffer` / `Uint8Array` |
|
|
42
|
+
| `path` | Node/Bun/Deno | Usar `URL` para paths |
|
|
43
|
+
| `child_process` | Node/Bun/Deno | Não existe em browsers |
|
|
44
|
+
| `require()` | Node/Bun | Usar `import` |
|
|
45
|
+
| `__dirname` | Node/Bun | Usar `import.meta.url` |
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
## 2. Tabela de Compliance dos Runtimes (2026)
|
|
50
|
+
|
|
51
|
+
| API | Node.js 24+ | Deno 2.x | Bun 1.x+ | Cloudflare Workers | Vercel Edge | Browsers |
|
|
52
|
+
|-----|-------------|----------|----------|-------------------|-------------|----------|
|
|
53
|
+
| `fetch` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
54
|
+
| `Request/Response/Headers` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
55
|
+
| `URL` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
56
|
+
| `URLPattern` | ✅ (v24) | ✅ | ✅ | ✅ | ✅ | ⚠️ Chrome/Edge |
|
|
57
|
+
| `crypto.subtle` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
58
|
+
| `ReadableStream` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
59
|
+
| `WritableStream` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
60
|
+
| `TransformStream` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
61
|
+
| `TextEncoder/Decoder` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
62
|
+
| `structuredClone` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
63
|
+
| `AbortController` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
64
|
+
| `atob/btoa` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
65
|
+
| `performance.now()` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
66
|
+
| `WebSocket` | ✅ | ✅ | ✅ | ✅ | ⚠️ | ✅ |
|
|
67
|
+
| `navigator` | ⚠️ parcial | ✅ | ⚠️ | ⚠️ | ⚠️ | ✅ |
|
|
68
|
+
|
|
69
|
+
> **Regra de ouro:** Se está na tabela do WinterTC, pode usar. Caso contrário, precisa de adapter.
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## 3. Padrão Universal JavaScript
|
|
74
|
+
|
|
75
|
+
A ideia central: **escreva para a API mínima comum, use adapters para o resto.**
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
src/
|
|
79
|
+
├── core/ # Lógica universal (100% WinterTC)
|
|
80
|
+
│ ├── scraper.ts # usa fetch, Response, TextEncoder
|
|
81
|
+
│ ├── parser.ts # usa URL, URLPattern
|
|
82
|
+
│ ├── crypto.ts # usa crypto.subtle
|
|
83
|
+
│ ├── streams.ts # usa ReadableStream, TransformStream
|
|
84
|
+
│ └── encoding.ts # usa TextEncoder, TextDecoder
|
|
85
|
+
├── adapters/ # Plataforma específica
|
|
86
|
+
│ ├── node.ts # node:fs, node:path
|
|
87
|
+
│ ├── deno.ts # Deno.open, Deno.stat
|
|
88
|
+
│ ├── bun.ts # Bun.file, Bun.write
|
|
89
|
+
│ ├── browser.ts # DOM APIs
|
|
90
|
+
│ └── cloudflare.ts # KV, Durable Objects
|
|
91
|
+
├── index.ts # Entry point universal
|
|
92
|
+
└── index.node.ts # Entry point com Node adapter
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Exemplo: Scraper universal
|
|
96
|
+
|
|
97
|
+
```typescript
|
|
98
|
+
// src/core/scraper.ts — 100% WinterTC compliant
|
|
99
|
+
import type { ScrapeResult } from './types.ts';
|
|
100
|
+
|
|
101
|
+
export async function scrape(url: string): Promise<ScrapeResult> {
|
|
102
|
+
// fetch, Request, Response, Headers — todos universais
|
|
103
|
+
const response = await fetch(url, {
|
|
104
|
+
headers: { 'User-Agent': 'UniversalScraper/1.0' },
|
|
105
|
+
signal: AbortSignal.timeout(30_000),
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
if (!response.ok) {
|
|
109
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const html = await response.text(); // Response.text() universal
|
|
113
|
+
return { url, html, status: response.status };
|
|
114
|
+
}
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
```typescript
|
|
118
|
+
// src/adapters/node.ts — Node.js específico
|
|
119
|
+
import { readFile, writeFile, mkdir } from 'node:fs/promises';
|
|
120
|
+
import { join, dirname } from 'node:path';
|
|
121
|
+
|
|
122
|
+
export async function saveToFile(path: string, content: string): Promise<void> {
|
|
123
|
+
await mkdir(dirname(path), { recursive: true });
|
|
124
|
+
await writeFile(path, content, 'utf-8');
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export async function loadFromFile(path: string): Promise<string> {
|
|
128
|
+
return readFile(path, 'utf-8');
|
|
129
|
+
}
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
```typescript
|
|
133
|
+
// src/adapters/deno.ts — Deno específico
|
|
134
|
+
import { ensureDir } from 'https://deno.land/std@0.224.0/fs/mod.ts';
|
|
135
|
+
|
|
136
|
+
export async function saveToFile(path: string, content: string): Promise<void> {
|
|
137
|
+
await ensureDir(dirname(path));
|
|
138
|
+
await Deno.writeTextFile(path, content);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export async function loadFromFile(path: string): Promise<string> {
|
|
142
|
+
return Deno.readTextFile(path);
|
|
143
|
+
}
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
```typescript
|
|
147
|
+
// src/adapters/browser.ts — Browser específico
|
|
148
|
+
export async function saveToFile(path: string, content: string): Promise<void> {
|
|
149
|
+
// IndexedDB, localStorage, ou download
|
|
150
|
+
const blob = new Blob([content], { type: 'text/plain' });
|
|
151
|
+
const a = document.createElement('a');
|
|
152
|
+
a.href = URL.createObjectURL(blob);
|
|
153
|
+
a.download = path;
|
|
154
|
+
a.click();
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export async function loadFromFile(path: string): Promise<string> {
|
|
158
|
+
// Via fetch local ou FileReader
|
|
159
|
+
const response = await fetch(path);
|
|
160
|
+
return response.text();
|
|
161
|
+
}
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
---
|
|
165
|
+
|
|
166
|
+
## 4. Imports Condicionais
|
|
167
|
+
|
|
168
|
+
### 4.1. dynamic import com try-catch (funcional, não preferido)
|
|
169
|
+
|
|
170
|
+
```typescript
|
|
171
|
+
let fs: typeof import('node:fs/promises');
|
|
172
|
+
|
|
173
|
+
try {
|
|
174
|
+
fs = await import('node:fs/promises');
|
|
175
|
+
} catch {
|
|
176
|
+
// Fallback para runtime sem node:fs
|
|
177
|
+
fs = null;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
export async function readFile(path: string): Promise<string | null> {
|
|
181
|
+
if (!fs) return null;
|
|
182
|
+
return fs.readFile(path, 'utf-8');
|
|
183
|
+
}
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
**Problema:** Não funciona bem com tree-shaking. O bundle inclui o polyfill mesmo que não necessário.
|
|
187
|
+
|
|
188
|
+
### 4.2. Conditional exports no package.json (recomendado)
|
|
189
|
+
|
|
190
|
+
```json
|
|
191
|
+
{
|
|
192
|
+
"name": "@devthink/webscrape",
|
|
193
|
+
"exports": {
|
|
194
|
+
".": {
|
|
195
|
+
"types": "./dist/index.d.ts",
|
|
196
|
+
"import": "./dist/index.js",
|
|
197
|
+
"default": "./dist/index.js"
|
|
198
|
+
},
|
|
199
|
+
"./node": {
|
|
200
|
+
"types": "./dist/node.d.ts",
|
|
201
|
+
"import": "./dist/node.js"
|
|
202
|
+
},
|
|
203
|
+
"./browser": {
|
|
204
|
+
"types": "./dist/browser.d.ts",
|
|
205
|
+
"import": "./dist/browser.js"
|
|
206
|
+
},
|
|
207
|
+
"./deno": {
|
|
208
|
+
"types": "./dist/deno.d.ts",
|
|
209
|
+
"import": "./dist/deno.js"
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
**O consumidor importa:**
|
|
216
|
+
|
|
217
|
+
```typescript
|
|
218
|
+
// Node.js
|
|
219
|
+
import { scrape } from '@devthink/webscrape/node';
|
|
220
|
+
|
|
221
|
+
// Browser
|
|
222
|
+
import { scrape } from '@devthink/webscrape/browser';
|
|
223
|
+
|
|
224
|
+
// Deno
|
|
225
|
+
import { scrape } from '@devthink/webscrape/deno';
|
|
226
|
+
|
|
227
|
+
// Universal (só APIs comuns)
|
|
228
|
+
import { scrape } from '@devthink/webscrape';
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
### 4.3. unenv polyfills
|
|
232
|
+
|
|
233
|
+
O [unenv](https://github.com/unjs/unenv) mapeia APIs de Node para implementações universais:
|
|
234
|
+
|
|
235
|
+
```typescript
|
|
236
|
+
// unenv transforma:
|
|
237
|
+
import { readFile } from 'node:fs/promises';
|
|
238
|
+
// Em:
|
|
239
|
+
import { readFile } from 'unenv/node/fs';
|
|
240
|
+
// Que usa implementations in-memory ou fetch-based
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
**Uso com build tools:**
|
|
244
|
+
|
|
245
|
+
```typescript
|
|
246
|
+
// vite.config.ts
|
|
247
|
+
import { nodePolyfills } from 'unenv';
|
|
248
|
+
|
|
249
|
+
export default {
|
|
250
|
+
resolve: {
|
|
251
|
+
alias: nodePolyfills(),
|
|
252
|
+
},
|
|
253
|
+
};
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
```typescript
|
|
257
|
+
// tsup.config.ts
|
|
258
|
+
import { defineConfig } from 'tsup';
|
|
259
|
+
|
|
260
|
+
export default defineConfig({
|
|
261
|
+
// unenv automaticamente polyfills node: imports
|
|
262
|
+
noExternal: [],
|
|
263
|
+
// ou usar unenv manualmente
|
|
264
|
+
});
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
---
|
|
268
|
+
|
|
269
|
+
## 5. node: Protocol Imports e Runtime Detection
|
|
270
|
+
|
|
271
|
+
### 5.1. Por que usar `node:` protocol?
|
|
272
|
+
|
|
273
|
+
```typescript
|
|
274
|
+
// ❌ Antes (ambíguo, pode colidir com packages npm)
|
|
275
|
+
import { readFile } from 'fs';
|
|
276
|
+
|
|
277
|
+
// ✅ Depois (explícito, sempre o built-in do Node)
|
|
278
|
+
import { readFile } from 'node:fs/promises';
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
### 5.2. Ordem de detecção de runtime
|
|
282
|
+
|
|
283
|
+
```typescript
|
|
284
|
+
type Runtime = 'deno' | 'bun' | 'node' | 'browser' | 'unknown';
|
|
285
|
+
|
|
286
|
+
function detectRuntime(): Runtime {
|
|
287
|
+
// Deno primeiro (tem globalThis.Deno)
|
|
288
|
+
if (typeof globalThis.Deno !== 'undefined') return 'deno';
|
|
289
|
+
|
|
290
|
+
// Bun segundo (tem globalThis.Bun)
|
|
291
|
+
if (typeof globalThis.Bun !== 'undefined') return 'bun';
|
|
292
|
+
|
|
293
|
+
// Node terceiro (tem globalThis.process com versions.node)
|
|
294
|
+
if (
|
|
295
|
+
typeof globalThis.process !== 'undefined' &&
|
|
296
|
+
typeof globalThis.process.versions?.node !== 'undefined'
|
|
297
|
+
) {
|
|
298
|
+
return 'node';
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// Browser quarto (tem window, document)
|
|
302
|
+
if (typeof window !== 'undefined' && typeof document !== 'undefined') {
|
|
303
|
+
return 'browser';
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
return 'unknown';
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// Factory pattern baseado no runtime
|
|
310
|
+
function createStorage(runtime: Runtime) {
|
|
311
|
+
switch (runtime) {
|
|
312
|
+
case 'deno':
|
|
313
|
+
return import('./adapters/deno.ts');
|
|
314
|
+
case 'bun':
|
|
315
|
+
return import('./adapters/bun.ts');
|
|
316
|
+
case 'node':
|
|
317
|
+
return import('./adapters/node.ts');
|
|
318
|
+
case 'browser':
|
|
319
|
+
return import('./adapters/browser.ts');
|
|
320
|
+
default:
|
|
321
|
+
return import('./adapters/memory.ts');
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// Uso
|
|
326
|
+
const runtime = detectRuntime();
|
|
327
|
+
const storage = await createStorage(runtime);
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
### 5.3. Detecção granular com versões
|
|
331
|
+
|
|
332
|
+
```typescript
|
|
333
|
+
function getRuntimeInfo() {
|
|
334
|
+
const runtime = detectRuntime();
|
|
335
|
+
|
|
336
|
+
if (runtime === 'deno') {
|
|
337
|
+
return {
|
|
338
|
+
name: 'deno',
|
|
339
|
+
version: globalThis.Deno.version.deno,
|
|
340
|
+
v8: globalThis.Deno.version.v8,
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
if (runtime === 'bun') {
|
|
345
|
+
return {
|
|
346
|
+
name: 'bun',
|
|
347
|
+
version: globalThis.Bun.version,
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
if (runtime === 'node') {
|
|
352
|
+
return {
|
|
353
|
+
name: 'node',
|
|
354
|
+
version: globalThis.process.version,
|
|
355
|
+
v8: globalThis.process.versions.v8,
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
return { name: runtime, version: 'unknown' };
|
|
360
|
+
}
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
---
|
|
364
|
+
|
|
365
|
+
## 6. APIs por Runtime: Quais Usar Onde
|
|
366
|
+
|
|
367
|
+
### APIs universais (pode usar em qualquer lugar)
|
|
368
|
+
|
|
369
|
+
```typescript
|
|
370
|
+
// fetch + Request + Response + Headers
|
|
371
|
+
const res = await fetch('https://api.example.com/data');
|
|
372
|
+
const data = await res.json();
|
|
373
|
+
|
|
374
|
+
// URL
|
|
375
|
+
const url = new URL('https://example.com/path?q=1');
|
|
376
|
+
url.searchParams.get('q');
|
|
377
|
+
|
|
378
|
+
// URLPattern
|
|
379
|
+
const pattern = new URLPattern({ pathname: '/users/:id' });
|
|
380
|
+
pattern.test('https://example.com/users/123');
|
|
381
|
+
|
|
382
|
+
// crypto.subtle
|
|
383
|
+
const hash = await crypto.subtle.digest(
|
|
384
|
+
'SHA-256',
|
|
385
|
+
new TextEncoder().encode('hello')
|
|
386
|
+
);
|
|
387
|
+
|
|
388
|
+
// Streams
|
|
389
|
+
const stream = new ReadableStream({
|
|
390
|
+
start(controller) {
|
|
391
|
+
controller.enqueue(new TextEncoder().encode('data'));
|
|
392
|
+
controller.close();
|
|
393
|
+
},
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
// TextEncoder/Decoder
|
|
397
|
+
const encoded = new TextEncoder().encode('utf-8 text');
|
|
398
|
+
const decoded = new TextDecoder().decode(encoded);
|
|
399
|
+
|
|
400
|
+
// AbortController
|
|
401
|
+
const controller = new AbortController();
|
|
402
|
+
setTimeout(() => controller.abort(), 5000);
|
|
403
|
+
await fetch(url, { signal: controller.signal });
|
|
404
|
+
|
|
405
|
+
// structuredClone
|
|
406
|
+
const clone = structuredClone(complexObject);
|
|
407
|
+
|
|
408
|
+
// performance.now()
|
|
409
|
+
const start = performance.now();
|
|
410
|
+
// ... operation
|
|
411
|
+
console.log(`Took ${performance.now() - start}ms`);
|
|
412
|
+
```
|
|
413
|
+
|
|
414
|
+
### APIs só em runtimes server (Node/Bun/Deno)
|
|
415
|
+
|
|
416
|
+
```typescript
|
|
417
|
+
// Node.js
|
|
418
|
+
import { readFile } from 'node:fs/promises';
|
|
419
|
+
import { join } from 'node:path';
|
|
420
|
+
import { createHash } from 'node:crypto';
|
|
421
|
+
import { fileURLToPath } from 'node:url';
|
|
422
|
+
|
|
423
|
+
// Deno
|
|
424
|
+
const text = await Deno.readTextFile('./config.json');
|
|
425
|
+
const info = await Deno.stat('./file.txt');
|
|
426
|
+
const proc = await Deno.run({ cmd: ['ls', '-la'] });
|
|
427
|
+
|
|
428
|
+
// Bun
|
|
429
|
+
const file = Bun.file('./data.json');
|
|
430
|
+
const content = await file.text();
|
|
431
|
+
await Bun.write('./output.txt', 'hello');
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
### APIs só em browsers
|
|
435
|
+
|
|
436
|
+
```typescript
|
|
437
|
+
// DOM
|
|
438
|
+
document.querySelector('.content');
|
|
439
|
+
window.localStorage.setItem('key', 'value');
|
|
440
|
+
window.open('https://example.com');
|
|
441
|
+
|
|
442
|
+
// IndexedDB
|
|
443
|
+
const db = await indexeddb.open('MyDB', 1);
|
|
444
|
+
|
|
445
|
+
// Service Workers
|
|
446
|
+
navigator.serviceWorker.register('/sw.js');
|
|
447
|
+
|
|
448
|
+
// Web Workers
|
|
449
|
+
const worker = new Worker('/worker.js');
|
|
450
|
+
```
|
|
451
|
+
|
|
452
|
+
---
|
|
453
|
+
|
|
454
|
+
## 7. Build Targets
|
|
455
|
+
|
|
456
|
+
### 7.1. tsup/tsdown
|
|
457
|
+
|
|
458
|
+
```typescript
|
|
459
|
+
// tsup.config.ts
|
|
460
|
+
import { defineConfig } from 'tsup';
|
|
461
|
+
|
|
462
|
+
export default defineConfig({
|
|
463
|
+
entry: {
|
|
464
|
+
index: 'src/index.ts',
|
|
465
|
+
node: 'src/node.ts',
|
|
466
|
+
browser: 'src/browser.ts',
|
|
467
|
+
deno: 'src/deno.ts',
|
|
468
|
+
},
|
|
469
|
+
format: ['esm'],
|
|
470
|
+
target: 'es2022',
|
|
471
|
+
dts: true,
|
|
472
|
+
splitting: true,
|
|
473
|
+
clean: true,
|
|
474
|
+
// Não polyfill APIs universais
|
|
475
|
+
noExternal: [],
|
|
476
|
+
// Manter node: imports como external
|
|
477
|
+
external: [/^node:/],
|
|
478
|
+
});
|
|
479
|
+
```
|
|
480
|
+
|
|
481
|
+
```typescript
|
|
482
|
+
// tsdown.config.ts (sucessor moderno do tsup)
|
|
483
|
+
import { defineConfig } from 'tsdown';
|
|
484
|
+
|
|
485
|
+
export default defineConfig({
|
|
486
|
+
entry: ['src/index.ts', 'src/node.ts', 'src/browser.ts'],
|
|
487
|
+
format: ['esm'],
|
|
488
|
+
dts: true,
|
|
489
|
+
// Auto-detecta platform
|
|
490
|
+
platform: 'neutral',
|
|
491
|
+
});
|
|
492
|
+
```
|
|
493
|
+
|
|
494
|
+
### 7.2. unbuild
|
|
495
|
+
|
|
496
|
+
```typescript
|
|
497
|
+
// build.config.ts
|
|
498
|
+
import { defineBuildConfig } from 'unbuild';
|
|
499
|
+
|
|
500
|
+
export default defineBuildConfig({
|
|
501
|
+
entries: [
|
|
502
|
+
'src/index',
|
|
503
|
+
'src/node',
|
|
504
|
+
'src/browser',
|
|
505
|
+
],
|
|
506
|
+
declaration: true,
|
|
507
|
+
rollup: {
|
|
508
|
+
emitCJS: false,
|
|
509
|
+
},
|
|
510
|
+
// unbuild usa mkdist ou rollup baseado no target
|
|
511
|
+
});
|
|
512
|
+
```
|
|
513
|
+
|
|
514
|
+
### 7.3. Rollup
|
|
515
|
+
|
|
516
|
+
```typescript
|
|
517
|
+
// rollup.config.js
|
|
518
|
+
export default {
|
|
519
|
+
input: {
|
|
520
|
+
index: 'src/index.ts',
|
|
521
|
+
node: 'src/node.ts',
|
|
522
|
+
browser: 'src/browser.ts',
|
|
523
|
+
},
|
|
524
|
+
output: {
|
|
525
|
+
dir: 'dist',
|
|
526
|
+
format: 'esm',
|
|
527
|
+
preserveModules: true,
|
|
528
|
+
},
|
|
529
|
+
external: [/^node:/, /^bun:/, /^https?:\/\//],
|
|
530
|
+
};
|
|
531
|
+
```
|
|
532
|
+
|
|
533
|
+
---
|
|
534
|
+
|
|
535
|
+
## 8. Package.json Conditional Exports (Types First)
|
|
536
|
+
|
|
537
|
+
### Padrão correto (types sempre primeiro)
|
|
538
|
+
|
|
539
|
+
```json
|
|
540
|
+
{
|
|
541
|
+
"exports": {
|
|
542
|
+
".": {
|
|
543
|
+
"types": "./dist/index.d.ts",
|
|
544
|
+
"import": "./dist/index.js",
|
|
545
|
+
"default": "./dist/index.js"
|
|
546
|
+
},
|
|
547
|
+
"./node": {
|
|
548
|
+
"types": "./dist/node.d.ts",
|
|
549
|
+
"import": "./dist/node.js",
|
|
550
|
+
"default": "./dist/node.js"
|
|
551
|
+
},
|
|
552
|
+
"./browser": {
|
|
553
|
+
"types": "./dist/browser.d.ts",
|
|
554
|
+
"import": "./dist/browser.js",
|
|
555
|
+
"default": "./dist/browser.js"
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
```
|
|
560
|
+
|
|
561
|
+
### Por que types primeiro?
|
|
562
|
+
|
|
563
|
+
1. **TypeScript resolve `types` antes de `import`** — garante autocomplete correto
|
|
564
|
+
2. **Sem types primeiro, o TypeScript pode pegar o wrong entry** — causa erros de tipo
|
|
565
|
+
3. **`default` é fallback** — para tools que não entendem `import`
|
|
566
|
+
|
|
567
|
+
### Conditional exports para polyfills
|
|
568
|
+
|
|
569
|
+
```json
|
|
570
|
+
{
|
|
571
|
+
"exports": {
|
|
572
|
+
".": {
|
|
573
|
+
"types": "./dist/index.d.ts",
|
|
574
|
+
"browser": {
|
|
575
|
+
"import": "./dist/browser.js"
|
|
576
|
+
},
|
|
577
|
+
"node": {
|
|
578
|
+
"import": "./dist/node.cjs",
|
|
579
|
+
"require": "./dist/node.cjs"
|
|
580
|
+
},
|
|
581
|
+
"import": "./dist/index.js",
|
|
582
|
+
"default": "./dist/index.js"
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
```
|
|
587
|
+
|
|
588
|
+
> **Nota:** A ordem das chaves importa — browsers resolvem `browser` antes de `import`.
|
|
589
|
+
|
|
590
|
+
---
|
|
591
|
+
|
|
592
|
+
## 9. Validation Tools
|
|
593
|
+
|
|
594
|
+
### 9.1. publint
|
|
595
|
+
|
|
596
|
+
Verifica se o package está publicado corretamente:
|
|
597
|
+
|
|
598
|
+
```bash
|
|
599
|
+
npx publint
|
|
600
|
+
```
|
|
601
|
+
|
|
602
|
+
**O que verifica:**
|
|
603
|
+
- Exports estão corretos
|
|
604
|
+
- Types estão incluídos
|
|
605
|
+
- Module resolution funciona
|
|
606
|
+
- Não há warnings de package.json
|
|
607
|
+
|
|
608
|
+
### 9.2. @arethetypeswrong/cli (attw)
|
|
609
|
+
|
|
610
|
+
Verifica se os tipos estão corretos para cada consumer:
|
|
611
|
+
|
|
612
|
+
```bash
|
|
613
|
+
npx @arethetypeswrong/cli
|
|
614
|
+
```
|
|
615
|
+
|
|
616
|
+
**Exemplo de output:**
|
|
617
|
+
```
|
|
618
|
+
@devthink/webscrape
|
|
619
|
+
|
|
620
|
+
"import" - ESM
|
|
621
|
+
✅ @devthink/webscrape → ./dist/index.js
|
|
622
|
+
✅ Types: ./dist/index.d.ts
|
|
623
|
+
|
|
624
|
+
"require" - CJS
|
|
625
|
+
❌ No CJS export found
|
|
626
|
+
💡 Add a "require" condition
|
|
627
|
+
|
|
628
|
+
"browser" - Browser ESM
|
|
629
|
+
✅ @devthink/webscrape → ./dist/browser.js
|
|
630
|
+
```
|
|
631
|
+
|
|
632
|
+
### 9.3. other tools
|
|
633
|
+
|
|
634
|
+
```bash
|
|
635
|
+
# Verificar se exports resolvem corretamente
|
|
636
|
+
npx arethetypeswrong --pack
|
|
637
|
+
|
|
638
|
+
# Verificar size do bundle
|
|
639
|
+
npx bundlephobia @devthink/webscrape
|
|
640
|
+
|
|
641
|
+
# Verificar dependências circulares
|
|
642
|
+
npx madge --circular src/
|
|
643
|
+
```
|
|
644
|
+
|
|
645
|
+
---
|
|
646
|
+
|
|
647
|
+
## 10. Em 2026: Não Use Mais
|
|
648
|
+
|
|
649
|
+
| Package | Por quê | Substituto |
|
|
650
|
+
|---------|---------|------------|
|
|
651
|
+
| `node-fetch` | `fetch` é global desde Node 18 | `fetch()` nativo |
|
|
652
|
+
| `cross-fetch` | Mesmo motivo | `fetch()` nativo |
|
|
653
|
+
| `isomorphic-fetch` | Deprecated, injeta global | `fetch()` nativo |
|
|
654
|
+
| `undici` como polyfill | `fetch` já é nativo | `fetch()` nativo |
|
|
655
|
+
| `buffer` package | `Buffer` já é global no Node | `Uint8Array` / `ArrayBuffer` |
|
|
656
|
+
| `path-browserify` | Use `new URL()` | `URL` nativo |
|
|
657
|
+
| `process` package | Não faça polyfill | Detecte o runtime |
|
|
658
|
+
| `stream-browserify` | Streams já são universais | `ReadableStream` nativo |
|
|
659
|
+
|
|
660
|
+
**Regra:** Se a API está no WinterTC Minimum Common, não polyfill. Se não está, use adapter.
|
|
661
|
+
|
|
662
|
+
---
|
|
663
|
+
|
|
664
|
+
## 11. Exemplos Completos
|
|
665
|
+
|
|
666
|
+
### 11.1. HTTP Client universal
|
|
667
|
+
|
|
668
|
+
```typescript
|
|
669
|
+
// src/core/http.ts
|
|
670
|
+
export interface HttpRequest {
|
|
671
|
+
url: string;
|
|
672
|
+
method?: string;
|
|
673
|
+
headers?: Record<string, string>;
|
|
674
|
+
body?: unknown;
|
|
675
|
+
timeout?: number;
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
export interface HttpResponse<T = unknown> {
|
|
679
|
+
status: number;
|
|
680
|
+
headers: Record<string, string>;
|
|
681
|
+
data: T;
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
export async function http<T = unknown>(request: HttpRequest): Promise<HttpResponse<T>> {
|
|
685
|
+
const { url, method = 'GET', headers = {}, body, timeout = 30_000 } = request;
|
|
686
|
+
|
|
687
|
+
const controller = new AbortController();
|
|
688
|
+
const timer = setTimeout(() => controller.abort(), timeout);
|
|
689
|
+
|
|
690
|
+
try {
|
|
691
|
+
const response = await fetch(url, {
|
|
692
|
+
method,
|
|
693
|
+
headers: {
|
|
694
|
+
'Content-Type': 'application/json',
|
|
695
|
+
...headers,
|
|
696
|
+
},
|
|
697
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
698
|
+
signal: controller.signal,
|
|
699
|
+
});
|
|
700
|
+
|
|
701
|
+
const data = await response.json() as T;
|
|
702
|
+
|
|
703
|
+
return {
|
|
704
|
+
status: response.status,
|
|
705
|
+
headers: Object.fromEntries(response.headers.entries()),
|
|
706
|
+
data,
|
|
707
|
+
};
|
|
708
|
+
} finally {
|
|
709
|
+
clearTimeout(timer);
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
```
|
|
713
|
+
|
|
714
|
+
### 11.2. Cache com TTL universal
|
|
715
|
+
|
|
716
|
+
```typescript
|
|
717
|
+
// src/core/cache.ts
|
|
718
|
+
interface CacheEntry<T> {
|
|
719
|
+
value: T;
|
|
720
|
+
expires: number;
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
export class UniversalCache<T = unknown> {
|
|
724
|
+
private store = new Map<string, CacheEntry<T>>();
|
|
725
|
+
private defaultTTL: number;
|
|
726
|
+
|
|
727
|
+
constructor(defaultTTL = 60_000) {
|
|
728
|
+
this.defaultTTL = defaultTTL;
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
set(key: string, value: T, ttl = this.defaultTTL): void {
|
|
732
|
+
this.store.set(key, {
|
|
733
|
+
value,
|
|
734
|
+
expires: performance.now() + ttl,
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
get(key: string): T | undefined {
|
|
739
|
+
const entry = this.store.get(key);
|
|
740
|
+
if (!entry) return undefined;
|
|
741
|
+
|
|
742
|
+
if (performance.now() > entry.expires) {
|
|
743
|
+
this.store.delete(key);
|
|
744
|
+
return undefined;
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
return entry.value;
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
has(key: string): boolean {
|
|
751
|
+
return this.get(key) !== undefined;
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
delete(key: string): void {
|
|
755
|
+
this.store.delete(key);
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
clear(): void {
|
|
759
|
+
this.store.clear();
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
get size(): number {
|
|
763
|
+
// Limpeza lazy de entradas expiradas
|
|
764
|
+
const now = performance.now();
|
|
765
|
+
for (const [key, entry] of this.store) {
|
|
766
|
+
if (now > entry.expires) {
|
|
767
|
+
this.store.delete(key);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
return this.store.size;
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
```
|
|
774
|
+
|
|
775
|
+
### 11.3. HTML Parser universal (Turndown)
|
|
776
|
+
|
|
777
|
+
```typescript
|
|
778
|
+
// src/core/parser.ts
|
|
779
|
+
import TurndownService from 'turndown';
|
|
780
|
+
|
|
781
|
+
// Turndown funciona em qualquer runtime que tenha DOM-like API
|
|
782
|
+
// Para runtimes server, usar jsdom (Node/Bun) ou Deno DOM
|
|
783
|
+
|
|
784
|
+
export interface ParseOptions {
|
|
785
|
+
headingStyle?: 'atx' | 'setext';
|
|
786
|
+
codeBlockStyle?: 'fenced' | 'indented';
|
|
787
|
+
bulletListMarker?: '-' | '*' | '+';
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
export function htmlToMarkdown(
|
|
791
|
+
html: string,
|
|
792
|
+
options: ParseOptions = {}
|
|
793
|
+
): string {
|
|
794
|
+
const turndown = new TurndownService({
|
|
795
|
+
headingStyle: options.headingStyle ?? 'atx',
|
|
796
|
+
codeBlockStyle: options.codeBlockStyle ?? 'fenced',
|
|
797
|
+
bulletListMarker: options.bulletListMarker ?? '-',
|
|
798
|
+
});
|
|
799
|
+
|
|
800
|
+
// Remover scripts e styles
|
|
801
|
+
turndown.remove(['script', 'style', 'noscript', 'iframe']);
|
|
802
|
+
|
|
803
|
+
return turndown.turndown(html);
|
|
804
|
+
}
|
|
805
|
+
```
|
|
806
|
+
|
|
807
|
+
### 11.4. Streaming com Progress
|
|
808
|
+
|
|
809
|
+
```typescript
|
|
810
|
+
// src/core/stream.ts
|
|
811
|
+
export interface StreamOptions {
|
|
812
|
+
url: string;
|
|
813
|
+
onProgress?: (loaded: number, total: number) => void;
|
|
814
|
+
signal?: AbortSignal;
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
export async function* streamResponse(options: StreamOptions): AsyncGenerator<Uint8Array> {
|
|
818
|
+
const { url, onProgress, signal } = options;
|
|
819
|
+
|
|
820
|
+
const response = await fetch(url, { signal });
|
|
821
|
+
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
|
822
|
+
|
|
823
|
+
const contentLength = Number(response.headers.get('content-length')) || 0;
|
|
824
|
+
const reader = response.body?.getReader();
|
|
825
|
+
|
|
826
|
+
if (!reader) throw new Error('No body');
|
|
827
|
+
|
|
828
|
+
let loaded = 0;
|
|
829
|
+
|
|
830
|
+
try {
|
|
831
|
+
while (true) {
|
|
832
|
+
const { done, value } = await reader.read();
|
|
833
|
+
if (done) break;
|
|
834
|
+
|
|
835
|
+
loaded += value.length;
|
|
836
|
+
onProgress?.(loaded, contentLength);
|
|
837
|
+
|
|
838
|
+
yield value;
|
|
839
|
+
}
|
|
840
|
+
} finally {
|
|
841
|
+
reader.releaseLock();
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
// Uso
|
|
846
|
+
const chunks: Uint8Array[] = [];
|
|
847
|
+
for await (const chunk of streamResponse({
|
|
848
|
+
url: 'https://example.com/large-file.bin',
|
|
849
|
+
onProgress: (loaded, total) => {
|
|
850
|
+
console.log(`${((loaded / total) * 100).toFixed(1)}%`);
|
|
851
|
+
},
|
|
852
|
+
})) {
|
|
853
|
+
chunks.push(chunk);
|
|
854
|
+
}
|
|
855
|
+
```
|
|
856
|
+
|
|
857
|
+
### 11.5. Config file loader universal
|
|
858
|
+
|
|
859
|
+
```typescript
|
|
860
|
+
// src/core/config.ts
|
|
861
|
+
const CONFIG_NAMES = ['webscrape.config', '.webscrape'];
|
|
862
|
+
const CONFIG_EXTENSIONS = ['.ts', '.js', '.json', '.mjs'];
|
|
863
|
+
|
|
864
|
+
export interface Config {
|
|
865
|
+
urls: string[];
|
|
866
|
+
format: string;
|
|
867
|
+
output: string;
|
|
868
|
+
timeout: number;
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
export async function loadConfig(root = '.'): Promise<Config | null> {
|
|
872
|
+
for (const name of CONFIG_NAMES) {
|
|
873
|
+
for (const ext of CONFIG_EXTENSIONS) {
|
|
874
|
+
const path = `${root}/${name}${ext}`;
|
|
875
|
+
try {
|
|
876
|
+
const res = await fetch(new URL(path, import.meta.url).href);
|
|
877
|
+
if (res.ok) {
|
|
878
|
+
const text = await res.text();
|
|
879
|
+
if (ext === '.json') {
|
|
880
|
+
return JSON.parse(text) as Config;
|
|
881
|
+
}
|
|
882
|
+
// Para .ts/.js/.mjs — usar dynamic import em runtimes server
|
|
883
|
+
// Em browser, usar eval (com cuidado) ou import map
|
|
884
|
+
return JSON.parse(text) as Config;
|
|
885
|
+
}
|
|
886
|
+
} catch {
|
|
887
|
+
continue;
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
return null;
|
|
892
|
+
}
|
|
893
|
+
```
|
|
894
|
+
|
|
895
|
+
### 11.6. Runtime-aware package entry points
|
|
896
|
+
|
|
897
|
+
```json
|
|
898
|
+
{
|
|
899
|
+
"name": "@devthink/webscrape",
|
|
900
|
+
"exports": {
|
|
901
|
+
".": {
|
|
902
|
+
"types": "./dist/index.d.ts",
|
|
903
|
+
"browser": {
|
|
904
|
+
"types": "./dist/browser.d.ts",
|
|
905
|
+
"import": "./dist/browser.js"
|
|
906
|
+
},
|
|
907
|
+
"deno": {
|
|
908
|
+
"types": "./dist/deno.d.ts",
|
|
909
|
+
"import": "./dist/deno.js"
|
|
910
|
+
},
|
|
911
|
+
"bun": {
|
|
912
|
+
"types": "./dist/bun.d.ts",
|
|
913
|
+
"import": "./dist/bun.js"
|
|
914
|
+
},
|
|
915
|
+
"node": {
|
|
916
|
+
"types": "./dist/node.d.ts",
|
|
917
|
+
"import": "./dist/node.js"
|
|
918
|
+
},
|
|
919
|
+
"import": "./dist/index.js",
|
|
920
|
+
"default": "./dist/index.js"
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
```
|
|
925
|
+
|
|
926
|
+
---
|
|
927
|
+
|
|
928
|
+
## 12. Checklist Universal
|
|
929
|
+
|
|
930
|
+
- [ ] Usar apenas APIs do WinterTC Minimum Common no core
|
|
931
|
+
- [ ] `fetch`, `Request`, `Response`, `Headers` como HTTP client
|
|
932
|
+
- [ ] `URL` e `URLPattern` para parsing de URLs
|
|
933
|
+
- [ ] `crypto.subtle` para hashing e criptografia
|
|
934
|
+
- [ ] `ReadableStream` / `TransformStream` para streaming
|
|
935
|
+
- [ ] `TextEncoder` / `TextDecoder` para encoding
|
|
936
|
+
- [ ] `performance.now()` para timing
|
|
937
|
+
- [ ] `node:` protocol para imports do Node.js
|
|
938
|
+
- [ ] Runtime detection: Deno → Bun → Node → Browser
|
|
939
|
+
- [ ] Conditional exports com types primeiro
|
|
940
|
+
- [ ] Não usar: `node-fetch`, `cross-fetch`, `isomorphic-fetch`
|
|
941
|
+
- [ ] Validar com `publint` e `@arethetypeswrong/cli`
|
|
942
|
+
- [ ] Build targets: tsup/tsdown com `format: ['esm']`
|
|
943
|
+
- [ ] Adapter pattern para APIs não-universais
|
|
944
|
+
- [ ] Testar em pelo menos 2 runtimes (Node + um browser)
|