@wenathlan/saddle 1.8.17 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +1339 -0
- package/README.md +68 -22
- package/alternatives.ts +1218 -0
- package/biome.json +153 -0
- package/boards.json +1517 -0
- package/compute.ts +3144 -0
- package/cores.json +1801 -0
- package/dist/acquisition.d.ts +359 -0
- package/dist/acquisition.d.ts.map +1 -0
- package/dist/acquisition.js +629 -0
- package/dist/acquisition.js.map +1 -0
- package/dist/alternatives.d.ts +509 -0
- package/dist/alternatives.d.ts.map +1 -0
- package/dist/alternatives.js +951 -0
- package/dist/alternatives.js.map +1 -0
- package/dist/automation.d.ts +426 -0
- package/dist/automation.d.ts.map +1 -0
- package/dist/automation.js +561 -0
- package/dist/automation.js.map +1 -0
- package/dist/browser.d.ts +562 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +1057 -0
- package/dist/browser.js.map +1 -0
- package/dist/capacitor.config.d.ts +13 -0
- package/dist/capacitor.config.d.ts.map +1 -0
- package/dist/capacitor.config.js +26 -0
- package/dist/capacitor.config.js.map +1 -0
- package/dist/cli.d.ts +17 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +111 -0
- package/dist/cli.js.map +1 -0
- package/dist/communication.d.ts +192 -0
- package/dist/communication.d.ts.map +1 -0
- package/dist/communication.js +418 -0
- package/dist/communication.js.map +1 -0
- package/dist/compute.d.ts +1161 -0
- package/dist/compute.d.ts.map +1 -0
- package/dist/compute.js +2273 -0
- package/dist/compute.js.map +1 -0
- package/dist/distribution.d.ts +529 -0
- package/dist/distribution.d.ts.map +1 -0
- package/dist/distribution.js +811 -0
- package/dist/distribution.js.map +1 -0
- package/dist/execution.d.ts +530 -0
- package/dist/execution.d.ts.map +1 -0
- package/dist/execution.js +761 -0
- package/dist/execution.js.map +1 -0
- package/dist/format.d.ts +19 -0
- package/dist/format.d.ts.map +1 -0
- package/dist/format.js +60 -0
- package/dist/format.js.map +1 -0
- package/dist/foundation.d.ts +176 -0
- package/dist/foundation.d.ts.map +1 -0
- package/dist/foundation.js +309 -0
- package/dist/foundation.js.map +1 -0
- package/dist/index.d.ts +733 -122
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1018 -122
- package/dist/index.js.map +1 -1
- package/dist/integration.d.ts +236 -0
- package/dist/integration.d.ts.map +1 -0
- package/dist/integration.js +236 -0
- package/dist/integration.js.map +1 -0
- package/dist/intelligence.d.ts +94 -0
- package/dist/intelligence.d.ts.map +1 -0
- package/dist/intelligence.js +145 -0
- package/dist/intelligence.js.map +1 -0
- package/dist/isolation.d.ts +129 -0
- package/dist/isolation.d.ts.map +1 -0
- package/dist/isolation.js +96 -0
- package/dist/isolation.js.map +1 -0
- package/dist/media.d.ts +344 -0
- package/dist/media.d.ts.map +1 -0
- package/dist/media.js +986 -0
- package/dist/media.js.map +1 -0
- package/dist/modes.d.ts +359 -0
- package/dist/modes.d.ts.map +1 -0
- package/dist/modes.js +134 -0
- package/dist/modes.js.map +1 -0
- package/dist/{library/public.d.ts → operations.d.ts} +15 -1
- package/dist/operations.d.ts.map +1 -0
- package/dist/{library/public.js → operations.js} +18 -7
- package/dist/operations.js.map +1 -0
- package/dist/orchestrator.d.ts +1339 -0
- package/dist/orchestrator.d.ts.map +1 -0
- package/dist/orchestrator.js +4308 -0
- package/dist/orchestrator.js.map +1 -0
- package/dist/performance.d.ts +416 -0
- package/dist/performance.d.ts.map +1 -0
- package/dist/performance.js +827 -0
- package/dist/performance.js.map +1 -0
- package/dist/quantum.d.ts +810 -0
- package/dist/quantum.d.ts.map +1 -0
- package/dist/quantum.js +2073 -0
- package/dist/quantum.js.map +1 -0
- package/dist/render.d.ts +381 -0
- package/dist/render.d.ts.map +1 -0
- package/dist/render.js +1073 -0
- package/dist/render.js.map +1 -0
- package/dist/scheduler.d.ts +596 -0
- package/dist/scheduler.d.ts.map +1 -0
- package/dist/scheduler.js +1259 -0
- package/dist/scheduler.js.map +1 -0
- package/dist/security.d.ts +501 -0
- package/dist/security.d.ts.map +1 -0
- package/dist/security.js +1170 -0
- package/dist/security.js.map +1 -0
- package/dist/server.d.ts +21 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/{server/node.js → server.js} +16 -2
- package/dist/server.js.map +1 -0
- package/dist/tiers.d.ts +1002 -0
- package/dist/tiers.d.ts.map +1 -0
- package/dist/tiers.js +2165 -0
- package/dist/tiers.js.map +1 -0
- package/dist/virtual.d.ts +641 -0
- package/dist/virtual.d.ts.map +1 -0
- package/dist/virtual.js +1285 -0
- package/dist/virtual.js.map +1 -0
- package/dist/virtualcpu.d.ts +316 -0
- package/dist/virtualcpu.d.ts.map +1 -0
- package/dist/virtualcpu.js +1180 -0
- package/dist/virtualcpu.js.map +1 -0
- package/dist/virtualgpu.d.ts +345 -0
- package/dist/virtualgpu.d.ts.map +1 -0
- package/dist/virtualgpu.js +1000 -0
- package/dist/virtualgpu.js.map +1 -0
- package/dist/virtualization.d.ts +607 -0
- package/dist/virtualization.d.ts.map +1 -0
- package/dist/virtualization.js +1187 -0
- package/dist/virtualization.js.map +1 -0
- package/dist/virtualmemory.d.ts +818 -0
- package/dist/virtualmemory.d.ts.map +1 -0
- package/dist/virtualmemory.js +1796 -0
- package/dist/virtualmemory.js.map +1 -0
- package/dist/webscrape.d.ts +720 -0
- package/dist/webscrape.d.ts.map +1 -0
- package/dist/webscrape.js +2697 -0
- package/dist/webscrape.js.map +1 -0
- package/docker.config +769 -0
- package/docs/CONVERSA.txt +1523 -0
- package/docs/alternatives.md +499 -0
- package/docs/architecture-1.8.19.md +76 -0
- package/docs/architecture-1.8.19.mmd +33 -0
- package/docs/architecture-1.8.19.png +0 -0
- package/docs/architecture.md +355 -0
- package/docs/artifactavailability.md +6 -0
- package/docs/brancharchive-2026-08-18.md +18 -0
- package/docs/browser.md +7 -0
- package/docs/consolidation.md +70 -0
- package/docs/e2ugh-engine.md +528 -0
- package/{dist/examples/localjob.js → docs/example-localjob.ts} +3 -3
- package/{dist/examples/publicapi.js → docs/example-publicapi.ts} +2 -2
- package/docs/hardware.md +342 -0
- package/docs/optimization.md +126 -0
- package/docs/performance.md +499 -0
- package/docs/planning.1.8.18.md +94 -0
- package/docs/planning.1.8.19.md +49 -0
- package/docs/releasenotes-1.8.17.md +6 -0
- package/docs/releasenotes-1.8.18.md +32 -0
- package/docs/releasenotes-1.8.19.md +35 -0
- package/docs/research-1.8.18-isolation.md +43 -0
- package/docs/research-1.8.19-virtual-browser.md +27 -0
- package/docs/saddle.archive.1.8.17.tar.gz.gpg +0 -0
- package/docs/security.md +454 -0
- package/docs/todo-1.8.16.md +56 -1
- package/docs/todo-1.8.18.md +164 -0
- package/docs/todo-1.8.19.md +241 -0
- package/docs/viability.md +777 -0
- package/docs/virtualization.md +335 -0
- package/docs/web-duplicatecleanup.md +9 -0
- package/docs/web-ideas.md +106 -0
- package/docs/workflowimprovements-2026-08-19.md +70 -0
- package/gpumonitor.cpp +1543 -0
- package/gpus.json +2408 -0
- package/index.ts +1478 -0
- package/media.ts +1337 -0
- package/mttg.config +1549 -0
- package/orchestrator.ts +5462 -0
- package/package.json +304 -55
- package/passage.config +1331 -0
- package/performance.ts +1049 -0
- package/processors.json +2217 -0
- package/qemu.config +1062 -0
- package/qemubridge.py +1340 -0
- package/quantum.ts +2393 -0
- package/render.ts +1325 -0
- package/scheduler.ts +1600 -0
- package/security.ts +1449 -0
- package/tiers.ts +2871 -0
- package/tsconfig.json +29 -0
- package/virtualcpu.ts +1273 -0
- package/virtualgpu.ts +1181 -0
- package/virtualhardware.c +1011 -0
- package/virtualhardware.json +739 -0
- package/virtualization.ts +1628 -0
- package/virtualizationcore.cpp +4635 -0
- package/virtualmemory.ts +2303 -0
- package/vm.config.json +1910 -0
- package/{extension → web/extension}/manifest.json +1 -1
- package/web/readme.md +382 -0
- package/web/tsconfig.json +25 -0
- package/dist/adapters/forge.d.ts +0 -17
- package/dist/adapters/forge.d.ts.map +0 -1
- package/dist/adapters/forge.js +0 -19
- package/dist/adapters/forge.js.map +0 -1
- package/dist/adapters/forgejo.d.ts +0 -49
- package/dist/adapters/forgejo.d.ts.map +0 -1
- package/dist/adapters/forgejo.js +0 -8
- package/dist/adapters/forgejo.js.map +0 -1
- package/dist/adapters/github.d.ts +0 -12
- package/dist/adapters/github.d.ts.map +0 -1
- package/dist/adapters/github.js +0 -20
- package/dist/adapters/github.js.map +0 -1
- package/dist/adapters/gitlab.d.ts +0 -17
- package/dist/adapters/gitlab.d.ts.map +0 -1
- package/dist/adapters/gitlab.js +0 -10
- package/dist/adapters/gitlab.js.map +0 -1
- package/dist/adapters/huggingface.d.ts +0 -17
- package/dist/adapters/huggingface.d.ts.map +0 -1
- package/dist/adapters/huggingface.js +0 -6
- package/dist/adapters/huggingface.js.map +0 -1
- package/dist/adapters/socket.d.ts +0 -11
- package/dist/adapters/socket.d.ts.map +0 -1
- package/dist/adapters/socket.js +0 -17
- package/dist/adapters/socket.js.map +0 -1
- package/dist/adapters/transport.d.ts +0 -7
- package/dist/adapters/transport.d.ts.map +0 -1
- package/dist/adapters/transport.js +0 -36
- package/dist/adapters/transport.js.map +0 -1
- package/dist/ai/chunk.d.ts +0 -2
- package/dist/ai/chunk.d.ts.map +0 -1
- package/dist/ai/chunk.js +0 -36
- package/dist/ai/chunk.js.map +0 -1
- package/dist/ai/llmstxt.d.ts +0 -6
- package/dist/ai/llmstxt.d.ts.map +0 -1
- package/dist/ai/llmstxt.js +0 -12
- package/dist/ai/llmstxt.js.map +0 -1
- package/dist/ai/provenance.d.ts +0 -22
- package/dist/ai/provenance.d.ts.map +0 -1
- package/dist/ai/provenance.js +0 -25
- package/dist/ai/provenance.js.map +0 -1
- package/dist/ai/rag.d.ts +0 -26
- package/dist/ai/rag.d.ts.map +0 -1
- package/dist/ai/rag.js +0 -22
- package/dist/ai/rag.js.map +0 -1
- package/dist/ai/tokens.d.ts +0 -15
- package/dist/ai/tokens.d.ts.map +0 -1
- package/dist/ai/tokens.js +0 -9
- package/dist/ai/tokens.js.map +0 -1
- package/dist/api/auth.d.ts +0 -16
- package/dist/api/auth.d.ts.map +0 -1
- package/dist/api/auth.js +0 -19
- package/dist/api/auth.js.map +0 -1
- package/dist/api/contracts.d.ts +0 -28
- package/dist/api/contracts.d.ts.map +0 -1
- package/dist/api/contracts.js +0 -14
- package/dist/api/contracts.js.map +0 -1
- package/dist/api/control.d.ts +0 -10
- package/dist/api/control.d.ts.map +0 -1
- package/dist/api/control.js +0 -35
- package/dist/api/control.js.map +0 -1
- package/dist/api/http.d.ts +0 -7
- package/dist/api/http.d.ts.map +0 -1
- package/dist/api/http.js +0 -12
- package/dist/api/http.js.map +0 -1
- package/dist/api/rate.d.ts +0 -28
- package/dist/api/rate.d.ts.map +0 -1
- package/dist/api/rate.js +0 -36
- package/dist/api/rate.js.map +0 -1
- package/dist/api/security.d.ts +0 -12
- package/dist/api/security.d.ts.map +0 -1
- package/dist/api/security.js +0 -56
- package/dist/api/security.js.map +0 -1
- package/dist/api/service.d.ts +0 -5
- package/dist/api/service.d.ts.map +0 -1
- package/dist/api/service.js +0 -70
- package/dist/api/service.js.map +0 -1
- package/dist/apps/registry.d.ts +0 -33
- package/dist/apps/registry.d.ts.map +0 -1
- package/dist/apps/registry.js +0 -24
- package/dist/apps/registry.js.map +0 -1
- package/dist/binary/archive.d.ts +0 -27
- package/dist/binary/archive.d.ts.map +0 -1
- package/dist/binary/archive.js +0 -46
- package/dist/binary/archive.js.map +0 -1
- package/dist/binary/build.d.ts +0 -29
- package/dist/binary/build.d.ts.map +0 -1
- package/dist/binary/build.js +0 -17
- package/dist/binary/build.js.map +0 -1
- package/dist/binary/transform.d.ts +0 -70
- package/dist/binary/transform.d.ts.map +0 -1
- package/dist/binary/transform.js +0 -105
- package/dist/binary/transform.js.map +0 -1
- package/dist/bot/adapter.d.ts +0 -5
- package/dist/bot/adapter.d.ts.map +0 -1
- package/dist/bot/adapter.js +0 -11
- package/dist/bot/adapter.js.map +0 -1
- package/dist/bot/bot.d.ts +0 -28
- package/dist/bot/bot.d.ts.map +0 -1
- package/dist/bot/bot.js +0 -52
- package/dist/bot/bot.js.map +0 -1
- package/dist/bot/commands.d.ts +0 -8
- package/dist/bot/commands.d.ts.map +0 -1
- package/dist/bot/commands.js +0 -19
- package/dist/bot/commands.js.map +0 -1
- package/dist/bot/permissions.d.ts +0 -13
- package/dist/bot/permissions.d.ts.map +0 -1
- package/dist/bot/permissions.js +0 -16
- package/dist/bot/permissions.js.map +0 -1
- package/dist/browser/actions.d.ts +0 -33
- package/dist/browser/actions.d.ts.map +0 -1
- package/dist/browser/actions.js +0 -44
- package/dist/browser/actions.js.map +0 -1
- package/dist/browser/agent.d.ts +0 -21
- package/dist/browser/agent.d.ts.map +0 -1
- package/dist/browser/agent.js +0 -13
- package/dist/browser/agent.js.map +0 -1
- package/dist/browser/context.d.ts +0 -43
- package/dist/browser/context.d.ts.map +0 -1
- package/dist/browser/context.js +0 -54
- package/dist/browser/context.js.map +0 -1
- package/dist/browser/fingerprint.d.ts +0 -4
- package/dist/browser/fingerprint.d.ts.map +0 -1
- package/dist/browser/fingerprint.js +0 -12
- package/dist/browser/fingerprint.js.map +0 -1
- package/dist/browser/index.d.ts +0 -11
- package/dist/browser/index.d.ts.map +0 -1
- package/dist/browser/index.js +0 -11
- package/dist/browser/index.js.map +0 -1
- package/dist/browser/playwright.d.ts +0 -11
- package/dist/browser/playwright.d.ts.map +0 -1
- package/dist/browser/playwright.js +0 -24
- package/dist/browser/playwright.js.map +0 -1
- package/dist/browser/recorder.d.ts +0 -21
- package/dist/browser/recorder.d.ts.map +0 -1
- package/dist/browser/recorder.js +0 -58
- package/dist/browser/recorder.js.map +0 -1
- package/dist/browser/session.d.ts +0 -14
- package/dist/browser/session.d.ts.map +0 -1
- package/dist/browser/session.js +0 -21
- package/dist/browser/session.js.map +0 -1
- package/dist/browser/snapshot.d.ts +0 -69
- package/dist/browser/snapshot.d.ts.map +0 -1
- package/dist/browser/snapshot.js +0 -133
- package/dist/browser/snapshot.js.map +0 -1
- package/dist/captcha/contract.d.ts +0 -35
- package/dist/captcha/contract.d.ts.map +0 -1
- package/dist/captcha/contract.js +0 -17
- package/dist/captcha/contract.js.map +0 -1
- package/dist/captcha/evidence.d.ts +0 -10
- package/dist/captcha/evidence.d.ts.map +0 -1
- package/dist/captcha/evidence.js +0 -9
- package/dist/captcha/evidence.js.map +0 -1
- package/dist/captcha/guard.d.ts +0 -23
- package/dist/captcha/guard.d.ts.map +0 -1
- package/dist/captcha/guard.js +0 -12
- package/dist/captcha/guard.js.map +0 -1
- package/dist/cli/main.d.ts +0 -4
- package/dist/cli/main.d.ts.map +0 -1
- package/dist/cli/main.js +0 -46
- package/dist/cli/main.js.map +0 -1
- package/dist/core/errors.d.ts +0 -84
- package/dist/core/errors.d.ts.map +0 -1
- package/dist/core/errors.js +0 -54
- package/dist/core/errors.js.map +0 -1
- package/dist/core/events.d.ts +0 -9
- package/dist/core/events.d.ts.map +0 -1
- package/dist/core/events.js +0 -21
- package/dist/core/events.js.map +0 -1
- package/dist/core/hash.d.ts +0 -10
- package/dist/core/hash.d.ts.map +0 -1
- package/dist/core/hash.js +0 -82
- package/dist/core/hash.js.map +0 -1
- package/dist/core/ids.d.ts +0 -10
- package/dist/core/ids.d.ts.map +0 -1
- package/dist/core/ids.js +0 -15
- package/dist/core/ids.js.map +0 -1
- package/dist/delivery/manifest.d.ts +0 -35
- package/dist/delivery/manifest.d.ts.map +0 -1
- package/dist/delivery/manifest.js +0 -68
- package/dist/delivery/manifest.js.map +0 -1
- package/dist/deploy/index.d.ts +0 -6
- package/dist/deploy/index.d.ts.map +0 -1
- package/dist/deploy/index.js +0 -6
- package/dist/deploy/index.js.map +0 -1
- package/dist/dispatch/resumable.d.ts +0 -102
- package/dist/dispatch/resumable.d.ts.map +0 -1
- package/dist/dispatch/resumable.js +0 -64
- package/dist/dispatch/resumable.js.map +0 -1
- package/dist/dispatch/workflow.d.ts +0 -6
- package/dist/dispatch/workflow.d.ts.map +0 -1
- package/dist/dispatch/workflow.js +0 -37
- package/dist/dispatch/workflow.js.map +0 -1
- package/dist/domain/artifacts.d.ts +0 -12
- package/dist/domain/artifacts.d.ts.map +0 -1
- package/dist/domain/artifacts.js +0 -14
- package/dist/domain/artifacts.js.map +0 -1
- package/dist/domain/jobs.d.ts +0 -12
- package/dist/domain/jobs.d.ts.map +0 -1
- package/dist/domain/jobs.js +0 -20
- package/dist/domain/jobs.js.map +0 -1
- package/dist/domain/providers.d.ts +0 -10
- package/dist/domain/providers.d.ts.map +0 -1
- package/dist/domain/providers.js +0 -8
- package/dist/domain/providers.js.map +0 -1
- package/dist/domain/runtime.d.ts +0 -14
- package/dist/domain/runtime.d.ts.map +0 -1
- package/dist/domain/runtime.js +0 -10
- package/dist/domain/runtime.js.map +0 -1
- package/dist/domain/sessions.d.ts +0 -12
- package/dist/domain/sessions.d.ts.map +0 -1
- package/dist/domain/sessions.js +0 -57
- package/dist/domain/sessions.js.map +0 -1
- package/dist/examples/localjob.d.ts +0 -2
- package/dist/examples/localjob.d.ts.map +0 -1
- package/dist/examples/localjob.js.map +0 -1
- package/dist/examples/publicapi.d.ts +0 -2
- package/dist/examples/publicapi.d.ts.map +0 -1
- package/dist/examples/publicapi.js.map +0 -1
- package/dist/extension/build.d.ts +0 -9
- package/dist/extension/build.d.ts.map +0 -1
- package/dist/extension/build.js +0 -85
- package/dist/extension/build.js.map +0 -1
- package/dist/extension/content.d.ts +0 -6
- package/dist/extension/content.d.ts.map +0 -1
- package/dist/extension/content.js +0 -135
- package/dist/extension/content.js.map +0 -1
- package/dist/extension/index.d.ts +0 -7
- package/dist/extension/index.d.ts.map +0 -1
- package/dist/extension/index.js +0 -7
- package/dist/extension/index.js.map +0 -1
- package/dist/extension/pagebridge.d.ts +0 -6
- package/dist/extension/pagebridge.d.ts.map +0 -1
- package/dist/extension/pagebridge.js +0 -42
- package/dist/extension/pagebridge.js.map +0 -1
- package/dist/extension/permissions.d.ts +0 -26
- package/dist/extension/permissions.d.ts.map +0 -1
- package/dist/extension/permissions.js +0 -28
- package/dist/extension/permissions.js.map +0 -1
- package/dist/extension/popup.d.ts +0 -5
- package/dist/extension/popup.d.ts.map +0 -1
- package/dist/extension/popup.js +0 -26
- package/dist/extension/popup.js.map +0 -1
- package/dist/extension/protocol.d.ts +0 -58
- package/dist/extension/protocol.d.ts.map +0 -1
- package/dist/extension/protocol.js +0 -101
- package/dist/extension/protocol.js.map +0 -1
- package/dist/extension/serviceworker.d.ts +0 -25
- package/dist/extension/serviceworker.d.ts.map +0 -1
- package/dist/extension/serviceworker.js +0 -102
- package/dist/extension/serviceworker.js.map +0 -1
- package/dist/extension/worker.d.ts +0 -28
- package/dist/extension/worker.d.ts.map +0 -1
- package/dist/extension/worker.js +0 -22
- package/dist/extension/worker.js.map +0 -1
- package/dist/format/check.d.ts +0 -8
- package/dist/format/check.d.ts.map +0 -1
- package/dist/format/check.js +0 -38
- package/dist/format/check.js.map +0 -1
- package/dist/library/public.d.ts.map +0 -1
- package/dist/library/public.js.map +0 -1
- package/dist/mcp/browser.d.ts +0 -9
- package/dist/mcp/browser.d.ts.map +0 -1
- package/dist/mcp/browser.js +0 -13
- package/dist/mcp/browser.js.map +0 -1
- package/dist/mcp/server.d.ts +0 -35
- package/dist/mcp/server.d.ts.map +0 -1
- package/dist/mcp/server.js +0 -40
- package/dist/mcp/server.js.map +0 -1
- package/dist/mcp/transport.d.ts +0 -7
- package/dist/mcp/transport.d.ts.map +0 -1
- package/dist/mcp/transport.js +0 -17
- package/dist/mcp/transport.js.map +0 -1
- package/dist/memory/bridge.d.ts +0 -14
- package/dist/memory/bridge.d.ts.map +0 -1
- package/dist/memory/bridge.js +0 -16
- package/dist/memory/bridge.js.map +0 -1
- package/dist/memory/engine.d.ts +0 -38
- package/dist/memory/engine.d.ts.map +0 -1
- package/dist/memory/engine.js +0 -109
- package/dist/memory/engine.js.map +0 -1
- package/dist/memory/modes.d.ts +0 -43
- package/dist/memory/modes.d.ts.map +0 -1
- package/dist/memory/modes.js +0 -66
- package/dist/memory/modes.js.map +0 -1
- package/dist/memory/objects.d.ts +0 -21
- package/dist/memory/objects.d.ts.map +0 -1
- package/dist/memory/objects.js +0 -20
- package/dist/memory/objects.js.map +0 -1
- package/dist/memory/planner.d.ts +0 -66
- package/dist/memory/planner.d.ts.map +0 -1
- package/dist/memory/planner.js +0 -108
- package/dist/memory/planner.js.map +0 -1
- package/dist/memory/targets.d.ts +0 -78
- package/dist/memory/targets.d.ts.map +0 -1
- package/dist/memory/targets.js +0 -29
- package/dist/memory/targets.js.map +0 -1
- package/dist/memory/transforms.d.ts +0 -19
- package/dist/memory/transforms.d.ts.map +0 -1
- package/dist/memory/transforms.js +0 -14
- package/dist/memory/transforms.js.map +0 -1
- package/dist/modes/matrix.d.ts +0 -23
- package/dist/modes/matrix.d.ts.map +0 -1
- package/dist/modes/matrix.js +0 -18
- package/dist/modes/matrix.js.map +0 -1
- package/dist/modes/modes.d.ts +0 -33
- package/dist/modes/modes.d.ts.map +0 -1
- package/dist/modes/modes.js +0 -15
- package/dist/modes/modes.js.map +0 -1
- package/dist/modes/resolve.d.ts +0 -96
- package/dist/modes/resolve.d.ts.map +0 -1
- package/dist/modes/resolve.js +0 -48
- package/dist/modes/resolve.js.map +0 -1
- package/dist/observability/metrics.d.ts +0 -14
- package/dist/observability/metrics.d.ts.map +0 -1
- package/dist/observability/metrics.js +0 -18
- package/dist/observability/metrics.js.map +0 -1
- package/dist/packager/manifest.d.ts +0 -66
- package/dist/packager/manifest.d.ts.map +0 -1
- package/dist/packager/manifest.js +0 -94
- package/dist/packager/manifest.js.map +0 -1
- package/dist/packager/publish.d.ts +0 -51
- package/dist/packager/publish.d.ts.map +0 -1
- package/dist/packager/publish.js +0 -18
- package/dist/packager/publish.js.map +0 -1
- package/dist/packager/targetcli.d.ts +0 -7
- package/dist/packager/targetcli.d.ts.map +0 -1
- package/dist/packager/targetcli.js +0 -22
- package/dist/packager/targetcli.js.map +0 -1
- package/dist/persistence/adapter.d.ts +0 -5
- package/dist/persistence/adapter.d.ts.map +0 -1
- package/dist/persistence/adapter.js +0 -11
- package/dist/persistence/adapter.js.map +0 -1
- package/dist/persistence/drizzle.d.ts +0 -2
- package/dist/persistence/drizzle.d.ts.map +0 -1
- package/dist/persistence/drizzle.js +0 -12
- package/dist/persistence/drizzle.js.map +0 -1
- package/dist/persistence/memory.d.ts +0 -2
- package/dist/persistence/memory.d.ts.map +0 -1
- package/dist/persistence/memory.js +0 -27
- package/dist/persistence/memory.js.map +0 -1
- package/dist/persistence/migrations.d.ts +0 -19
- package/dist/persistence/migrations.d.ts.map +0 -1
- package/dist/persistence/migrations.js +0 -13
- package/dist/persistence/migrations.js.map +0 -1
- package/dist/persistence/prisma.d.ts +0 -2
- package/dist/persistence/prisma.d.ts.map +0 -1
- package/dist/persistence/prisma.js +0 -25
- package/dist/persistence/prisma.js.map +0 -1
- package/dist/persistence/schema.d.ts +0 -63
- package/dist/persistence/schema.d.ts.map +0 -1
- package/dist/persistence/schema.js +0 -28
- package/dist/persistence/schema.js.map +0 -1
- package/dist/persistence/sql.d.ts +0 -3
- package/dist/persistence/sql.d.ts.map +0 -1
- package/dist/persistence/sql.js +0 -37
- package/dist/persistence/sql.js.map +0 -1
- package/dist/protocol/blocks.d.ts +0 -10
- package/dist/protocol/blocks.d.ts.map +0 -1
- package/dist/protocol/blocks.js +0 -26
- package/dist/protocol/blocks.js.map +0 -1
- package/dist/protocol/json.d.ts +0 -6
- package/dist/protocol/json.d.ts.map +0 -1
- package/dist/protocol/json.js +0 -6
- package/dist/protocol/json.js.map +0 -1
- package/dist/protocol/ndjson.d.ts +0 -3
- package/dist/protocol/ndjson.d.ts.map +0 -1
- package/dist/protocol/ndjson.js +0 -20
- package/dist/protocol/ndjson.js.map +0 -1
- package/dist/protocol/sse.d.ts +0 -7
- package/dist/protocol/sse.d.ts.map +0 -1
- package/dist/protocol/sse.js +0 -25
- package/dist/protocol/sse.js.map +0 -1
- package/dist/proxy/pool.d.ts +0 -9
- package/dist/proxy/pool.d.ts.map +0 -1
- package/dist/proxy/pool.js +0 -27
- package/dist/proxy/pool.js.map +0 -1
- package/dist/queue/idempotency.d.ts +0 -10
- package/dist/queue/idempotency.d.ts.map +0 -1
- package/dist/queue/idempotency.js +0 -13
- package/dist/queue/idempotency.js.map +0 -1
- package/dist/queue/persistent.d.ts +0 -14
- package/dist/queue/persistent.d.ts.map +0 -1
- package/dist/queue/persistent.js +0 -66
- package/dist/queue/persistent.js.map +0 -1
- package/dist/queue/queue.d.ts +0 -7
- package/dist/queue/queue.d.ts.map +0 -1
- package/dist/queue/queue.js +0 -55
- package/dist/queue/queue.js.map +0 -1
- package/dist/queue/saga.d.ts +0 -5
- package/dist/queue/saga.d.ts.map +0 -1
- package/dist/queue/saga.js +0 -20
- package/dist/queue/saga.js.map +0 -1
- package/dist/release/assets.d.ts +0 -144
- package/dist/release/assets.d.ts.map +0 -1
- package/dist/release/assets.js +0 -157
- package/dist/release/assets.js.map +0 -1
- package/dist/release/evidence.d.ts +0 -92
- package/dist/release/evidence.d.ts.map +0 -1
- package/dist/release/evidence.js +0 -148
- package/dist/release/evidence.js.map +0 -1
- package/dist/release/verify.d.ts +0 -12
- package/dist/release/verify.d.ts.map +0 -1
- package/dist/release/verify.js +0 -112
- package/dist/release/verify.js.map +0 -1
- package/dist/runners/chain.d.ts +0 -125
- package/dist/runners/chain.d.ts.map +0 -1
- package/dist/runners/chain.js +0 -95
- package/dist/runners/chain.js.map +0 -1
- package/dist/runners/health.d.ts +0 -36
- package/dist/runners/health.d.ts.map +0 -1
- package/dist/runners/health.js +0 -26
- package/dist/runners/health.js.map +0 -1
- package/dist/runners/heartbeat.d.ts +0 -23
- package/dist/runners/heartbeat.d.ts.map +0 -1
- package/dist/runners/heartbeat.js +0 -29
- package/dist/runners/heartbeat.js.map +0 -1
- package/dist/runners/inprocess.d.ts +0 -17
- package/dist/runners/inprocess.d.ts.map +0 -1
- package/dist/runners/inprocess.js +0 -20
- package/dist/runners/inprocess.js.map +0 -1
- package/dist/runners/scheduler.d.ts +0 -5
- package/dist/runners/scheduler.d.ts.map +0 -1
- package/dist/runners/scheduler.js +0 -19
- package/dist/runners/scheduler.js.map +0 -1
- package/dist/runtime/abort.d.ts +0 -8
- package/dist/runtime/abort.d.ts.map +0 -1
- package/dist/runtime/abort.js +0 -13
- package/dist/runtime/abort.js.map +0 -1
- package/dist/runtime/compatibility.d.ts +0 -37
- package/dist/runtime/compatibility.d.ts.map +0 -1
- package/dist/runtime/compatibility.js +0 -10
- package/dist/runtime/compatibility.js.map +0 -1
- package/dist/runtime/detect.d.ts +0 -13
- package/dist/runtime/detect.d.ts.map +0 -1
- package/dist/runtime/detect.js +0 -18
- package/dist/runtime/detect.js.map +0 -1
- package/dist/runtime/engine.d.ts +0 -20
- package/dist/runtime/engine.d.ts.map +0 -1
- package/dist/runtime/engine.js +0 -61
- package/dist/runtime/engine.js.map +0 -1
- package/dist/runtime/retry.d.ts +0 -19
- package/dist/runtime/retry.d.ts.map +0 -1
- package/dist/runtime/retry.js +0 -54
- package/dist/runtime/retry.js.map +0 -1
- package/dist/runtime/worker.d.ts +0 -10
- package/dist/runtime/worker.d.ts.map +0 -1
- package/dist/runtime/worker.js +0 -25
- package/dist/runtime/worker.js.map +0 -1
- package/dist/scrape/cache.d.ts +0 -45
- package/dist/scrape/cache.d.ts.map +0 -1
- package/dist/scrape/cache.js +0 -45
- package/dist/scrape/cache.js.map +0 -1
- package/dist/scrape/crawl.d.ts +0 -47
- package/dist/scrape/crawl.d.ts.map +0 -1
- package/dist/scrape/crawl.js +0 -114
- package/dist/scrape/crawl.js.map +0 -1
- package/dist/scrape/extract.d.ts +0 -63
- package/dist/scrape/extract.d.ts.map +0 -1
- package/dist/scrape/extract.js +0 -47
- package/dist/scrape/extract.js.map +0 -1
- package/dist/scrape/normalize.d.ts +0 -52
- package/dist/scrape/normalize.d.ts.map +0 -1
- package/dist/scrape/normalize.js +0 -102
- package/dist/scrape/normalize.js.map +0 -1
- package/dist/scrape/robots.d.ts +0 -37
- package/dist/scrape/robots.d.ts.map +0 -1
- package/dist/scrape/robots.js +0 -71
- package/dist/scrape/robots.js.map +0 -1
- package/dist/scrape/schema.d.ts +0 -23
- package/dist/scrape/schema.d.ts.map +0 -1
- package/dist/scrape/schema.js +0 -92
- package/dist/scrape/schema.js.map +0 -1
- package/dist/scrape/scraper.d.ts +0 -6
- package/dist/scrape/scraper.d.ts.map +0 -1
- package/dist/scrape/scraper.js +0 -46
- package/dist/scrape/scraper.js.map +0 -1
- package/dist/scrape/semantic.d.ts +0 -25
- package/dist/scrape/semantic.d.ts.map +0 -1
- package/dist/scrape/semantic.js +0 -27
- package/dist/scrape/semantic.js.map +0 -1
- package/dist/server/node.d.ts +0 -7
- package/dist/server/node.d.ts.map +0 -1
- package/dist/server/node.js.map +0 -1
- package/dist/sessions/file.d.ts +0 -25
- package/dist/sessions/file.d.ts.map +0 -1
- package/dist/sessions/file.js +0 -13
- package/dist/sessions/file.js.map +0 -1
- package/dist/sessions/replay.d.ts +0 -9
- package/dist/sessions/replay.d.ts.map +0 -1
- package/dist/sessions/replay.js +0 -70
- package/dist/sessions/replay.js.map +0 -1
- package/dist/sessions/store.d.ts +0 -25
- package/dist/sessions/store.d.ts.map +0 -1
- package/dist/sessions/store.js +0 -13
- package/dist/sessions/store.js.map +0 -1
- package/dist/storage/adapter.d.ts +0 -5
- package/dist/storage/adapter.d.ts.map +0 -1
- package/dist/storage/adapter.js +0 -11
- package/dist/storage/adapter.js.map +0 -1
- package/dist/storage/cache.d.ts +0 -23
- package/dist/storage/cache.d.ts.map +0 -1
- package/dist/storage/cache.js +0 -79
- package/dist/storage/cache.js.map +0 -1
- package/dist/storage/checksum.d.ts +0 -3
- package/dist/storage/checksum.d.ts.map +0 -1
- package/dist/storage/checksum.js +0 -19
- package/dist/storage/checksum.js.map +0 -1
- package/dist/storage/chunked.d.ts +0 -21
- package/dist/storage/chunked.d.ts.map +0 -1
- package/dist/storage/chunked.js +0 -67
- package/dist/storage/chunked.js.map +0 -1
- package/dist/storage/content.d.ts +0 -24
- package/dist/storage/content.d.ts.map +0 -1
- package/dist/storage/content.js +0 -44
- package/dist/storage/content.js.map +0 -1
- package/dist/storage/filehosting.d.ts +0 -2
- package/dist/storage/filehosting.d.ts.map +0 -1
- package/dist/storage/filehosting.js +0 -24
- package/dist/storage/filehosting.js.map +0 -1
- package/dist/storage/githubcontents.d.ts +0 -2
- package/dist/storage/githubcontents.d.ts.map +0 -1
- package/dist/storage/githubcontents.js +0 -29
- package/dist/storage/githubcontents.js.map +0 -1
- package/dist/storage/index.d.ts +0 -12
- package/dist/storage/index.d.ts.map +0 -1
- package/dist/storage/index.js +0 -12
- package/dist/storage/index.js.map +0 -1
- package/dist/storage/local.d.ts +0 -2
- package/dist/storage/local.d.ts.map +0 -1
- package/dist/storage/local.js +0 -72
- package/dist/storage/local.js.map +0 -1
- package/dist/storage/memory.d.ts +0 -6
- package/dist/storage/memory.d.ts.map +0 -1
- package/dist/storage/memory.js +0 -36
- package/dist/storage/memory.js.map +0 -1
- package/dist/storage/pool.d.ts +0 -95
- package/dist/storage/pool.d.ts.map +0 -1
- package/dist/storage/pool.js +0 -202
- package/dist/storage/pool.js.map +0 -1
- package/dist/storage/s3compatible.d.ts +0 -3
- package/dist/storage/s3compatible.d.ts.map +0 -1
- package/dist/storage/s3compatible.js +0 -70
- package/dist/storage/s3compatible.js.map +0 -1
- package/dist/storage/sync.d.ts +0 -47
- package/dist/storage/sync.d.ts.map +0 -1
- package/dist/storage/sync.js +0 -75
- package/dist/storage/sync.js.map +0 -1
- package/dist/surfaces/adapters.d.ts +0 -97
- package/dist/surfaces/adapters.d.ts.map +0 -1
- package/dist/surfaces/adapters.js +0 -50
- package/dist/surfaces/adapters.js.map +0 -1
- package/dist/surfaces/controls.d.ts +0 -18
- package/dist/surfaces/controls.d.ts.map +0 -1
- package/dist/surfaces/controls.js +0 -45
- package/dist/surfaces/controls.js.map +0 -1
- package/dist/surfaces/manifest.d.ts +0 -68
- package/dist/surfaces/manifest.d.ts.map +0 -1
- package/dist/surfaces/manifest.js +0 -42
- package/dist/surfaces/manifest.js.map +0 -1
- package/dist/surfaces/n8n.d.ts +0 -25
- package/dist/surfaces/n8n.d.ts.map +0 -1
- package/dist/surfaces/n8n.js +0 -31
- package/dist/surfaces/n8n.js.map +0 -1
- package/dist/surfaces/operations.d.ts +0 -32
- package/dist/surfaces/operations.d.ts.map +0 -1
- package/dist/surfaces/operations.js +0 -53
- package/dist/surfaces/operations.js.map +0 -1
- package/dist/surfaces/requirements.d.ts +0 -29
- package/dist/surfaces/requirements.d.ts.map +0 -1
- package/dist/surfaces/requirements.js +0 -50
- package/dist/surfaces/requirements.js.map +0 -1
- package/dist/surfaces/targets.d.ts +0 -196
- package/dist/surfaces/targets.d.ts.map +0 -1
- package/dist/surfaces/targets.js +0 -27
- package/dist/surfaces/targets.js.map +0 -1
- package/dist/webhook/delivery.d.ts +0 -18
- package/dist/webhook/delivery.d.ts.map +0 -1
- package/dist/webhook/delivery.js +0 -36
- package/dist/webhook/delivery.js.map +0 -1
- package/dist/webhook/receiver.d.ts +0 -26
- package/dist/webhook/receiver.d.ts.map +0 -1
- package/dist/webhook/receiver.js +0 -24
- package/dist/webhook/receiver.js.map +0 -1
- package/dist/webhook/signature.d.ts +0 -3
- package/dist/webhook/signature.d.ts.map +0 -1
- package/dist/webhook/signature.js +0 -7
- package/dist/webhook/signature.js.map +0 -1
- package/dist/workflow/manifest.d.ts +0 -22
- package/dist/workflow/manifest.d.ts.map +0 -1
- package/dist/workflow/manifest.js +0 -21
- package/dist/workflow/manifest.js.map +0 -1
- package/dist/workflow/registry.d.ts +0 -7
- package/dist/workflow/registry.d.ts.map +0 -1
- package/dist/workflow/registry.js +0 -18
- package/dist/workflow/registry.js.map +0 -1
- package/dist/workflow/templates.d.ts +0 -6
- package/dist/workflow/templates.d.ts.map +0 -1
- package/dist/workflow/templates.js +0 -18
- package/dist/workflow/templates.js.map +0 -1
- package/dist/workflow/triggers.d.ts +0 -64
- package/dist/workflow/triggers.d.ts.map +0 -1
- package/dist/workflow/triggers.js +0 -99
- package/dist/workflow/triggers.js.map +0 -1
- package/docs/logs/.gitkeep +0 -0
- package/docs/plans/00.index.md +0 -50
- package/docs/plans/01.architecture.md +0 -86
- package/docs/plans/02.research.computer.use.md +0 -58
- package/docs/plans/03.research.captcha.bypass.md +0 -68
- package/docs/plans/04.research.sandbox.ai.md +0 -52
- package/docs/plans/05.capture.platform.md +0 -57
- package/docs/plans/06.dependencies.md +0 -97
- package/docs/plans/07.captcha.test.page.md +0 -41
- package/docs/plans/08.production.infra.md +0 -70
- package/docs/plans/09.database.schema.md +0 -121
- package/docs/plans/10.cloudinary.storage.md +0 -57
- package/docs/plans/11.movement.logs.json.md +0 -72
- package/docs/plans/12.research.atlas.agent.browser.md +0 -79
- package/docs/plans/13.research.anti.detection.md +0 -898
- package/docs/plans/14.research.proxy.md +0 -1495
- package/docs/plans/15.research.retry.rate.limit.md +0 -1958
- package/docs/plans/16.research.crawling.md +0 -1417
- package/docs/plans/17.research.caching.md +0 -1610
- package/docs/plans/18.research.content.extraction.md +0 -1952
- package/docs/plans/19.research.errors.events.md +0 -1523
- package/docs/plans/20.research.zod.validation.md +0 -1350
- package/docs/plans/21.research.batch.concurrency.md +0 -1888
- package/docs/plans/22.research.universal.runtime.md +0 -944
- package/docs/plans/23.research.ai.integration.md +0 -1465
- package/docs/plans/24.research.memory.persistence.md +0 -1979
- package/docs/plans/25.research.server.api.md +0 -342
- package/docs/plans/26.research.compilation.md +0 -249
- package/docs/plans/27.research.html.parsing.md +0 -251
- package/docs/plans/28.action.plan.md +0 -50
- package/docs/plans/29.api.reference.md +0 -174
- package/docs/plans/30.architecture.plan.md +0 -94
- package/docs/plans/31.auditoria.dados.md +0 -163
- package/docs/plans/32.bots.automacao.computacional.md +0 -214
- package/docs/plans/33.bots.codigo.revisao.md +0 -220
- package/docs/plans/34.bots.seguranca.cicd.md +0 -366
- package/docs/plans/35.comparativo.concorrencia.md +0 -464
- package/docs/plans/36.computational.memory.md +0 -340
- package/docs/plans/37.deploystrategy.md +0 -394
- package/docs/plans/38.flow.md +0 -155
- package/docs/plans/39.multi.platform.bot.md +0 -252
- package/docs/plans/40.npm.publish.md +0 -250
- package/docs/plans/41.o.que.falta.md +0 -407
- package/docs/plans/42.pesquisa.concorrencia.md +0 -721
- package/docs/plans/43.plan.universal.architecture.md +0 -496
- package/docs/plans/44.reference.md +0 -100
- package/docs/plans/45.robotarchitecture.md +0 -237
- package/docs/plans/46.scdnintegration.md +0 -284
- package/docs/plans/47.multiforge.readme.md +0 -129
- package/docs/plans/48.theory.v4.repo.os.md +0 -152
- package/docs/plans/49.third.party.infra.md +0 -12
- package/docs/plans/50.file.as.compute.md +0 -39
- package/docs/plans/51.architecture.virtual.processor.md +0 -80
- package/docs/plans/52.manifesto.v8.md +0 -11
- package/docs/plans/58.cdn.list.md +0 -23
- package/docs/plans/59.sql.frameworks.md +0 -33
- package/docs/plans/60.sql.thirdparty.md +0 -26
- package/docs/plans/61.objective.multiforge.md +0 -63
- package/docs/plans/62.huggingface.upload.md +0 -26
- package/docs/plans/63.kaggle.upload.md +0 -24
- package/docs/plans/64.npm.storage.md +0 -30
- package/docs/plans/65.rclone.terabox.md +0 -32
- package/docs/plans/66.buckets.and.models.todo.md +0 -14
- package/docs/plans/67.database.todo.md +0 -13
- package/docs/plans/68.deploy.packages.todo.md +0 -12
- package/docs/plans/69.report.human.operator.md +0 -133
- package/docs/plans/70.report.brain2qwerty.ems.md +0 -135
- package/docs/plans/71.report.hd.infinito.vram.md +0 -155
- package/docs/plans/72.plan.hd.infinito.node.md +0 -146
- package/docs/plans/73.plan.scifi.repos.md +0 -125
- package/docs/plans/74.000.manifesto.v8.flat.2..md +0 -11
- package/docs/plans/README.md +0 -489
- package/docs/plans/aggregate_platforms.mjs +0 -146
- package/docs/plans/examplesession.json +0 -36
- package/docs/plans/missing-facts.md +0 -192
- package/docs/plans/models.md +0 -64
- package/docs/plans/organize.cjs +0 -270
- package/docs/plans/platforms.md +0 -2887
- package/docs/plans/sites.md +0 -31322
- package/docs/sources/farm.py +0 -117
- package/docs/sources/html/saddle1.html +0 -132
- package/docs/sources/html/saddle2.html +0 -157
- package/docs/sources/html/saddle3.html +0 -119
- package/docs/sources/html/saddle4.html +0 -144
- package/docs/sources/html/saddle5.html +0 -72
- package/docs/sources/html/saddle6.html +0 -171
- package/docs/sources/html/saddle7.html +0 -236
- package/docs/sources/saddle.ts +0 -74
- package/docs/sources/schema.prisma +0 -88
- package/docs/sources/script.sh +0 -64
- package/docs/sources/workflows.yml +0 -458
- package/docs/talks1/_body.txt +0 -14
- package/docs/talks1/_index.md +0 -15
- package/docs/talks1/_screenshot.png +0 -0
- package/docs/talks1/assistant-01.md +0 -5
- package/docs/talks1/assistant-02.md +0 -5
- package/docs/talks1/assistant-03.md +0 -531
- package/docs/talks1/assistant-04.md +0 -26
- package/docs/talks1/assistant-05.md +0 -774
- package/docs/talks1/assistant-06.md +0 -1718
- package/docs/talks1/scrape-share.cjs +0 -185
- package/docs/talks1/scrape-share.ts +0 -183
- package/docs/talks1/user-01.md +0 -3
- package/docs/talks1/user-02.md +0 -3
- package/docs/talks1/user-03.md +0 -88
- package/docs/talks1/user-04.md +0 -3
- package/docs/talks1/user-05.md +0 -3
- package/docs/talks1/user-06.md +0 -88
- package/docs/talks1/user-07.md +0 -88
- package/docs/talks2/_body.txt +0 -14
- package/docs/talks2/_index.md +0 -16
- package/docs/talks2/_screenshot.png +0 -0
- package/docs/talks2/assistant-01.md +0 -5
- package/docs/talks2/assistant-02.md +0 -5
- package/docs/talks2/assistant-03.md +0 -424
- package/docs/talks2/assistant-04.md +0 -598
- package/docs/talks2/assistant-05.md +0 -1280
- package/docs/talks2/assistant-06.md +0 -1227
- package/docs/talks2/assistant-07.md +0 -1252
- package/docs/talks2/user-01.md +0 -3
- package/docs/talks2/user-02.md +0 -3
- package/docs/talks2/user-03.md +0 -88
- package/docs/talks2/user-04.md +0 -88
- package/docs/talks2/user-05.md +0 -88
- package/docs/talks2/user-06.md +0 -88
- package/docs/talks2/user-07.md +0 -3
- package/docs/talks3/_body.txt +0 -467
- package/docs/talks3/_index.md +0 -10
- package/docs/talks3/_screenshot.png +0 -0
- package/docs/talks3/assistant-01.md +0 -417
- package/docs/talks3/assistant-02.md +0 -417
- package/docs/talks3/assistant-03.md +0 -29
- package/docs/talks3/assistant-04.md +0 -727
- package/docs/talks3/user-01.md +0 -88
- package/docs/talks3/user-02.md +0 -88
- package/docs/talks3/user-03.md +0 -3
- package/docs/talks3/user-04.md +0 -3
- package/docs/talks4/_body.txt +0 -14
- package/docs/talks4/_index.md +0 -12
- package/docs/talks4/_screenshot.png +0 -0
- package/docs/talks4/assistant-01.md +0 -5
- package/docs/talks4/assistant-02.md +0 -5
- package/docs/talks4/assistant-03.md +0 -35
- package/docs/talks4/assistant-04.md +0 -512
- package/docs/talks4/assistant-05.md +0 -599
- package/docs/talks4/user-01.md +0 -3
- package/docs/talks4/user-02.md +0 -3
- package/docs/talks4/user-03.md +0 -88
- package/docs/talks4/user-04.md +0 -88
- package/docs/talks4/user-05.md +0 -7
- package/docs/talks5/_body.txt +0 -14
- package/docs/talks5/_index.md +0 -13
- package/docs/talks5/_screenshot.png +0 -0
- package/docs/talks5/assistant-01.md +0 -5
- package/docs/talks5/assistant-02.md +0 -5
- package/docs/talks5/assistant-03.md +0 -690
- package/docs/talks5/assistant-04.md +0 -758
- package/docs/talks5/assistant-05.md +0 -974
- package/docs/talks5/user-01.md +0 -3
- package/docs/talks5/user-02.md +0 -3
- package/docs/talks5/user-03.md +0 -105
- package/docs/talks5/user-04.md +0 -105
- package/docs/talks5/user-05.md +0 -63
- package/docs/talks5/user-06.md +0 -105
- package/docs/talks6/_body.txt +0 -14
- package/docs/talks6/_index.md +0 -9
- package/docs/talks6/_screenshot.png +0 -0
- package/docs/talks6/assistant-01.md +0 -5
- package/docs/talks6/assistant-02.md +0 -5
- package/docs/talks6/assistant-03.md +0 -1499
- package/docs/talks6/user-01.md +0 -3
- package/docs/talks6/user-02.md +0 -3
- package/docs/talks6/user-03.md +0 -88
- package/docs/talks6/user-04.md +0 -88
- package/docs/talks7/_body.txt +0 -14
- package/docs/talks7/_index.md +0 -10
- package/docs/talks7/_screenshot.png +0 -0
- package/docs/talks7/assistant-01.md +0 -5
- package/docs/talks7/assistant-02.md +0 -5
- package/docs/talks7/assistant-03.md +0 -523
- package/docs/talks7/assistant-04.md +0 -617
- package/docs/talks7/user-01.md +0 -3
- package/docs/talks7/user-02.md +0 -3
- package/docs/talks7/user-03.md +0 -105
- package/docs/talks7/user-04.md +0 -67
- package/docs/talks8/conversa1.txt +0 -1322
- package/docs/talks8/conversa2.txt +0 -237
- package/docs/talks9/Beyond the Obvious_ 50 Plataformas Auto-Hospedadas de Forja de C/303/263digo para Al/303/251m de Gitea e GitLab.md" +0 -174
- package/docs/talks9/De NPM a Multi-Linguagem_ Uma Arquitetura T/303/251cnica para a Execu/303/247/303/243o Integrada de C/303/263digo no Ecossistema Node.js.md" +0 -59
- package/docs/talks9/De NPM a VMs Virtuais_ Uma An/303/241lise Arquitet/303/264nica para a Realiza/303/247/303/243o do Ciclo de Vida do Projeto SADDLE.md" +0 -91
- package/docs/talks9/Mapeamento da Engrenagem Computacional_ Uma Arquitetura para Execu/303/247/303/243o Isolada e Persist/303/252ncia em Ambientes Distribu/303/255dos.md" +0 -116
- package/docs/talks9/O Cen/303/241rio Pr/303/241tico do SADDLE_ Uma An/303/241lise de Viabilidade e Modelo de Ciclo de Vida Integrado.md" +0 -128
- package/docs/talks9/README (2).md +0 -489
- package/docs/talks9/README.md +0 -198
- package/docs/talks9/Viabilidade do Saddle_ Uma An/303/241lise T/303/251cnica da Transforma/303/247/303/243o de Armazenamento Remoto em Mem/303/263ria Computacional.md" +0 -80
- package/docs/talks9/conversa.txt +0 -544
- package/docs/talks9/other (2).md +0 -39
- package/docs/talks9/other.md +0 -57
- package/docs/talks9/outro.txt +0 -24
- /package/{extension/README.md → docs/extension.md} +0 -0
|
@@ -1,1979 +0,0 @@
|
|
|
1
|
-
# Pesquisa: Memory e Persistence Patterns (2026)
|
|
2
|
-
|
|
3
|
-
> Padrões de armazenamento em memória, browser storage, server storage, edge KV, request deduplication, e persistência de estado para scraping.
|
|
4
|
-
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## 1. In-Memory Storage
|
|
8
|
-
|
|
9
|
-
### 1.1. Map (nativo)
|
|
10
|
-
|
|
11
|
-
```typescript
|
|
12
|
-
// Map simples — bom para uso básico
|
|
13
|
-
const cache = new Map<string, unknown>();
|
|
14
|
-
|
|
15
|
-
cache.set('key', { data: 'value' });
|
|
16
|
-
cache.get('key'); // { data: 'value' }
|
|
17
|
-
cache.has('key'); // true
|
|
18
|
-
cache.delete('key'); // true
|
|
19
|
-
cache.size; // 0
|
|
20
|
-
|
|
21
|
-
// Iterar
|
|
22
|
-
for (const [key, value] of cache) {
|
|
23
|
-
console.log(key, value);
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
// Limitação: sem TTL, sem maxSize, sem cleanup automático
|
|
27
|
-
```
|
|
28
|
-
|
|
29
|
-
### 1.2. WeakMap (metadata, sem memory leak)
|
|
30
|
-
|
|
31
|
-
```typescript
|
|
32
|
-
// WeakMap — keys devem ser objects, garbage collected automaticamente
|
|
33
|
-
const metadata = new WeakMap<object, {
|
|
34
|
-
scrapedAt: Date;
|
|
35
|
-
accessCount: number;
|
|
36
|
-
lastAccess: Date;
|
|
37
|
-
}>();
|
|
38
|
-
|
|
39
|
-
const pageData = { html: '<p>content</p>' };
|
|
40
|
-
metadata.set(pageData, {
|
|
41
|
-
scrapedAt: new Date(),
|
|
42
|
-
accessCount: 0,
|
|
43
|
-
lastAccess: new Date(),
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
// Quando pageData for GC'd, o metadata entry também é removido
|
|
47
|
-
// Ideal para: metadata associada a objects que podem ser liberados
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
### 1.3. lru-cache v11 (bounded, TTL, fetchMethod)
|
|
51
|
-
|
|
52
|
-
```typescript
|
|
53
|
-
import { LRUCache } from 'lru-cache';
|
|
54
|
-
|
|
55
|
-
// Cache com limite de tamanho e TTL
|
|
56
|
-
const cache = new LRUCache<string, unknown>({
|
|
57
|
-
max: 500, // Máximo 500 entries
|
|
58
|
-
ttl: 1000 * 60 * 5, // TTL: 5 minutos
|
|
59
|
-
updateAgeOnGet: true, // Reset TTL ao acessar
|
|
60
|
-
updateAgeOnHas: true, // Reset TTL ao verificar
|
|
61
|
-
|
|
62
|
-
// fetchMethod: busca automática com cache
|
|
63
|
-
fetchMethod: async (key: string) => {
|
|
64
|
-
const response = await fetch(`https://api.example.com/data/${key}`);
|
|
65
|
-
return response.json();
|
|
66
|
-
},
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
// Uso com fetchMethod — busca e cacheia automaticamente
|
|
70
|
-
const data = await cache.fetch('my-key');
|
|
71
|
-
// Segunda vez: retorna do cache (sem fetch)
|
|
72
|
-
|
|
73
|
-
// Operações básicas
|
|
74
|
-
cache.set('key', 'value', { ttl: 1000 * 60 }); // TTL customizado
|
|
75
|
-
cache.get('key'); // 'value'
|
|
76
|
-
cache.has('key'); // true
|
|
77
|
-
cache.delete('key'); // true
|
|
78
|
-
cache.clear(); // Limpa tudo
|
|
79
|
-
|
|
80
|
-
// Estatísticas
|
|
81
|
-
cache.size; // Entries atuais
|
|
82
|
-
cache.calculatedSize; // Tamanho estimado em bytes
|
|
83
|
-
cache.dump(); // Serializar todo o cache
|
|
84
|
-
cache.load(dumped); // Restaurar cache
|
|
85
|
-
|
|
86
|
-
// Iterator
|
|
87
|
-
for (const [key, value] of cache.entries()) {
|
|
88
|
-
console.log(key, value);
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// Cleanup periódico
|
|
92
|
-
const interval = setInterval(() => {
|
|
93
|
-
cache.purgeStale(); // Remove entries expiradas
|
|
94
|
-
}, 60_000);
|
|
95
|
-
```
|
|
96
|
-
|
|
97
|
-
**Exemplo: Cache de scraping com lru-cache**
|
|
98
|
-
|
|
99
|
-
```typescript
|
|
100
|
-
interface ScrapeCacheEntry {
|
|
101
|
-
html: string;
|
|
102
|
-
markdown: string;
|
|
103
|
-
metadata: Record<string, unknown>;
|
|
104
|
-
scrapedAt: number;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
const scrapeCache = new LRUCache<string, ScrapeCacheEntry>({
|
|
108
|
-
max: 1000,
|
|
109
|
-
ttl: 1000 * 60 * 30, // 30 minutos
|
|
110
|
-
updateAgeOnGet: true,
|
|
111
|
-
fetchMethod: async (url: string) => {
|
|
112
|
-
console.log(`Fetching: ${url}`);
|
|
113
|
-
const result = await scrape(url);
|
|
114
|
-
return {
|
|
115
|
-
html: result.content,
|
|
116
|
-
markdown: result.text,
|
|
117
|
-
metadata: result.metadata,
|
|
118
|
-
scrapedAt: Date.now(),
|
|
119
|
-
};
|
|
120
|
-
},
|
|
121
|
-
});
|
|
122
|
-
|
|
123
|
-
// Transparent cache — busca se não existe
|
|
124
|
-
const cached = await scrapeCache.fetch('https://example.com');
|
|
125
|
-
// Segunda vez: retorna do cache instantaneamente
|
|
126
|
-
```
|
|
127
|
-
|
|
128
|
-
### 1.4. @keyv/bigmap (20M+ entries)
|
|
129
|
-
|
|
130
|
-
```typescript
|
|
131
|
-
import { KeyvBigMap } from '@keyv/bigmap';
|
|
132
|
-
|
|
133
|
-
// Para caches enormes — suporta 20M+ entries
|
|
134
|
-
const bigCache = new KeyvBigMap({
|
|
135
|
-
maxSize: 50_000_000, // 50M entries
|
|
136
|
-
ttl: 1000 * 60 * 60, // 1 hora
|
|
137
|
-
});
|
|
138
|
-
|
|
139
|
-
await bigCache.set('key-1', { data: 'large payload' });
|
|
140
|
-
const value = await bigCache.get('key-1');
|
|
141
|
-
|
|
142
|
-
// Mais eficiente que Map para grandes volumes
|
|
143
|
-
// Usa WeakRef para memória eficiente
|
|
144
|
-
```
|
|
145
|
-
|
|
146
|
-
---
|
|
147
|
-
|
|
148
|
-
## 2. Browser Storage
|
|
149
|
-
|
|
150
|
-
### 2.1. localStorage (5-10MB, síncrono)
|
|
151
|
-
|
|
152
|
-
```typescript
|
|
153
|
-
// Limite: 5-10MB por origin
|
|
154
|
-
// Síncrono: bloqueia a thread principal
|
|
155
|
-
// Sem TTL nativo
|
|
156
|
-
|
|
157
|
-
interface StoredItem<T> {
|
|
158
|
-
value: T;
|
|
159
|
-
timestamp: number;
|
|
160
|
-
expires?: number;
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
class LocalStorageAdapter<T = unknown> {
|
|
164
|
-
private prefix: string;
|
|
165
|
-
|
|
166
|
-
constructor(prefix = 'ws_') {
|
|
167
|
-
this.prefix = prefix;
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
set(key: string, value: T, ttlMs?: number): void {
|
|
171
|
-
const item: StoredItem<T> = {
|
|
172
|
-
value,
|
|
173
|
-
timestamp: Date.now(),
|
|
174
|
-
expires: ttlMs ? Date.now() + ttlMs : undefined,
|
|
175
|
-
};
|
|
176
|
-
localStorage.setItem(this.prefix + key, JSON.stringify(item));
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
get(key: string): T | null {
|
|
180
|
-
const raw = localStorage.getItem(this.prefix + key);
|
|
181
|
-
if (!raw) return null;
|
|
182
|
-
|
|
183
|
-
try {
|
|
184
|
-
const item: StoredItem<T> = JSON.parse(raw);
|
|
185
|
-
if (item.expires && Date.now() > item.expires) {
|
|
186
|
-
this.delete(key);
|
|
187
|
-
return null;
|
|
188
|
-
}
|
|
189
|
-
return item.value;
|
|
190
|
-
} catch {
|
|
191
|
-
this.delete(key);
|
|
192
|
-
return null;
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
has(key: string): boolean {
|
|
197
|
-
return this.get(key) !== null;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
delete(key: string): void {
|
|
201
|
-
localStorage.removeItem(this.prefix + key);
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
clear(): void {
|
|
205
|
-
const keys = Object.keys(localStorage)
|
|
206
|
-
.filter(k => k.startsWith(this.prefix));
|
|
207
|
-
keys.forEach(k => localStorage.removeItem(k));
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
get size(): number {
|
|
211
|
-
let count = 0;
|
|
212
|
-
for (let i = 0; i < localStorage.length; i++) {
|
|
213
|
-
const key = localStorage.key(i);
|
|
214
|
-
if (key?.startsWith(this.prefix)) count++;
|
|
215
|
-
}
|
|
216
|
-
return count;
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
```
|
|
220
|
-
|
|
221
|
-
### 2.2. sessionStorage (tab-scoped)
|
|
222
|
-
|
|
223
|
-
```typescript
|
|
224
|
-
// Mesmo API que localStorage, mas:
|
|
225
|
-
// - Dados persistem só durante a sessão da tab
|
|
226
|
-
// - Tab != Window (abrir link em nova tab = novo sessionStorage)
|
|
227
|
-
// - Útil para: estado temporário de uma sessão de scraping
|
|
228
|
-
|
|
229
|
-
const session = new LocalStorageAdapter('session_');
|
|
230
|
-
// Mesmo adapter, diferente prefix e lifecycle
|
|
231
|
-
```
|
|
232
|
-
|
|
233
|
-
### 2.3. IndexedDB (async, large)
|
|
234
|
-
|
|
235
|
-
```typescript
|
|
236
|
-
// Async, grande capacidade (GBs), estruturado
|
|
237
|
-
// Não bloqueia a thread principal
|
|
238
|
-
|
|
239
|
-
interface IDBConfig {
|
|
240
|
-
dbName: string;
|
|
241
|
-
version?: number;
|
|
242
|
-
storeName: string;
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
class IndexedDBAdapter<T = unknown> {
|
|
246
|
-
private db: IDBDatabase | null = null;
|
|
247
|
-
private config: IDBConfig;
|
|
248
|
-
|
|
249
|
-
constructor(config: IDBConfig) {
|
|
250
|
-
this.config = { version: 1, ...config };
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
async init(): Promise<void> {
|
|
254
|
-
return new Promise((resolve, reject) => {
|
|
255
|
-
const request = indexedDB.open(
|
|
256
|
-
this.config.dbName,
|
|
257
|
-
this.config.version
|
|
258
|
-
);
|
|
259
|
-
|
|
260
|
-
request.onupgradeneeded = (event) => {
|
|
261
|
-
const db = (event.target as IDBOpenDBRequest).result;
|
|
262
|
-
if (!db.objectStoreNames.contains(this.config.storeName)) {
|
|
263
|
-
db.createObjectStore(this.config.storeName, { keyPath: 'id' });
|
|
264
|
-
}
|
|
265
|
-
};
|
|
266
|
-
|
|
267
|
-
request.onsuccess = (event) => {
|
|
268
|
-
this.db = (event.target as IDBOpenDBRequest).result;
|
|
269
|
-
resolve();
|
|
270
|
-
};
|
|
271
|
-
|
|
272
|
-
request.onerror = () => reject(request.error);
|
|
273
|
-
});
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
async set(key: string, value: T): Promise<void> {
|
|
277
|
-
if (!this.db) throw new Error('DB not initialized');
|
|
278
|
-
|
|
279
|
-
return new Promise((resolve, reject) => {
|
|
280
|
-
const tx = this.db!.transaction(this.config.storeName, 'readwrite');
|
|
281
|
-
const store = tx.objectStore(this.config.storeName);
|
|
282
|
-
const request = store.put({ id: key, value, timestamp: Date.now() });
|
|
283
|
-
|
|
284
|
-
request.onsuccess = () => resolve();
|
|
285
|
-
request.onerror = () => reject(request.error);
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
async get(key: string): Promise<T | null> {
|
|
290
|
-
if (!this.db) throw new Error('DB not initialized');
|
|
291
|
-
|
|
292
|
-
return new Promise((resolve, reject) => {
|
|
293
|
-
const tx = this.db!.transaction(this.config.storeName, 'readonly');
|
|
294
|
-
const store = tx.objectStore(this.config.storeName);
|
|
295
|
-
const request = store.get(key);
|
|
296
|
-
|
|
297
|
-
request.onsuccess = () => {
|
|
298
|
-
const result = request.result;
|
|
299
|
-
resolve(result?.value ?? null);
|
|
300
|
-
};
|
|
301
|
-
request.onerror = () => reject(request.error);
|
|
302
|
-
});
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
async delete(key: string): Promise<void> {
|
|
306
|
-
if (!this.db) throw new Error('DB not initialized');
|
|
307
|
-
|
|
308
|
-
return new Promise((resolve, reject) => {
|
|
309
|
-
const tx = this.db!.transaction(this.config.storeName, 'readwrite');
|
|
310
|
-
const store = tx.objectStore(this.config.storeName);
|
|
311
|
-
const request = store.delete(key);
|
|
312
|
-
|
|
313
|
-
request.onsuccess = () => resolve();
|
|
314
|
-
request.onerror = () => reject(request.error);
|
|
315
|
-
});
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
async getAll(): Promise<Map<string, T>> {
|
|
319
|
-
if (!this.db) throw new Error('DB not initialized');
|
|
320
|
-
|
|
321
|
-
return new Promise((resolve, reject) => {
|
|
322
|
-
const tx = this.db!.transaction(this.config.storeName, 'readonly');
|
|
323
|
-
const store = tx.objectStore(this.config.storeName);
|
|
324
|
-
const request = store.openCursor();
|
|
325
|
-
|
|
326
|
-
const map = new Map<string, T>();
|
|
327
|
-
|
|
328
|
-
request.onsuccess = () => {
|
|
329
|
-
const cursor = request.result;
|
|
330
|
-
if (cursor) {
|
|
331
|
-
map.set(cursor.key as string, cursor.value.value);
|
|
332
|
-
cursor.continue();
|
|
333
|
-
} else {
|
|
334
|
-
resolve(map);
|
|
335
|
-
}
|
|
336
|
-
};
|
|
337
|
-
request.onerror = () => reject(request.error);
|
|
338
|
-
});
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
|
-
```
|
|
342
|
-
|
|
343
|
-
### 2.4. store2 2.14.3 (wrapper cross-browser)
|
|
344
|
-
|
|
345
|
-
```typescript
|
|
346
|
-
import store from 'store2';
|
|
347
|
-
|
|
348
|
-
// Abstração unificada sobre localStorage/sessionStorage/IndexedDB
|
|
349
|
-
store.namespace('webscrape_'); // Prefix automático
|
|
350
|
-
|
|
351
|
-
store.set('key', { data: 'value' }); // Auto-JSON
|
|
352
|
-
store.get('key'); // { data: 'value' }
|
|
353
|
-
store.has('key'); // true
|
|
354
|
-
store.remove('key'); // true
|
|
355
|
-
|
|
356
|
-
// TTL manual (store2 não tem nativo)
|
|
357
|
-
store.set('key', 'value', { expire: 3600 }); // Não funciona nativamente
|
|
358
|
-
|
|
359
|
-
// Área de session (sessionStorage)
|
|
360
|
-
store.session.set('key', 'value');
|
|
361
|
-
store.session.get('key');
|
|
362
|
-
|
|
363
|
-
// Área de area (IndexedDB)
|
|
364
|
-
store.area.set('key', 'value');
|
|
365
|
-
store.area.get('key');
|
|
366
|
-
|
|
367
|
-
// Limits
|
|
368
|
-
store.set('key', 'value', false); // Não pode ser sobrescrito
|
|
369
|
-
store.has('key'); // true
|
|
370
|
-
// store.set('key', 'other'); // Silenciosamente falha
|
|
371
|
-
|
|
372
|
-
// Callbacks
|
|
373
|
-
store.on('set', (key: string, value: unknown) => {
|
|
374
|
-
console.log(`Stored: ${key}`);
|
|
375
|
-
});
|
|
376
|
-
|
|
377
|
-
// Serialização customizada
|
|
378
|
-
store.defaults({
|
|
379
|
-
serialize: JSON.stringify,
|
|
380
|
-
deserialize: JSON.parse,
|
|
381
|
-
});
|
|
382
|
-
```
|
|
383
|
-
|
|
384
|
-
---
|
|
385
|
-
|
|
386
|
-
## 3. Server Storage
|
|
387
|
-
|
|
388
|
-
### 3.1. better-sqlite3 (síncrono, rápido)
|
|
389
|
-
|
|
390
|
-
```typescript
|
|
391
|
-
import Database from 'better-sqlite3';
|
|
392
|
-
|
|
393
|
-
const db = new Database(':memory:'); // Ou './data.db'
|
|
394
|
-
|
|
395
|
-
// Criar tabela
|
|
396
|
-
db.exec(`
|
|
397
|
-
CREATE TABLE IF NOT EXISTS cache (
|
|
398
|
-
key TEXT PRIMARY KEY,
|
|
399
|
-
value TEXT NOT NULL,
|
|
400
|
-
created_at INTEGER DEFAULT (unixepoch()),
|
|
401
|
-
expires_at INTEGER,
|
|
402
|
-
access_count INTEGER DEFAULT 0
|
|
403
|
-
)
|
|
404
|
-
`);
|
|
405
|
-
|
|
406
|
-
// Inserir
|
|
407
|
-
const insert = db.prepare(
|
|
408
|
-
'INSERT OR REPLACE INTO cache (key, value, expires_at) VALUES (?, ?, ?)'
|
|
409
|
-
);
|
|
410
|
-
insert.run('url:https://example.com', JSON.stringify({ html: '...' }), Date.now() + 300_000);
|
|
411
|
-
|
|
412
|
-
// Buscar com TTL check
|
|
413
|
-
const find = db.prepare(
|
|
414
|
-
'SELECT value FROM cache WHERE key = ? AND (expires_at IS NULL OR expires_at > ?)'
|
|
415
|
-
);
|
|
416
|
-
const row = find.get('url:https://example.com', Date.now()) as { value: string } | undefined;
|
|
417
|
-
const data = row ? JSON.parse(row.value) : null;
|
|
418
|
-
|
|
419
|
-
// Batch operations (transação)
|
|
420
|
-
const insertMany = db.transaction((entries: [string, string, number][]) => {
|
|
421
|
-
for (const [key, value, expires] of entries) {
|
|
422
|
-
insert.run(key, value, expires);
|
|
423
|
-
}
|
|
424
|
-
});
|
|
425
|
-
|
|
426
|
-
insertMany([
|
|
427
|
-
['url:https://a.com', '{"html":"..."}', Date.now() + 300_000],
|
|
428
|
-
['url:https://b.com', '{"html":"..."}', Date.now() + 300_000],
|
|
429
|
-
]);
|
|
430
|
-
|
|
431
|
-
// Cleanup de entries expiradas
|
|
432
|
-
const cleanup = db.prepare(
|
|
433
|
-
'DELETE FROM cache WHERE expires_at IS NOT NULL AND expires_at < ?'
|
|
434
|
-
);
|
|
435
|
-
cleanup.run(Date.now());
|
|
436
|
-
|
|
437
|
-
// Estatísticas
|
|
438
|
-
const count = db.prepare('SELECT COUNT(*) as count FROM cache').get();
|
|
439
|
-
console.log(`Cache size: ${count.count}`);
|
|
440
|
-
```
|
|
441
|
-
|
|
442
|
-
### 3.2. @keyv/sqlite
|
|
443
|
-
|
|
444
|
-
```typescript
|
|
445
|
-
import Keyv from 'keyv';
|
|
446
|
-
import KeyvSqlite from '@keyv/sqlite';
|
|
447
|
-
|
|
448
|
-
const keyv = new Keyv({
|
|
449
|
-
store: new KeyvSqlite({
|
|
450
|
-
uri: 'sqlite://./data.sqlite',
|
|
451
|
-
busyTimeout: 5000,
|
|
452
|
-
}),
|
|
453
|
-
ttl: 1000 * 60 * 30, // 30 minutos
|
|
454
|
-
});
|
|
455
|
-
|
|
456
|
-
await keyv.set('key', { data: 'value' });
|
|
457
|
-
await keyv.get('key'); // { data: 'value' }
|
|
458
|
-
await keyv.has('key'); // true
|
|
459
|
-
await keyv.delete('key'); // true
|
|
460
|
-
|
|
461
|
-
// Iterator
|
|
462
|
-
for await (const [key, value] of keyv.iterator()) {
|
|
463
|
-
console.log(key, value);
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
// Namespace
|
|
467
|
-
const keyvA = new Keyv({ namespace: 'a' });
|
|
468
|
-
const keyvB = new Keyv({ namespace: 'b' });
|
|
469
|
-
// keyvA e keyvB são independentes
|
|
470
|
-
```
|
|
471
|
-
|
|
472
|
-
### 3.3. @keyv/redis / ioredis
|
|
473
|
-
|
|
474
|
-
```typescript
|
|
475
|
-
import Keyv from 'keyv';
|
|
476
|
-
import KeyvRedis from '@keyv/redis';
|
|
477
|
-
|
|
478
|
-
// Keyv com Redis adapter
|
|
479
|
-
const keyv = new Keyv({
|
|
480
|
-
store: new KeyvRedis('redis://localhost:6379'),
|
|
481
|
-
ttl: 1000 * 60 * 60, // 1 hora
|
|
482
|
-
});
|
|
483
|
-
|
|
484
|
-
await keyv.set('session:abc', { userId: 123, role: 'admin' });
|
|
485
|
-
await keyv.get('session:abc');
|
|
486
|
-
|
|
487
|
-
// ioredis direto (mais controle)
|
|
488
|
-
import Redis from 'ioredis';
|
|
489
|
-
|
|
490
|
-
const redis = new Redis({
|
|
491
|
-
host: 'localhost',
|
|
492
|
-
port: 6379,
|
|
493
|
-
maxRetriesPerRequest: 3,
|
|
494
|
-
retryStrategy(times) {
|
|
495
|
-
return Math.min(times * 50, 2000);
|
|
496
|
-
},
|
|
497
|
-
});
|
|
498
|
-
|
|
499
|
-
// Comandos básicos
|
|
500
|
-
await redis.set('key', JSON.stringify({ data: 'value' }), 'EX', 300); // TTL 5min
|
|
501
|
-
const value = JSON.parse((await redis.get('key')) ?? 'null');
|
|
502
|
-
await redis.del('key');
|
|
503
|
-
|
|
504
|
-
// Pipeline (batch)
|
|
505
|
-
const pipeline = redis.pipeline();
|
|
506
|
-
pipeline.set('key1', 'value1', 'EX', 60);
|
|
507
|
-
pipeline.set('key2', 'value2', 'EX', 60);
|
|
508
|
-
pipeline.get('key1');
|
|
509
|
-
const results = await pipeline.exec();
|
|
510
|
-
|
|
511
|
-
// Pub/Sub para invalidation
|
|
512
|
-
const subscriber = new Redis('redis://localhost:6379');
|
|
513
|
-
const publisher = new Redis('redis://localhost:6379');
|
|
514
|
-
|
|
515
|
-
subscriber.subscribe('cache:invalidate');
|
|
516
|
-
subscriber.on('message', (channel, key) => {
|
|
517
|
-
if (channel === 'cache:invalidate') {
|
|
518
|
-
console.log(`Invalidating: ${key}`);
|
|
519
|
-
}
|
|
520
|
-
});
|
|
521
|
-
|
|
522
|
-
publisher.publish('cache:invalidate', 'url:https://example.com');
|
|
523
|
-
```
|
|
524
|
-
|
|
525
|
-
---
|
|
526
|
-
|
|
527
|
-
## 4. Cross-Runtime KV: keyv 5.6.0
|
|
528
|
-
|
|
529
|
-
### Por que keyv?
|
|
530
|
-
|
|
531
|
-
- **104M+ weekly downloads** — padrão de facto
|
|
532
|
-
- **Pluggable adapters** — in-memory, SQLite, Redis, PostgreSQL, MongoDB
|
|
533
|
-
- **TTL nativo** — suporte a expiração
|
|
534
|
-
- **Iterator** — percorre todas as entries
|
|
535
|
-
- **Namespace** — multi-tenant no mesmo store
|
|
536
|
-
- **Universal** — funciona em Node, Deno, Bun, browsers
|
|
537
|
-
|
|
538
|
-
```typescript
|
|
539
|
-
import Keyv from 'keyv';
|
|
540
|
-
|
|
541
|
-
// Store padrão (in-memory com Map)
|
|
542
|
-
const keyv = new Keyv();
|
|
543
|
-
await keyv.set('key', 'value', 5000); // TTL 5s
|
|
544
|
-
await keyv.get('key'); // 'value'
|
|
545
|
-
|
|
546
|
-
// Com adapter Redis
|
|
547
|
-
import KeyvRedis from '@keyv/redis';
|
|
548
|
-
const redisKeyv = new Keyv({ store: new KeyvRedis('redis://localhost') });
|
|
549
|
-
|
|
550
|
-
// Com adapter SQLite
|
|
551
|
-
import KeyvSqlite from '@keyv/sqlite';
|
|
552
|
-
const sqliteKeyv = new Keyv({ store: new KeyvSqlite('sqlite://data.db') });
|
|
553
|
-
|
|
554
|
-
// Namespace
|
|
555
|
-
const sessions = new Keyv({ namespace: 'sessions' });
|
|
556
|
-
const cache = new Keyv({ namespace: 'cache' });
|
|
557
|
-
|
|
558
|
-
// Multiple namespaces no mesmo Redis
|
|
559
|
-
const keyvA = new Keyv({ store: new KeyvRedis('redis://localhost'), namespace: 'a' });
|
|
560
|
-
const keyvB = new Keyv({ store: new KeyvRedis('redis://localhost'), namespace: 'b' });
|
|
561
|
-
```
|
|
562
|
-
|
|
563
|
-
### keyv patterns para scraping
|
|
564
|
-
|
|
565
|
-
```typescript
|
|
566
|
-
// Pattern 1: Request cache
|
|
567
|
-
const requestCache = new Keyv({
|
|
568
|
-
namespace: 'requests',
|
|
569
|
-
ttl: 1000 * 60 * 10, // 10 minutos
|
|
570
|
-
});
|
|
571
|
-
|
|
572
|
-
async function cachedFetch(url: string): Promise<Response> {
|
|
573
|
-
const cached = await requestCache.get(url);
|
|
574
|
-
if (cached) return new Response(cached.body, cached);
|
|
575
|
-
|
|
576
|
-
const response = await fetch(url);
|
|
577
|
-
if (response.ok) {
|
|
578
|
-
const clone = response.clone();
|
|
579
|
-
const body = await clone.text();
|
|
580
|
-
await requestCache.set(url, {
|
|
581
|
-
body,
|
|
582
|
-
status: response.status,
|
|
583
|
-
headers: Object.fromEntries(response.headers.entries()),
|
|
584
|
-
});
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
return response;
|
|
588
|
-
}
|
|
589
|
-
|
|
590
|
-
// Pattern 2: Rate limiting
|
|
591
|
-
const rateLimiter = new Keyv({
|
|
592
|
-
namespace: 'ratelimit',
|
|
593
|
-
ttl: 1000 * 60, // 1 minuto window
|
|
594
|
-
});
|
|
595
|
-
|
|
596
|
-
async function isRateLimited(domain: string, maxRequests = 10): Promise<boolean> {
|
|
597
|
-
const count = (await rateLimiter.get(domain)) ?? 0;
|
|
598
|
-
if (count >= maxRequests) return true;
|
|
599
|
-
await rateLimiter.set(domain, count + 1);
|
|
600
|
-
return false;
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
// Pattern 3: Deduplicação
|
|
604
|
-
const dedup = new Keyv({
|
|
605
|
-
namespace: 'dedup',
|
|
606
|
-
ttl: 1000 * 60 * 60, // 1 hora
|
|
607
|
-
});
|
|
608
|
-
|
|
609
|
-
async function isDuplicate(content: string): Promise<boolean> {
|
|
610
|
-
const hash = await contentHash(content);
|
|
611
|
-
if (await dedup.has(hash)) return true;
|
|
612
|
-
await dedup.set(hash, true);
|
|
613
|
-
return false;
|
|
614
|
-
}
|
|
615
|
-
```
|
|
616
|
-
|
|
617
|
-
---
|
|
618
|
-
|
|
619
|
-
## 5. Edge Runtime Storage
|
|
620
|
-
|
|
621
|
-
### 5.1. Cloudflare Workers KV
|
|
622
|
-
|
|
623
|
-
```typescript
|
|
624
|
-
// Cloudflare Workers KV
|
|
625
|
-
// API global em workers, TTL mínimo 60s
|
|
626
|
-
|
|
627
|
-
export default {
|
|
628
|
-
async fetch(request: Request, env: Env): Promise<Response> {
|
|
629
|
-
const url = new URL(request.url);
|
|
630
|
-
const key = url.pathname.slice(1);
|
|
631
|
-
|
|
632
|
-
if (request.method === 'GET') {
|
|
633
|
-
const value = await env.KV.get(key, 'json');
|
|
634
|
-
if (!value) return new Response('Not found', { status: 404 });
|
|
635
|
-
return Response.json(value);
|
|
636
|
-
}
|
|
637
|
-
|
|
638
|
-
if (request.method === 'PUT') {
|
|
639
|
-
const body = await request.json();
|
|
640
|
-
await env.KV.put(key, JSON.stringify(body), {
|
|
641
|
-
expirationTtl: 3600, // 1 hora (mínimo 60s)
|
|
642
|
-
});
|
|
643
|
-
return new Response('OK');
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
return new Response('Method not allowed', { status: 405);
|
|
647
|
-
},
|
|
648
|
-
};
|
|
649
|
-
|
|
650
|
-
// Interface
|
|
651
|
-
interface Env {
|
|
652
|
-
KV: KVNamespace;
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
// Batch operations
|
|
656
|
-
async function batchGet(keys: string[], env: Env): Promise<Map<string, unknown>> {
|
|
657
|
-
const results = new Map<string, unknown>();
|
|
658
|
-
|
|
659
|
-
// KV não tem batch get nativo — fazer parallel
|
|
660
|
-
await Promise.all(keys.map(async (key) => {
|
|
661
|
-
const value = await env.KV.get(key, 'json');
|
|
662
|
-
if (value) results.set(key, value);
|
|
663
|
-
}));
|
|
664
|
-
|
|
665
|
-
return results;
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
// List with prefix
|
|
669
|
-
async function listByPrefix(prefix: string, env: Env): Promise<string[]> {
|
|
670
|
-
const list = await env.KV.list({ prefix });
|
|
671
|
-
return list.keys.map(k => k.name);
|
|
672
|
-
}
|
|
673
|
-
```
|
|
674
|
-
|
|
675
|
-
### 5.2. Upstash Redis
|
|
676
|
-
|
|
677
|
-
```typescript
|
|
678
|
-
import { Redis } from '@upstash/redis';
|
|
679
|
-
|
|
680
|
-
const redis = new Redis({
|
|
681
|
-
url: process.env.UPSTASH_REDIS_REST_URL!,
|
|
682
|
-
token: process.env.UPSTASH_REDIS_REST_TOKEN!,
|
|
683
|
-
});
|
|
684
|
-
|
|
685
|
-
// Operações básicas
|
|
686
|
-
await redis.set('key', { data: 'value' }, { ex: 300 }); // TTL 5min
|
|
687
|
-
await redis.get('key');
|
|
688
|
-
await redis.del('key');
|
|
689
|
-
|
|
690
|
-
// Rate limiting
|
|
691
|
-
async function upstashRateLimit(
|
|
692
|
-
key: string,
|
|
693
|
-
limit: number,
|
|
694
|
-
windowSec: number
|
|
695
|
-
): Promise<{ allowed: boolean; remaining: number }> {
|
|
696
|
-
const now = Date.now();
|
|
697
|
-
const windowStart = now - windowSec * 1000;
|
|
698
|
-
|
|
699
|
-
const pipeline = redis.pipeline();
|
|
700
|
-
pipeline.zremrangebyscore(key, 0, windowStart);
|
|
701
|
-
pipeline.zadd(key, { score: now, member: now.toString() });
|
|
702
|
-
pipeline.zcard(key);
|
|
703
|
-
pipeline.expire(key, windowSec);
|
|
704
|
-
|
|
705
|
-
const results = await pipeline.exec();
|
|
706
|
-
const count = results[2] as number;
|
|
707
|
-
|
|
708
|
-
return {
|
|
709
|
-
allowed: count <= limit,
|
|
710
|
-
remaining: Math.max(0, limit - count),
|
|
711
|
-
};
|
|
712
|
-
}
|
|
713
|
-
|
|
714
|
-
// Caching com TTL
|
|
715
|
-
async function cached<T>(
|
|
716
|
-
key: string,
|
|
717
|
-
fn: () => Promise<T>,
|
|
718
|
-
ttlSec = 300
|
|
719
|
-
): Promise<T> {
|
|
720
|
-
const cached = await redis.get<T>(key);
|
|
721
|
-
if (cached) return cached;
|
|
722
|
-
|
|
723
|
-
const value = await fn();
|
|
724
|
-
await redis.set(key, JSON.stringify(value), { ex: ttlSec });
|
|
725
|
-
return value;
|
|
726
|
-
}
|
|
727
|
-
```
|
|
728
|
-
|
|
729
|
-
### 5.3. Deno KV
|
|
730
|
-
|
|
731
|
-
```typescript
|
|
732
|
-
// Deno KV — built-in no Deno 2.x
|
|
733
|
-
const kv = await Deno.openKv();
|
|
734
|
-
|
|
735
|
-
// Operações básicas
|
|
736
|
-
await kv.set(['cache', 'url', 'https://example.com'], { html: '...' }, {
|
|
737
|
-
expireIn: 1000 * 60 * 30, // 30 minutos
|
|
738
|
-
});
|
|
739
|
-
|
|
740
|
-
const entry = await kv.get(['cache', 'url', 'https://example.com']);
|
|
741
|
-
if (entry.value) {
|
|
742
|
-
console.log(entry.value);
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
// Delete
|
|
746
|
-
await kv.delete(['cache', 'url', 'https://example.com']);
|
|
747
|
-
|
|
748
|
-
// List com prefix
|
|
749
|
-
const entries = await kv.list({ prefix: ['cache', 'url'] });
|
|
750
|
-
for await (const entry of entries) {
|
|
751
|
-
console.log(entry.key, entry.value);
|
|
752
|
-
}
|
|
753
|
-
|
|
754
|
-
// Atomic operations
|
|
755
|
-
const res = await kv.atomic()
|
|
756
|
-
.check({ key: ['counter'], versionstamp: null })
|
|
757
|
-
.sum(['counter'], 1n)
|
|
758
|
-
.commit();
|
|
759
|
-
|
|
760
|
-
if (!res.ok) {
|
|
761
|
-
console.log('Atomic operation failed');
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
// Transactions
|
|
765
|
-
async function atomicCacheUpdate(
|
|
766
|
-
key: string[],
|
|
767
|
-
value: unknown,
|
|
768
|
-
ttl: number
|
|
769
|
-
): Promise<boolean> {
|
|
770
|
-
const result = await kv.atomic()
|
|
771
|
-
.set(key, value, { expireIn: ttl })
|
|
772
|
-
.commit();
|
|
773
|
-
return result.ok;
|
|
774
|
-
}
|
|
775
|
-
```
|
|
776
|
-
|
|
777
|
-
---
|
|
778
|
-
|
|
779
|
-
## 6. Request Deduplication
|
|
780
|
-
|
|
781
|
-
### 6.1. Keyv com TTL
|
|
782
|
-
|
|
783
|
-
```typescript
|
|
784
|
-
import Keyv from 'keyv';
|
|
785
|
-
|
|
786
|
-
const dedupStore = new Keyv({
|
|
787
|
-
namespace: 'request-dedup',
|
|
788
|
-
ttl: 1000 * 60 * 5, // 5 minutos
|
|
789
|
-
});
|
|
790
|
-
|
|
791
|
-
async function deduplicatedFetch(url: string): Promise<Response> {
|
|
792
|
-
const key = `req:${url}`;
|
|
793
|
-
|
|
794
|
-
// Verificar se já está em andamento
|
|
795
|
-
if (await dedupStore.has(key)) {
|
|
796
|
-
// Retornar do cache ou aguardar
|
|
797
|
-
return cachedResponse(url);
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
// Marcar como em andamento
|
|
801
|
-
await dedupStore.set(key, { status: 'pending', started: Date.now() });
|
|
802
|
-
|
|
803
|
-
try {
|
|
804
|
-
const response = await fetch(url);
|
|
805
|
-
const body = await response.text();
|
|
806
|
-
|
|
807
|
-
// Cache a resposta
|
|
808
|
-
await dedupStore.set(key, {
|
|
809
|
-
status: 'done',
|
|
810
|
-
body,
|
|
811
|
-
status_code: response.status,
|
|
812
|
-
headers: Object.fromEntries(response.headers.entries()),
|
|
813
|
-
});
|
|
814
|
-
|
|
815
|
-
return new Response(body, {
|
|
816
|
-
status: response.status,
|
|
817
|
-
headers: response.headers,
|
|
818
|
-
});
|
|
819
|
-
} catch (error) {
|
|
820
|
-
await dedupStore.delete(key);
|
|
821
|
-
throw error;
|
|
822
|
-
}
|
|
823
|
-
}
|
|
824
|
-
```
|
|
825
|
-
|
|
826
|
-
### 6.2. Bloom Filter
|
|
827
|
-
|
|
828
|
-
```typescript
|
|
829
|
-
// Bloom filter — probabilidade controlada de falsos positivos
|
|
830
|
-
// Não tem falsos negativos — eficiente em memória
|
|
831
|
-
|
|
832
|
-
class BloomFilter {
|
|
833
|
-
private bits: Uint8Array;
|
|
834
|
-
private size: number;
|
|
835
|
-
private hashes: number;
|
|
836
|
-
|
|
837
|
-
constructor(expectedItems: number, falsePositiveRate = 0.01) {
|
|
838
|
-
// Calcular tamanho ótimo
|
|
839
|
-
this.size = Math.ceil(
|
|
840
|
-
-(expectedItems * Math.log(falsePositiveRate)) / (Math.log(2) ** 2)
|
|
841
|
-
);
|
|
842
|
-
this.hashes = Math.ceil((this.size / expectedItems) * Math.log(2));
|
|
843
|
-
this.bits = new Uint8Array(Math.ceil(this.size / 8));
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
private hash(value: string, seed: number): number {
|
|
847
|
-
let hash = 0;
|
|
848
|
-
for (let i = 0; i < value.length; i++) {
|
|
849
|
-
hash = ((hash << 5) - hash + value.charCodeAt(i) + seed) | 0;
|
|
850
|
-
}
|
|
851
|
-
return Math.abs(hash) % this.size;
|
|
852
|
-
}
|
|
853
|
-
|
|
854
|
-
add(value: string): void {
|
|
855
|
-
for (let i = 0; i < this.hashes; i++) {
|
|
856
|
-
const pos = this.hash(value, i);
|
|
857
|
-
this.bits[Math.floor(pos / 8)] |= 1 << (pos % 8);
|
|
858
|
-
}
|
|
859
|
-
}
|
|
860
|
-
|
|
861
|
-
mightContain(value: string): boolean {
|
|
862
|
-
for (let i = 0; i < this.hashes; i++) {
|
|
863
|
-
const pos = this.hash(value, i);
|
|
864
|
-
if (!(this.bits[Math.floor(pos / 8)] & (1 << (pos % 8)))) {
|
|
865
|
-
return false;
|
|
866
|
-
}
|
|
867
|
-
}
|
|
868
|
-
return true;
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
// Serializar para armazenamento
|
|
872
|
-
serialize(): Uint8Array {
|
|
873
|
-
return this.bits;
|
|
874
|
-
}
|
|
875
|
-
|
|
876
|
-
// Restaurar de armazenamento
|
|
877
|
-
static deserialize(data: Uint8Array, size: number, hashes: number): BloomFilter {
|
|
878
|
-
const filter = new BloomFilter(1);
|
|
879
|
-
filter.bits = data;
|
|
880
|
-
filter.size = size;
|
|
881
|
-
filter.hashes = hashes;
|
|
882
|
-
return filter;
|
|
883
|
-
}
|
|
884
|
-
}
|
|
885
|
-
|
|
886
|
-
// Uso
|
|
887
|
-
const urlsSeen = new BloomFilter(100_000); // 100K URLs, 1% FPR
|
|
888
|
-
|
|
889
|
-
async function shouldScrape(url: string): Promise<boolean> {
|
|
890
|
-
if (urlsSeen.mightContain(url)) {
|
|
891
|
-
console.log(`Probably seen: ${url}`);
|
|
892
|
-
return false; // Pode ser falso positivo
|
|
893
|
-
}
|
|
894
|
-
urlsSeen.add(url);
|
|
895
|
-
return true;
|
|
896
|
-
}
|
|
897
|
-
```
|
|
898
|
-
|
|
899
|
-
### 6.3. ETag Conditional Requests
|
|
900
|
-
|
|
901
|
-
```typescript
|
|
902
|
-
interface ETagCache {
|
|
903
|
-
get(url: string): Promise<{ etag: string; lastModified: string } | null>;
|
|
904
|
-
set(url: string, etag: string, lastModified: string): Promise<void>;
|
|
905
|
-
}
|
|
906
|
-
|
|
907
|
-
async function conditionalFetch(
|
|
908
|
-
url: string,
|
|
909
|
-
cache: ETagCache
|
|
910
|
-
): Promise<{ data: string; fromCache: boolean }> {
|
|
911
|
-
const cached = await cache.get(url);
|
|
912
|
-
|
|
913
|
-
const headers: Record<string, string> = {};
|
|
914
|
-
if (cached?.etag) headers['If-None-Match'] = cached.etag;
|
|
915
|
-
if (cached?.lastModified) headers['If-Modified-Since'] = cached.lastModified;
|
|
916
|
-
|
|
917
|
-
const response = await fetch(url, { headers });
|
|
918
|
-
|
|
919
|
-
if (response.status === 304) {
|
|
920
|
-
// Não modificado — usar cache
|
|
921
|
-
return { data: '', fromCache: true };
|
|
922
|
-
}
|
|
923
|
-
|
|
924
|
-
// Atualizar cache
|
|
925
|
-
const etag = response.headers.get('etag');
|
|
926
|
-
const lastModified = response.headers.get('last-modified');
|
|
927
|
-
if (etag) await cache.set(url, etag, lastModified ?? '');
|
|
928
|
-
|
|
929
|
-
const data = await response.text();
|
|
930
|
-
return { data, fromCache: false };
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
// Implementação com Keyv
|
|
934
|
-
class KeyvETagCache implements ETagCache {
|
|
935
|
-
private store: Keyv;
|
|
936
|
-
|
|
937
|
-
constructor() {
|
|
938
|
-
this.store = new Keyv({ namespace: 'etag', ttl: 1000 * 60 * 60 * 24 });
|
|
939
|
-
}
|
|
940
|
-
|
|
941
|
-
async get(url: string) {
|
|
942
|
-
return this.store.get(url) ?? null;
|
|
943
|
-
}
|
|
944
|
-
|
|
945
|
-
async set(url: string, etag: string, lastModified: string) {
|
|
946
|
-
await this.store.set(url, { etag, lastModified });
|
|
947
|
-
}
|
|
948
|
-
}
|
|
949
|
-
```
|
|
950
|
-
|
|
951
|
-
---
|
|
952
|
-
|
|
953
|
-
## 7. Persistindo Estado
|
|
954
|
-
|
|
955
|
-
### 7.1. Cookies
|
|
956
|
-
|
|
957
|
-
```typescript
|
|
958
|
-
// Cookies — persists across requests, limited to ~4KB
|
|
959
|
-
|
|
960
|
-
// Definir cookie
|
|
961
|
-
document.cookie = 'session=abc123; path=/; max-age=3600; SameSite=Lax; Secure';
|
|
962
|
-
|
|
963
|
-
// Ler cookie
|
|
964
|
-
function getCookie(name: string): string | null {
|
|
965
|
-
const match = document.cookie.match(new RegExp(`(?:^|; )${name}=([^;]*)`));
|
|
966
|
-
return match ? decodeURIComponent(match[1]) : null;
|
|
967
|
-
}
|
|
968
|
-
|
|
969
|
-
// Server-side (Set-Cookie header)
|
|
970
|
-
// res.setHeader('Set-Cookie', 'session=abc123; Path=/; Max-Age=3600; HttpOnly; Secure');
|
|
971
|
-
|
|
972
|
-
// Em fetch (não funciona cross-origin por padrão)
|
|
973
|
-
await fetch(url, {
|
|
974
|
-
credentials: 'include', // Enviar cookies
|
|
975
|
-
});
|
|
976
|
-
```
|
|
977
|
-
|
|
978
|
-
### 7.2. Auth Tokens
|
|
979
|
-
|
|
980
|
-
```typescript
|
|
981
|
-
interface AuthState {
|
|
982
|
-
accessToken: string;
|
|
983
|
-
refreshToken: string;
|
|
984
|
-
expiresAt: number;
|
|
985
|
-
}
|
|
986
|
-
|
|
987
|
-
class AuthManager {
|
|
988
|
-
private storage: Keyv<AuthState>;
|
|
989
|
-
|
|
990
|
-
constructor() {
|
|
991
|
-
this.storage = new Keyv({ namespace: 'auth', ttl: 1000 * 60 * 60 * 24 });
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
async saveTokens(url: string, tokens: AuthState): Promise<void> {
|
|
995
|
-
await this.storage.set(this.key(url), tokens);
|
|
996
|
-
}
|
|
997
|
-
|
|
998
|
-
async getValidToken(url: string): Promise<string | null> {
|
|
999
|
-
const tokens = await this.storage.get(this.key(url));
|
|
1000
|
-
if (!tokens) return null;
|
|
1001
|
-
|
|
1002
|
-
// Verificar se expirou
|
|
1003
|
-
if (Date.now() > tokens.expiresAt) {
|
|
1004
|
-
// Tentar refresh
|
|
1005
|
-
const refreshed = await this.refresh(tokens.refreshToken);
|
|
1006
|
-
if (refreshed) {
|
|
1007
|
-
await this.saveTokens(url, refreshed);
|
|
1008
|
-
return refreshed.accessToken;
|
|
1009
|
-
}
|
|
1010
|
-
await this.storage.delete(this.key(url));
|
|
1011
|
-
return null;
|
|
1012
|
-
}
|
|
1013
|
-
|
|
1014
|
-
return tokens.accessToken;
|
|
1015
|
-
}
|
|
1016
|
-
|
|
1017
|
-
private key(url: string): string {
|
|
1018
|
-
return new URL(url).origin;
|
|
1019
|
-
}
|
|
1020
|
-
|
|
1021
|
-
private async refresh(refreshToken: string): Promise<AuthState | null> {
|
|
1022
|
-
try {
|
|
1023
|
-
const response = await fetch('/auth/refresh', {
|
|
1024
|
-
method: 'POST',
|
|
1025
|
-
headers: { 'Content-Type': 'application/json' },
|
|
1026
|
-
body: JSON.stringify({ refreshToken }),
|
|
1027
|
-
});
|
|
1028
|
-
return response.json();
|
|
1029
|
-
} catch {
|
|
1030
|
-
return null;
|
|
1031
|
-
}
|
|
1032
|
-
}
|
|
1033
|
-
}
|
|
1034
|
-
```
|
|
1035
|
-
|
|
1036
|
-
### 7.3. Browser Fingerprints
|
|
1037
|
-
|
|
1038
|
-
```typescript
|
|
1039
|
-
interface BrowserFingerprint {
|
|
1040
|
-
canvas: string;
|
|
1041
|
-
webgl: string;
|
|
1042
|
-
audio: string;
|
|
1043
|
-
fonts: string[];
|
|
1044
|
-
screen: { width: number; height: number; colorDepth: number };
|
|
1045
|
-
timezone: string;
|
|
1046
|
-
language: string;
|
|
1047
|
-
platform: string;
|
|
1048
|
-
hash: string;
|
|
1049
|
-
}
|
|
1050
|
-
|
|
1051
|
-
async function generateFingerprint(): Promise<BrowserFingerprint> {
|
|
1052
|
-
const components: string[] = [];
|
|
1053
|
-
|
|
1054
|
-
// Canvas fingerprint
|
|
1055
|
-
const canvas = document.createElement('canvas');
|
|
1056
|
-
const ctx = canvas.getContext('2d')!;
|
|
1057
|
-
ctx.textBaseline = 'top';
|
|
1058
|
-
ctx.font = '14px Arial';
|
|
1059
|
-
ctx.fillText('fingerprint', 2, 2);
|
|
1060
|
-
const canvasHash = await hashString(canvas.toDataURL());
|
|
1061
|
-
components.push(canvasHash);
|
|
1062
|
-
|
|
1063
|
-
// WebGL fingerprint
|
|
1064
|
-
const gl = document.createElement('canvas').getContext('webgl')!;
|
|
1065
|
-
const debugInfo = gl.getExtension('WEBGL_debug_renderer_info');
|
|
1066
|
-
const gpu = debugInfo
|
|
1067
|
-
? gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL)
|
|
1068
|
-
: 'unknown';
|
|
1069
|
-
const webglHash = await hashString(gpu);
|
|
1070
|
-
components.push(webglHash);
|
|
1071
|
-
|
|
1072
|
-
// Audio fingerprint
|
|
1073
|
-
const audioCtx = new AudioContext();
|
|
1074
|
-
const oscillator = audioCtx.createOscillator();
|
|
1075
|
-
const analyser = audioCtx.createAnalyser();
|
|
1076
|
-
const gain = audioCtx.createGain();
|
|
1077
|
-
const scriptProcessor = audioCtx.createScriptProcessor(4096, 1, 1);
|
|
1078
|
-
|
|
1079
|
-
oscillator.type = 'triangle';
|
|
1080
|
-
oscillator.frequency.value = 10000;
|
|
1081
|
-
|
|
1082
|
-
const buffer = new Float32Array(analyser.frequencyBinCount);
|
|
1083
|
-
// ... coletar audio data
|
|
1084
|
-
const audioHash = await hashString(buffer.toString());
|
|
1085
|
-
components.push(audioHash);
|
|
1086
|
-
|
|
1087
|
-
// Screen info
|
|
1088
|
-
const screenInfo = `${screen.width}x${screen.height}x${screen.colorDepth}`;
|
|
1089
|
-
components.push(screenInfo);
|
|
1090
|
-
|
|
1091
|
-
// Timezone
|
|
1092
|
-
const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
1093
|
-
components.push(timezone);
|
|
1094
|
-
|
|
1095
|
-
// Hash final
|
|
1096
|
-
const hash = await hashString(components.join('|||'));
|
|
1097
|
-
|
|
1098
|
-
return {
|
|
1099
|
-
canvas: canvasHash,
|
|
1100
|
-
webgl: webglHash,
|
|
1101
|
-
audio: audioHash,
|
|
1102
|
-
fonts: [],
|
|
1103
|
-
screen: { width: screen.width, height: screen.height, colorDepth: screen.colorDepth },
|
|
1104
|
-
timezone,
|
|
1105
|
-
language: navigator.language,
|
|
1106
|
-
platform: navigator.platform,
|
|
1107
|
-
hash,
|
|
1108
|
-
};
|
|
1109
|
-
}
|
|
1110
|
-
|
|
1111
|
-
async function hashString(str: string): Promise<string> {
|
|
1112
|
-
const encoder = new TextEncoder();
|
|
1113
|
-
const data = encoder.encode(str);
|
|
1114
|
-
const hashBuffer = await crypto.subtle.digest('SHA-256', data);
|
|
1115
|
-
return Array.from(new Uint8Array(hashBuffer))
|
|
1116
|
-
.map(b => b.toString(16).padStart(2, '0'))
|
|
1117
|
-
.join('');
|
|
1118
|
-
}
|
|
1119
|
-
|
|
1120
|
-
// Persistir fingerprint
|
|
1121
|
-
class FingerprintManager {
|
|
1122
|
-
private storage: Keyv<BrowserFingerprint>;
|
|
1123
|
-
|
|
1124
|
-
constructor() {
|
|
1125
|
-
this.storage = new Keyv({ namespace: 'fingerprint' });
|
|
1126
|
-
}
|
|
1127
|
-
|
|
1128
|
-
async getOrCreate(): Promise<BrowserFingerprint> {
|
|
1129
|
-
const existing = await this.storage.get('current');
|
|
1130
|
-
if (existing) return existing;
|
|
1131
|
-
|
|
1132
|
-
const fp = await generateFingerprint();
|
|
1133
|
-
await this.storage.set('current', fp);
|
|
1134
|
-
return fp;
|
|
1135
|
-
}
|
|
1136
|
-
|
|
1137
|
-
async hasChanged(): Promise<boolean> {
|
|
1138
|
-
const current = await this.storage.get('current');
|
|
1139
|
-
if (!current) return false;
|
|
1140
|
-
|
|
1141
|
-
const newFp = await generateFingerprint();
|
|
1142
|
-
return current.hash !== newFp.hash;
|
|
1143
|
-
}
|
|
1144
|
-
}
|
|
1145
|
-
```
|
|
1146
|
-
|
|
1147
|
-
### 7.4. Checkpoints (resume scraping)
|
|
1148
|
-
|
|
1149
|
-
```typescript
|
|
1150
|
-
interface ScrapeCheckpoint {
|
|
1151
|
-
url: string;
|
|
1152
|
-
pagesScraped: number;
|
|
1153
|
-
lastPage: string;
|
|
1154
|
-
extractedData: unknown[];
|
|
1155
|
-
errors: string[];
|
|
1156
|
-
timestamp: number;
|
|
1157
|
-
}
|
|
1158
|
-
|
|
1159
|
-
class CheckpointManager {
|
|
1160
|
-
private storage: Keyv<ScrapeCheckpoint>;
|
|
1161
|
-
|
|
1162
|
-
constructor() {
|
|
1163
|
-
this.storage = new Keyv({
|
|
1164
|
-
namespace: 'checkpoints',
|
|
1165
|
-
ttl: 1000 * 60 * 60 * 24 * 7, // 7 dias
|
|
1166
|
-
});
|
|
1167
|
-
}
|
|
1168
|
-
|
|
1169
|
-
async save(jobId: string, checkpoint: ScrapeCheckpoint): Promise<void> {
|
|
1170
|
-
await this.storage.set(jobId, {
|
|
1171
|
-
...checkpoint,
|
|
1172
|
-
timestamp: Date.now(),
|
|
1173
|
-
});
|
|
1174
|
-
console.log(`Checkpoint saved: ${checkpoint.pagesScraped} pages`);
|
|
1175
|
-
}
|
|
1176
|
-
|
|
1177
|
-
async resume(jobId: string): Promise<ScrapeCheckpoint | null> {
|
|
1178
|
-
return this.storage.get(jobId) ?? null;
|
|
1179
|
-
}
|
|
1180
|
-
|
|
1181
|
-
async list(): Promise<ScrapeCheckpoint[]> {
|
|
1182
|
-
const checkpoints: ScrapeCheckpoint[] = [];
|
|
1183
|
-
for await (const [, value] of this.storage.iterator()) {
|
|
1184
|
-
checkpoints.push(value);
|
|
1185
|
-
}
|
|
1186
|
-
return checkpoints.sort((a, b) => b.timestamp - a.timestamp);
|
|
1187
|
-
}
|
|
1188
|
-
|
|
1189
|
-
async delete(jobId: string): Promise<void> {
|
|
1190
|
-
await this.storage.delete(jobId);
|
|
1191
|
-
}
|
|
1192
|
-
}
|
|
1193
|
-
|
|
1194
|
-
// Uso: scraping com resume
|
|
1195
|
-
async function scrapeWithResume(urls: string[]): Promise<void> {
|
|
1196
|
-
const jobId = await hashString(urls.join('\n'));
|
|
1197
|
-
const checkpointMgr = new CheckpointManager();
|
|
1198
|
-
|
|
1199
|
-
// Tentar retomar de checkpoint
|
|
1200
|
-
let checkpoint = await checkpointMgr.resume(jobId);
|
|
1201
|
-
const startIdx = checkpoint?.pagesScraped ?? 0;
|
|
1202
|
-
const data = checkpoint?.extractedData ?? [];
|
|
1203
|
-
|
|
1204
|
-
for (let i = startIdx; i < urls.length; i++) {
|
|
1205
|
-
try {
|
|
1206
|
-
const result = await scrape(urls[i]);
|
|
1207
|
-
data.push(result);
|
|
1208
|
-
|
|
1209
|
-
// Save checkpoint a cada 10 páginas
|
|
1210
|
-
if ((i + 1) % 10 === 0) {
|
|
1211
|
-
await checkpointMgr.save(jobId, {
|
|
1212
|
-
url: urls[i],
|
|
1213
|
-
pagesScraped: i + 1,
|
|
1214
|
-
lastPage: urls[i],
|
|
1215
|
-
extractedData: data,
|
|
1216
|
-
errors: checkpoint?.errors ?? [],
|
|
1217
|
-
timestamp: Date.now(),
|
|
1218
|
-
});
|
|
1219
|
-
}
|
|
1220
|
-
} catch (error) {
|
|
1221
|
-
checkpoint = checkpoint ?? {
|
|
1222
|
-
url: urls[i],
|
|
1223
|
-
pagesScraped: i,
|
|
1224
|
-
lastPage: urls[i],
|
|
1225
|
-
extractedData: data,
|
|
1226
|
-
errors: [],
|
|
1227
|
-
timestamp: Date.now(),
|
|
1228
|
-
};
|
|
1229
|
-
checkpoint.errors.push(`${urls[i]}: ${error}`);
|
|
1230
|
-
}
|
|
1231
|
-
}
|
|
1232
|
-
|
|
1233
|
-
// Limpar checkpoint após conclusão
|
|
1234
|
-
await checkpointMgr.delete(jobId);
|
|
1235
|
-
console.log(`Done! Scraped ${data.length} pages`);
|
|
1236
|
-
}
|
|
1237
|
-
```
|
|
1238
|
-
|
|
1239
|
-
---
|
|
1240
|
-
|
|
1241
|
-
## 8. Browser-Compatible Storage Layer
|
|
1242
|
-
|
|
1243
|
-
### Runtime detection com fallback chain
|
|
1244
|
-
|
|
1245
|
-
```typescript
|
|
1246
|
-
type StorageBackend = 'indexeddb' | 'localstorage' | 'memory';
|
|
1247
|
-
|
|
1248
|
-
function detectStorageBackend(): StorageBackend {
|
|
1249
|
-
// 1. IndexedDB (melhor para browsers)
|
|
1250
|
-
if (typeof indexedDB !== 'undefined') return 'indexeddb';
|
|
1251
|
-
|
|
1252
|
-
// 2. localStorage (fallback browsers)
|
|
1253
|
-
if (typeof localStorage !== 'undefined') return 'localstorage';
|
|
1254
|
-
|
|
1255
|
-
// 3. Memory (fallback universal)
|
|
1256
|
-
return 'memory';
|
|
1257
|
-
}
|
|
1258
|
-
|
|
1259
|
-
// Universal storage interface
|
|
1260
|
-
interface UniversalStorage {
|
|
1261
|
-
get<T>(key: string): Promise<T | null>;
|
|
1262
|
-
set<T>(key: string, value: T, ttlMs?: number): Promise<void>;
|
|
1263
|
-
delete(key: string): Promise<void>;
|
|
1264
|
-
has(key: string): Promise<boolean>;
|
|
1265
|
-
clear(): Promise<void>;
|
|
1266
|
-
}
|
|
1267
|
-
|
|
1268
|
-
// Factory
|
|
1269
|
-
function createStorage(namespace?: string): UniversalStorage {
|
|
1270
|
-
const backend = detectStorageBackend();
|
|
1271
|
-
|
|
1272
|
-
switch (backend) {
|
|
1273
|
-
case 'indexeddb':
|
|
1274
|
-
return new IndexedDBStorage(namespace ?? 'webscrape');
|
|
1275
|
-
case 'localstorage':
|
|
1276
|
-
return new LocalStorageStorage(namespace ?? 'ws_');
|
|
1277
|
-
case 'memory':
|
|
1278
|
-
return new MemoryStorage();
|
|
1279
|
-
}
|
|
1280
|
-
}
|
|
1281
|
-
|
|
1282
|
-
// Memory storage
|
|
1283
|
-
class MemoryStorage implements UniversalStorage {
|
|
1284
|
-
private store = new Map<string, { value: unknown; expires?: number }>();
|
|
1285
|
-
|
|
1286
|
-
async get<T>(key: string): Promise<T | null> {
|
|
1287
|
-
const entry = this.store.get(key);
|
|
1288
|
-
if (!entry) return null;
|
|
1289
|
-
if (entry.expires && Date.now() > entry.expires) {
|
|
1290
|
-
this.store.delete(key);
|
|
1291
|
-
return null;
|
|
1292
|
-
}
|
|
1293
|
-
return entry.value as T;
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
async set<T>(key: string, value: T, ttlMs?: number): Promise<void> {
|
|
1297
|
-
this.store.set(key, {
|
|
1298
|
-
value,
|
|
1299
|
-
expires: ttlMs ? Date.now() + ttlMs : undefined,
|
|
1300
|
-
});
|
|
1301
|
-
}
|
|
1302
|
-
|
|
1303
|
-
async delete(key: string): Promise<void> {
|
|
1304
|
-
this.store.delete(key);
|
|
1305
|
-
}
|
|
1306
|
-
|
|
1307
|
-
async has(key: string): Promise<boolean> {
|
|
1308
|
-
return (await this.get(key)) !== null;
|
|
1309
|
-
}
|
|
1310
|
-
|
|
1311
|
-
async clear(): Promise<void> {
|
|
1312
|
-
this.store.clear();
|
|
1313
|
-
}
|
|
1314
|
-
}
|
|
1315
|
-
|
|
1316
|
-
// LocalStorage storage
|
|
1317
|
-
class LocalStorageStorage implements UniversalStorage {
|
|
1318
|
-
private prefix: string;
|
|
1319
|
-
|
|
1320
|
-
constructor(prefix: string) {
|
|
1321
|
-
this.prefix = prefix;
|
|
1322
|
-
}
|
|
1323
|
-
|
|
1324
|
-
async get<T>(key: string): Promise<T | null> {
|
|
1325
|
-
const raw = localStorage.getItem(this.prefix + key);
|
|
1326
|
-
if (!raw) return null;
|
|
1327
|
-
try {
|
|
1328
|
-
const item = JSON.parse(raw);
|
|
1329
|
-
if (item.expires && Date.now() > item.expires) {
|
|
1330
|
-
this.delete(key);
|
|
1331
|
-
return null;
|
|
1332
|
-
}
|
|
1333
|
-
return item.value;
|
|
1334
|
-
} catch {
|
|
1335
|
-
return null;
|
|
1336
|
-
}
|
|
1337
|
-
}
|
|
1338
|
-
|
|
1339
|
-
async set<T>(key: string, value: T, ttlMs?: number): Promise<void> {
|
|
1340
|
-
localStorage.setItem(this.prefix + key, JSON.stringify({
|
|
1341
|
-
value,
|
|
1342
|
-
expires: ttlMs ? Date.now() + ttlMs : undefined,
|
|
1343
|
-
}));
|
|
1344
|
-
}
|
|
1345
|
-
|
|
1346
|
-
async delete(key: string): Promise<void> {
|
|
1347
|
-
localStorage.removeItem(this.prefix + key);
|
|
1348
|
-
}
|
|
1349
|
-
|
|
1350
|
-
async has(key: string): Promise<boolean> {
|
|
1351
|
-
return (await this.get(key)) !== null;
|
|
1352
|
-
}
|
|
1353
|
-
|
|
1354
|
-
async clear(): Promise<void> {
|
|
1355
|
-
const keys = Object.keys(localStorage)
|
|
1356
|
-
.filter(k => k.startsWith(this.prefix));
|
|
1357
|
-
keys.forEach(k => localStorage.removeItem(k));
|
|
1358
|
-
}
|
|
1359
|
-
}
|
|
1360
|
-
|
|
1361
|
-
// IndexedDB storage
|
|
1362
|
-
class IndexedDBStorage implements UniversalStorage {
|
|
1363
|
-
private dbName: string;
|
|
1364
|
-
private db: IDBDatabase | null = null;
|
|
1365
|
-
|
|
1366
|
-
constructor(dbName: string) {
|
|
1367
|
-
this.dbName = dbName;
|
|
1368
|
-
}
|
|
1369
|
-
|
|
1370
|
-
private async init(): Promise<IDBDatabase> {
|
|
1371
|
-
if (this.db) return this.db;
|
|
1372
|
-
|
|
1373
|
-
return new Promise((resolve, reject) => {
|
|
1374
|
-
const request = indexedDB.open(this.dbName, 1);
|
|
1375
|
-
|
|
1376
|
-
request.onupgradeneeded = () => {
|
|
1377
|
-
const db = request.result;
|
|
1378
|
-
if (!db.objectStoreNames.contains('store')) {
|
|
1379
|
-
db.createObjectStore('store', { keyPath: 'key' });
|
|
1380
|
-
}
|
|
1381
|
-
};
|
|
1382
|
-
|
|
1383
|
-
request.onsuccess = () => {
|
|
1384
|
-
this.db = request.result;
|
|
1385
|
-
resolve(this.db);
|
|
1386
|
-
};
|
|
1387
|
-
|
|
1388
|
-
request.onerror = () => reject(request.error);
|
|
1389
|
-
});
|
|
1390
|
-
}
|
|
1391
|
-
|
|
1392
|
-
async get<T>(key: string): Promise<T | null> {
|
|
1393
|
-
const db = await this.init();
|
|
1394
|
-
return new Promise((resolve) => {
|
|
1395
|
-
const tx = db.transaction('store', 'readonly');
|
|
1396
|
-
const store = tx.objectStore('store');
|
|
1397
|
-
const request = store.get(key);
|
|
1398
|
-
|
|
1399
|
-
request.onsuccess = () => {
|
|
1400
|
-
const result = request.result;
|
|
1401
|
-
if (!result) return resolve(null);
|
|
1402
|
-
if (result.expires && Date.now() > result.expires) {
|
|
1403
|
-
this.delete(key);
|
|
1404
|
-
return resolve(null);
|
|
1405
|
-
}
|
|
1406
|
-
resolve(result.value);
|
|
1407
|
-
};
|
|
1408
|
-
|
|
1409
|
-
request.onerror = () => resolve(null);
|
|
1410
|
-
});
|
|
1411
|
-
}
|
|
1412
|
-
|
|
1413
|
-
async set<T>(key: string, value: T, ttlMs?: number): Promise<void> {
|
|
1414
|
-
const db = await this.init();
|
|
1415
|
-
return new Promise((resolve, reject) => {
|
|
1416
|
-
const tx = db.transaction('store', 'readwrite');
|
|
1417
|
-
const store = tx.objectStore('store');
|
|
1418
|
-
const request = store.put({
|
|
1419
|
-
key,
|
|
1420
|
-
value,
|
|
1421
|
-
expires: ttlMs ? Date.now() + ttlMs : undefined,
|
|
1422
|
-
});
|
|
1423
|
-
|
|
1424
|
-
request.onsuccess = () => resolve();
|
|
1425
|
-
request.onerror = () => reject(request.error);
|
|
1426
|
-
});
|
|
1427
|
-
}
|
|
1428
|
-
|
|
1429
|
-
async delete(key: string): Promise<void> {
|
|
1430
|
-
const db = await this.init();
|
|
1431
|
-
return new Promise((resolve, reject) => {
|
|
1432
|
-
const tx = db.transaction('store', 'readwrite');
|
|
1433
|
-
const store = tx.objectStore('store');
|
|
1434
|
-
const request = store.delete(key);
|
|
1435
|
-
|
|
1436
|
-
request.onsuccess = () => resolve();
|
|
1437
|
-
request.onerror = () => reject(request.error);
|
|
1438
|
-
});
|
|
1439
|
-
}
|
|
1440
|
-
|
|
1441
|
-
async has(key: string): Promise<boolean> {
|
|
1442
|
-
return (await this.get(key)) !== null;
|
|
1443
|
-
}
|
|
1444
|
-
|
|
1445
|
-
async clear(): Promise<void> {
|
|
1446
|
-
const db = await this.init();
|
|
1447
|
-
return new Promise((resolve, reject) => {
|
|
1448
|
-
const tx = db.transaction('store', 'readwrite');
|
|
1449
|
-
const store = tx.objectStore('store');
|
|
1450
|
-
const request = store.clear();
|
|
1451
|
-
|
|
1452
|
-
request.onsuccess = () => resolve();
|
|
1453
|
-
request.onerror = () => reject(request.error);
|
|
1454
|
-
});
|
|
1455
|
-
}
|
|
1456
|
-
}
|
|
1457
|
-
```
|
|
1458
|
-
|
|
1459
|
-
---
|
|
1460
|
-
|
|
1461
|
-
## 9. Storage Lifecycle
|
|
1462
|
-
|
|
1463
|
-
### 9.1. TTL Strategies
|
|
1464
|
-
|
|
1465
|
-
#### Eager eviction (remove on access)
|
|
1466
|
-
|
|
1467
|
-
```typescript
|
|
1468
|
-
// Remove item imediatamente quando expirado
|
|
1469
|
-
class EagerTTLCache<T> {
|
|
1470
|
-
private store = new Map<string, { value: T; expires: number }>();
|
|
1471
|
-
|
|
1472
|
-
get(key: string): T | null {
|
|
1473
|
-
const entry = this.store.get(key);
|
|
1474
|
-
if (!entry) return null;
|
|
1475
|
-
|
|
1476
|
-
if (Date.now() > entry.expires) {
|
|
1477
|
-
this.store.delete(key); // Eager: remove agora
|
|
1478
|
-
return null;
|
|
1479
|
-
}
|
|
1480
|
-
|
|
1481
|
-
return entry.value;
|
|
1482
|
-
}
|
|
1483
|
-
|
|
1484
|
-
set(key: string, value: T, ttlMs: number): void {
|
|
1485
|
-
this.store.set(key, {
|
|
1486
|
-
value,
|
|
1487
|
-
expires: Date.now() + ttlMs,
|
|
1488
|
-
});
|
|
1489
|
-
}
|
|
1490
|
-
}
|
|
1491
|
-
```
|
|
1492
|
-
|
|
1493
|
-
#### Lazy eviction (remove in background)
|
|
1494
|
-
|
|
1495
|
-
```typescript
|
|
1496
|
-
// Remove expiradas em background interval
|
|
1497
|
-
class LazyTTLCache<T> {
|
|
1498
|
-
private store = new Map<string, { value: T; expires: number }>();
|
|
1499
|
-
private cleanupInterval: ReturnType<typeof setInterval>;
|
|
1500
|
-
|
|
1501
|
-
constructor(cleanupMs = 60_000) {
|
|
1502
|
-
this.cleanupInterval = setInterval(() => {
|
|
1503
|
-
this.cleanup();
|
|
1504
|
-
}, cleanupMs);
|
|
1505
|
-
}
|
|
1506
|
-
|
|
1507
|
-
private cleanup(): void {
|
|
1508
|
-
const now = Date.now();
|
|
1509
|
-
for (const [key, entry] of this.store) {
|
|
1510
|
-
if (now > entry.expires) {
|
|
1511
|
-
this.store.delete(key);
|
|
1512
|
-
}
|
|
1513
|
-
}
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
destroy(): void {
|
|
1517
|
-
clearInterval(this.cleanupInterval);
|
|
1518
|
-
}
|
|
1519
|
-
|
|
1520
|
-
get(key: string): T | null {
|
|
1521
|
-
const entry = this.store.get(key);
|
|
1522
|
-
return entry?.value ?? null;
|
|
1523
|
-
}
|
|
1524
|
-
|
|
1525
|
-
set(key: string, value: T, ttlMs: number): void {
|
|
1526
|
-
this.store.set(key, {
|
|
1527
|
-
value,
|
|
1528
|
-
expires: Date.now() + ttlMs,
|
|
1529
|
-
});
|
|
1530
|
-
}
|
|
1531
|
-
}
|
|
1532
|
-
```
|
|
1533
|
-
|
|
1534
|
-
#### Scheduled eviction (event-driven)
|
|
1535
|
-
|
|
1536
|
-
```typescript
|
|
1537
|
-
// Agenda limpeza baseada no próximo item que expira
|
|
1538
|
-
class ScheduledTTLCache<T> {
|
|
1539
|
-
private store = new Map<string, { value: T; expires: number }>();
|
|
1540
|
-
private nextCleanup: ReturnType<typeof setTimeout> | null = null;
|
|
1541
|
-
|
|
1542
|
-
set(key: string, value: T, ttlMs: number): void {
|
|
1543
|
-
const expires = Date.now() + ttlMs;
|
|
1544
|
-
this.store.set(key, { value, expires });
|
|
1545
|
-
|
|
1546
|
-
// Agendar cleanup se este expira antes do próximo
|
|
1547
|
-
if (!this.nextCleanup || expires < this.nextCleanupAt) {
|
|
1548
|
-
this.scheduleCleanup(expires);
|
|
1549
|
-
}
|
|
1550
|
-
}
|
|
1551
|
-
|
|
1552
|
-
get(key: string): T | null {
|
|
1553
|
-
return this.store.get(key)?.value ?? null;
|
|
1554
|
-
}
|
|
1555
|
-
|
|
1556
|
-
private nextCleanupAt = 0;
|
|
1557
|
-
|
|
1558
|
-
private scheduleCleanup(at: number): void {
|
|
1559
|
-
if (this.nextCleanup) clearTimeout(this.nextCleanup);
|
|
1560
|
-
this.nextCleanupAt = at;
|
|
1561
|
-
this.nextCleanup = setTimeout(() => {
|
|
1562
|
-
this.cleanup();
|
|
1563
|
-
}, Math.max(0, at - Date.now()));
|
|
1564
|
-
}
|
|
1565
|
-
|
|
1566
|
-
private cleanup(): void {
|
|
1567
|
-
const now = Date.now();
|
|
1568
|
-
for (const [key, entry] of this.store) {
|
|
1569
|
-
if (now > entry.expires) {
|
|
1570
|
-
this.store.delete(key);
|
|
1571
|
-
}
|
|
1572
|
-
}
|
|
1573
|
-
|
|
1574
|
-
// Reagendar baseado no próximo item
|
|
1575
|
-
let nextExpiry = Infinity;
|
|
1576
|
-
for (const entry of this.store.values()) {
|
|
1577
|
-
if (entry.expires < nextExpiry) {
|
|
1578
|
-
nextExpiry = entry.expires;
|
|
1579
|
-
}
|
|
1580
|
-
}
|
|
1581
|
-
if (nextExpiry < Infinity) {
|
|
1582
|
-
this.scheduleCleanup(nextExpiry);
|
|
1583
|
-
}
|
|
1584
|
-
}
|
|
1585
|
-
}
|
|
1586
|
-
```
|
|
1587
|
-
|
|
1588
|
-
### 9.2. Eviction Strategies
|
|
1589
|
-
|
|
1590
|
-
```typescript
|
|
1591
|
-
// LRU — Least Recently Used
|
|
1592
|
-
class LRUCache<T> {
|
|
1593
|
-
private store = new Map<string, { value: T; lastAccess: number }>();
|
|
1594
|
-
private maxSize: number;
|
|
1595
|
-
|
|
1596
|
-
constructor(maxSize = 100) {
|
|
1597
|
-
this.maxSize = maxSize;
|
|
1598
|
-
}
|
|
1599
|
-
|
|
1600
|
-
get(key: string): T | null {
|
|
1601
|
-
const entry = this.store.get(key);
|
|
1602
|
-
if (!entry) return null;
|
|
1603
|
-
entry.lastAccess = Date.now();
|
|
1604
|
-
return entry.value;
|
|
1605
|
-
}
|
|
1606
|
-
|
|
1607
|
-
set(key: string, value: T): void {
|
|
1608
|
-
if (this.store.size >= this.maxSize) {
|
|
1609
|
-
this.evict();
|
|
1610
|
-
}
|
|
1611
|
-
this.store.set(key, { value, lastAccess: Date.now() });
|
|
1612
|
-
}
|
|
1613
|
-
|
|
1614
|
-
private evict(): void {
|
|
1615
|
-
let oldest: string | null = null;
|
|
1616
|
-
let oldestTime = Infinity;
|
|
1617
|
-
|
|
1618
|
-
for (const [key, entry] of this.store) {
|
|
1619
|
-
if (entry.lastAccess < oldestTime) {
|
|
1620
|
-
oldestTime = entry.lastAccess;
|
|
1621
|
-
oldest = key;
|
|
1622
|
-
}
|
|
1623
|
-
}
|
|
1624
|
-
|
|
1625
|
-
if (oldest) this.store.delete(oldest);
|
|
1626
|
-
}
|
|
1627
|
-
}
|
|
1628
|
-
|
|
1629
|
-
// LFU — Least Frequently Used
|
|
1630
|
-
class LFUCache<T> {
|
|
1631
|
-
private store = new Map<string, { value: T; count: number }>();
|
|
1632
|
-
private maxSize: number;
|
|
1633
|
-
|
|
1634
|
-
constructor(maxSize = 100) {
|
|
1635
|
-
this.maxSize = maxSize;
|
|
1636
|
-
}
|
|
1637
|
-
|
|
1638
|
-
get(key: string): T | null {
|
|
1639
|
-
const entry = this.store.get(key);
|
|
1640
|
-
if (!entry) return null;
|
|
1641
|
-
entry.count++;
|
|
1642
|
-
return entry.value;
|
|
1643
|
-
}
|
|
1644
|
-
|
|
1645
|
-
set(key: string, value: T): void {
|
|
1646
|
-
if (this.store.size >= this.maxSize) {
|
|
1647
|
-
this.evict();
|
|
1648
|
-
}
|
|
1649
|
-
this.store.set(key, { value, count: 0 });
|
|
1650
|
-
}
|
|
1651
|
-
|
|
1652
|
-
private evict(): void {
|
|
1653
|
-
let leastUsed: string | null = null;
|
|
1654
|
-
let minCount = Infinity;
|
|
1655
|
-
|
|
1656
|
-
for (const [key, entry] of this.store) {
|
|
1657
|
-
if (entry.count < minCount) {
|
|
1658
|
-
minCount = entry.count;
|
|
1659
|
-
leastUsed = key;
|
|
1660
|
-
}
|
|
1661
|
-
}
|
|
1662
|
-
|
|
1663
|
-
if (leastUsed) this.store.delete(leastUsed);
|
|
1664
|
-
}
|
|
1665
|
-
}
|
|
1666
|
-
|
|
1667
|
-
// FIFO — First In First Out
|
|
1668
|
-
class FIFOCache<T> {
|
|
1669
|
-
private store = new Map<string, T>();
|
|
1670
|
-
private maxSize: number;
|
|
1671
|
-
|
|
1672
|
-
constructor(maxSize = 100) {
|
|
1673
|
-
this.maxSize = maxSize;
|
|
1674
|
-
}
|
|
1675
|
-
|
|
1676
|
-
get(key: string): T | null {
|
|
1677
|
-
return this.store.get(key) ?? null;
|
|
1678
|
-
}
|
|
1679
|
-
|
|
1680
|
-
set(key: string, value: T): void {
|
|
1681
|
-
if (this.store.size >= this.maxSize) {
|
|
1682
|
-
// Remove a primeira entrada (mais antiga)
|
|
1683
|
-
const firstKey = this.store.keys().next().value;
|
|
1684
|
-
if (firstKey) this.store.delete(firstKey);
|
|
1685
|
-
}
|
|
1686
|
-
this.store.set(key, value);
|
|
1687
|
-
}
|
|
1688
|
-
}
|
|
1689
|
-
|
|
1690
|
-
// Size-based eviction
|
|
1691
|
-
class SizeBasedCache<T> {
|
|
1692
|
-
private store = new Map<string, { value: T; size: number }>();
|
|
1693
|
-
private maxSize: number;
|
|
1694
|
-
private currentSize = 0;
|
|
1695
|
-
|
|
1696
|
-
constructor(maxSize: number) {
|
|
1697
|
-
this.maxSize = maxSize;
|
|
1698
|
-
}
|
|
1699
|
-
|
|
1700
|
-
get(key: string): T | null {
|
|
1701
|
-
return this.store.get(key)?.value ?? null;
|
|
1702
|
-
}
|
|
1703
|
-
|
|
1704
|
-
set(key: string, value: T, sizeFn: (v: T) => number): void {
|
|
1705
|
-
const size = sizeFn(value);
|
|
1706
|
-
|
|
1707
|
-
// Remove entradas até caber
|
|
1708
|
-
while (this.currentSize + size > this.maxSize && this.store.size > 0) {
|
|
1709
|
-
const firstKey = this.store.keys().next().value;
|
|
1710
|
-
if (firstKey) {
|
|
1711
|
-
this.currentSize -= this.store.get(firstKey)!.size;
|
|
1712
|
-
this.store.delete(firstKey);
|
|
1713
|
-
}
|
|
1714
|
-
}
|
|
1715
|
-
|
|
1716
|
-
this.store.set(key, { value, size });
|
|
1717
|
-
this.currentSize += size;
|
|
1718
|
-
}
|
|
1719
|
-
}
|
|
1720
|
-
```
|
|
1721
|
-
|
|
1722
|
-
---
|
|
1723
|
-
|
|
1724
|
-
## 10. Best Practices
|
|
1725
|
-
|
|
1726
|
-
### Separar stores por volatilidade
|
|
1727
|
-
|
|
1728
|
-
```
|
|
1729
|
-
Cache Architecture:
|
|
1730
|
-
├── Hot Cache (in-memory, LRU, 1K items, TTL 5min)
|
|
1731
|
-
│ └── Requisições frequentes, dados que mudam rápido
|
|
1732
|
-
├── Warm Cache (SQLite/Redis, 100K items, TTL 1 hora)
|
|
1733
|
-
│ └── Dados semi-estáveis, sessões de usuário
|
|
1734
|
-
├── Cold Cache (IndexedDB/KV, unlimited, TTL 7 dias)
|
|
1735
|
-
│ └── Dados estáticos, checkpoints, logs
|
|
1736
|
-
└── Archive (Disk, unlimited, sem TTL)
|
|
1737
|
-
└── Resultados de scraping permanentes
|
|
1738
|
-
```
|
|
1739
|
-
|
|
1740
|
-
### Compression
|
|
1741
|
-
|
|
1742
|
-
```typescript
|
|
1743
|
-
// Comprimir dados grandes antes de armazenar
|
|
1744
|
-
async function compress(data: string): Promise<Uint8Array> {
|
|
1745
|
-
const stream = new Blob([data]).stream();
|
|
1746
|
-
const compressed = stream.pipeThrough(new CompressionStream('gzip'));
|
|
1747
|
-
return new Uint8Array(await new Response(compressed).arrayBuffer());
|
|
1748
|
-
}
|
|
1749
|
-
|
|
1750
|
-
async function decompress(data: Uint8Array): Promise<string> {
|
|
1751
|
-
const stream = new Blob([data]).stream();
|
|
1752
|
-
const decompressed = stream.pipeThrough(new DecompressionStream('gzip'));
|
|
1753
|
-
return new Response(decompressed).text();
|
|
1754
|
-
}
|
|
1755
|
-
|
|
1756
|
-
// Uso
|
|
1757
|
-
const largeData = JSON.stringify(bigObject);
|
|
1758
|
-
const compressed = await compress(largeData);
|
|
1759
|
-
await keyv.set('key', compressed);
|
|
1760
|
-
// Reduz 60-80% do tamanho
|
|
1761
|
-
```
|
|
1762
|
-
|
|
1763
|
-
### Atomic writes
|
|
1764
|
-
|
|
1765
|
-
```typescript
|
|
1766
|
-
// Usar transações para writes atômicos
|
|
1767
|
-
async function atomicWrite<T>(
|
|
1768
|
-
key: string,
|
|
1769
|
-
value: T,
|
|
1770
|
-
storage: UniversalStorage
|
|
1771
|
-
): Promise<void> {
|
|
1772
|
-
// Write temporário
|
|
1773
|
-
const tempKey = `${key}:temp:${Date.now()}`;
|
|
1774
|
-
await storage.set(tempKey, value);
|
|
1775
|
-
|
|
1776
|
-
// Rename atômico (não existe em todos os storages)
|
|
1777
|
-
// Para Keyv: usar transação do adapter
|
|
1778
|
-
const existing = await storage.get(key);
|
|
1779
|
-
try {
|
|
1780
|
-
await storage.set(key, value);
|
|
1781
|
-
await storage.delete(tempKey);
|
|
1782
|
-
} catch (error) {
|
|
1783
|
-
// Rollback
|
|
1784
|
-
if (existing) await storage.set(key, existing);
|
|
1785
|
-
await storage.delete(tempKey);
|
|
1786
|
-
throw error;
|
|
1787
|
-
}
|
|
1788
|
-
}
|
|
1789
|
-
```
|
|
1790
|
-
|
|
1791
|
-
### Graceful degradation
|
|
1792
|
-
|
|
1793
|
-
```typescript
|
|
1794
|
-
async function resilientGet<T>(key: string): Promise<T | null> {
|
|
1795
|
-
const stores: UniversalStorage[] = [
|
|
1796
|
-
memoryCache, // Rápido, volatile
|
|
1797
|
-
sqliteCache, // Persistente, síncrono
|
|
1798
|
-
redisCache, // Persistente, async
|
|
1799
|
-
];
|
|
1800
|
-
|
|
1801
|
-
for (const store of stores) {
|
|
1802
|
-
try {
|
|
1803
|
-
const value = await store.get<T>(key);
|
|
1804
|
-
if (value !== null) return value;
|
|
1805
|
-
} catch (error) {
|
|
1806
|
-
console.warn(`Storage ${store.constructor.name} failed:`, error);
|
|
1807
|
-
continue;
|
|
1808
|
-
}
|
|
1809
|
-
}
|
|
1810
|
-
|
|
1811
|
-
return null;
|
|
1812
|
-
}
|
|
1813
|
-
|
|
1814
|
-
async function resilientSet<T>(key: string, value: T, ttlMs?: number): Promise<void> {
|
|
1815
|
-
const stores: UniversalStorage[] = [
|
|
1816
|
-
memoryCache,
|
|
1817
|
-
sqliteCache,
|
|
1818
|
-
redisCache,
|
|
1819
|
-
];
|
|
1820
|
-
|
|
1821
|
-
for (const store of stores) {
|
|
1822
|
-
try {
|
|
1823
|
-
await store.set(key, value, ttlMs);
|
|
1824
|
-
} catch (error) {
|
|
1825
|
-
console.warn(`Storage ${store.constructor.name} write failed:`, error);
|
|
1826
|
-
continue;
|
|
1827
|
-
}
|
|
1828
|
-
}
|
|
1829
|
-
}
|
|
1830
|
-
```
|
|
1831
|
-
|
|
1832
|
-
---
|
|
1833
|
-
|
|
1834
|
-
## 11. Exemplos Completos
|
|
1835
|
-
|
|
1836
|
-
### 11.1. Multi-tier scraping cache
|
|
1837
|
-
|
|
1838
|
-
```typescript
|
|
1839
|
-
class MultiTierCache<T> {
|
|
1840
|
-
private hot: LRUCache<T>;
|
|
1841
|
-
private warm: Keyv<T>;
|
|
1842
|
-
private cold: UniversalStorage;
|
|
1843
|
-
|
|
1844
|
-
constructor() {
|
|
1845
|
-
this.hot = new LRUCache<T>({ max: 1000, ttl: 1000 * 60 * 5 });
|
|
1846
|
-
this.warm = new Keyv({
|
|
1847
|
-
namespace: 'warm',
|
|
1848
|
-
ttl: 1000 * 60 * 60,
|
|
1849
|
-
});
|
|
1850
|
-
this.cold = createStorage('cold');
|
|
1851
|
-
}
|
|
1852
|
-
|
|
1853
|
-
async get(key: string): Promise<T | null> {
|
|
1854
|
-
// Hot
|
|
1855
|
-
const hotValue = this.hot.get(key);
|
|
1856
|
-
if (hotValue) return hotValue;
|
|
1857
|
-
|
|
1858
|
-
// Warm
|
|
1859
|
-
const warmValue = await this.warm.get(key);
|
|
1860
|
-
if (warmValue) {
|
|
1861
|
-
this.hot.set(key, warmValue);
|
|
1862
|
-
return warmValue;
|
|
1863
|
-
}
|
|
1864
|
-
|
|
1865
|
-
// Cold
|
|
1866
|
-
const coldValue = await this.cold.get<T>(key);
|
|
1867
|
-
if (coldValue) {
|
|
1868
|
-
this.hot.set(key, coldValue);
|
|
1869
|
-
await this.warm.set(key, coldValue);
|
|
1870
|
-
return coldValue;
|
|
1871
|
-
}
|
|
1872
|
-
|
|
1873
|
-
return null;
|
|
1874
|
-
}
|
|
1875
|
-
|
|
1876
|
-
async set(key: string, value: T): Promise<void> {
|
|
1877
|
-
this.hot.set(key, value);
|
|
1878
|
-
await this.warm.set(key, value);
|
|
1879
|
-
await this.cold.set(key, value);
|
|
1880
|
-
}
|
|
1881
|
-
|
|
1882
|
-
async invalidate(key: string): Promise<void> {
|
|
1883
|
-
this.hot.delete(key);
|
|
1884
|
-
await this.warm.delete(key);
|
|
1885
|
-
await this.cold.delete(key);
|
|
1886
|
-
}
|
|
1887
|
-
}
|
|
1888
|
-
```
|
|
1889
|
-
|
|
1890
|
-
### 11.2. Persistent scraping session
|
|
1891
|
-
|
|
1892
|
-
```typescript
|
|
1893
|
-
class ScrapingSession {
|
|
1894
|
-
private checkpointMgr: CheckpointManager;
|
|
1895
|
-
private authMgr: AuthManager;
|
|
1896
|
-
private cache: MultiTierCache<unknown>;
|
|
1897
|
-
|
|
1898
|
-
constructor() {
|
|
1899
|
-
this.checkpointMgr = new CheckpointManager();
|
|
1900
|
-
this.authMgr = new AuthManager();
|
|
1901
|
-
this.cache = new MultiTierCache();
|
|
1902
|
-
}
|
|
1903
|
-
|
|
1904
|
-
async scrape(urls: string[]): Promise<void> {
|
|
1905
|
-
const jobId = await this.generateJobId(urls);
|
|
1906
|
-
|
|
1907
|
-
// Restaurar de checkpoint
|
|
1908
|
-
const checkpoint = await this.checkpointMgr.resume(jobId);
|
|
1909
|
-
const startIdx = checkpoint?.pagesScraped ?? 0;
|
|
1910
|
-
|
|
1911
|
-
for (let i = startIdx; i < urls.length; i++) {
|
|
1912
|
-
const url = urls[i];
|
|
1913
|
-
|
|
1914
|
-
// Verificar cache
|
|
1915
|
-
const cached = await this.cache.get(`scrape:${url}`);
|
|
1916
|
-
if (cached) {
|
|
1917
|
-
console.log(`Cache hit: ${url}`);
|
|
1918
|
-
continue;
|
|
1919
|
-
}
|
|
1920
|
-
|
|
1921
|
-
// Obter auth se necessário
|
|
1922
|
-
const token = await this.authMgr.getValidToken(url);
|
|
1923
|
-
|
|
1924
|
-
// Scrape
|
|
1925
|
-
try {
|
|
1926
|
-
const result = await fetch(url, {
|
|
1927
|
-
headers: token ? { Authorization: `Bearer ${token}` } : {},
|
|
1928
|
-
});
|
|
1929
|
-
|
|
1930
|
-
const data = await result.json();
|
|
1931
|
-
await this.cache.set(`scrape:${url}`, data);
|
|
1932
|
-
|
|
1933
|
-
// Checkpoint a cada 10 páginas
|
|
1934
|
-
if ((i + 1) % 10 === 0) {
|
|
1935
|
-
await this.checkpointMgr.save(jobId, {
|
|
1936
|
-
url,
|
|
1937
|
-
pagesScraped: i + 1,
|
|
1938
|
-
lastPage: url,
|
|
1939
|
-
extractedData: [],
|
|
1940
|
-
errors: [],
|
|
1941
|
-
timestamp: Date.now(),
|
|
1942
|
-
});
|
|
1943
|
-
}
|
|
1944
|
-
} catch (error) {
|
|
1945
|
-
console.error(`Failed: ${url}`, error);
|
|
1946
|
-
}
|
|
1947
|
-
}
|
|
1948
|
-
}
|
|
1949
|
-
|
|
1950
|
-
private async generateJobId(urls: string[]): Promise<string> {
|
|
1951
|
-
return crypto.subtle
|
|
1952
|
-
.digest('SHA-256', new TextEncoder().encode(urls.join('\n')))
|
|
1953
|
-
.then(buf => Array.from(new Uint8Array(buf))
|
|
1954
|
-
.map(b => b.toString(16).padStart(2, '0'))
|
|
1955
|
-
.join('')
|
|
1956
|
-
.slice(0, 16));
|
|
1957
|
-
}
|
|
1958
|
-
}
|
|
1959
|
-
```
|
|
1960
|
-
|
|
1961
|
-
---
|
|
1962
|
-
|
|
1963
|
-
## 12. Checklist
|
|
1964
|
-
|
|
1965
|
-
- [ ] Usar Map para caches simples, lru-cache para bounded
|
|
1966
|
-
- [ ] WeakMap para metadata associada a objects
|
|
1967
|
-
- [ ] Browser: IndexedDB para dados grandes, localStorage para pequenos
|
|
1968
|
-
- [ ] Server: better-sqlite3 para síncrono, Redis para distribuído
|
|
1969
|
-
- [ ] Cross-runtime: keyv 5.6.0 com adapters pluggable
|
|
1970
|
-
- [ ] Edge: Cloudflare KV, Upstash Redis, Deno KV
|
|
1971
|
-
- [ ] Request dedup com Keyv TTL ou Bloom filter
|
|
1972
|
-
- [ ] ETag conditional requests para freshness
|
|
1973
|
-
- [ ] Persistir: auth tokens, checkpoints, fingerprints
|
|
1974
|
-
- [ ] Separar stores por volatilidade (hot/warm/cold)
|
|
1975
|
-
- [ ] TTL: eager para hot, lazy para warm, scheduled para cold
|
|
1976
|
-
- [ ] Eviction: LRU para hot, size-based para warm
|
|
1977
|
-
- [ ] Compression para dados > 10KB
|
|
1978
|
-
- [ ] Graceful degradation: memory → SQLite → Redis
|
|
1979
|
-
- [ ] Atomic writes para consistência
|