@wenathlan/saddle 1.8.17 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1005) hide show
  1. package/Dockerfile +1339 -0
  2. package/README.md +68 -22
  3. package/alternatives.ts +1218 -0
  4. package/biome.json +153 -0
  5. package/boards.json +1517 -0
  6. package/compute.ts +3144 -0
  7. package/cores.json +1801 -0
  8. package/dist/acquisition.d.ts +359 -0
  9. package/dist/acquisition.d.ts.map +1 -0
  10. package/dist/acquisition.js +629 -0
  11. package/dist/acquisition.js.map +1 -0
  12. package/dist/alternatives.d.ts +509 -0
  13. package/dist/alternatives.d.ts.map +1 -0
  14. package/dist/alternatives.js +951 -0
  15. package/dist/alternatives.js.map +1 -0
  16. package/dist/automation.d.ts +426 -0
  17. package/dist/automation.d.ts.map +1 -0
  18. package/dist/automation.js +561 -0
  19. package/dist/automation.js.map +1 -0
  20. package/dist/browser.d.ts +562 -0
  21. package/dist/browser.d.ts.map +1 -0
  22. package/dist/browser.js +1057 -0
  23. package/dist/browser.js.map +1 -0
  24. package/dist/capacitor.config.d.ts +13 -0
  25. package/dist/capacitor.config.d.ts.map +1 -0
  26. package/dist/capacitor.config.js +26 -0
  27. package/dist/capacitor.config.js.map +1 -0
  28. package/dist/cli.d.ts +17 -0
  29. package/dist/cli.d.ts.map +1 -0
  30. package/dist/cli.js +111 -0
  31. package/dist/cli.js.map +1 -0
  32. package/dist/communication.d.ts +192 -0
  33. package/dist/communication.d.ts.map +1 -0
  34. package/dist/communication.js +418 -0
  35. package/dist/communication.js.map +1 -0
  36. package/dist/compute.d.ts +1161 -0
  37. package/dist/compute.d.ts.map +1 -0
  38. package/dist/compute.js +2273 -0
  39. package/dist/compute.js.map +1 -0
  40. package/dist/distribution.d.ts +529 -0
  41. package/dist/distribution.d.ts.map +1 -0
  42. package/dist/distribution.js +811 -0
  43. package/dist/distribution.js.map +1 -0
  44. package/dist/execution.d.ts +530 -0
  45. package/dist/execution.d.ts.map +1 -0
  46. package/dist/execution.js +761 -0
  47. package/dist/execution.js.map +1 -0
  48. package/dist/format.d.ts +19 -0
  49. package/dist/format.d.ts.map +1 -0
  50. package/dist/format.js +60 -0
  51. package/dist/format.js.map +1 -0
  52. package/dist/foundation.d.ts +176 -0
  53. package/dist/foundation.d.ts.map +1 -0
  54. package/dist/foundation.js +309 -0
  55. package/dist/foundation.js.map +1 -0
  56. package/dist/index.d.ts +733 -122
  57. package/dist/index.d.ts.map +1 -1
  58. package/dist/index.js +1018 -122
  59. package/dist/index.js.map +1 -1
  60. package/dist/integration.d.ts +236 -0
  61. package/dist/integration.d.ts.map +1 -0
  62. package/dist/integration.js +236 -0
  63. package/dist/integration.js.map +1 -0
  64. package/dist/intelligence.d.ts +94 -0
  65. package/dist/intelligence.d.ts.map +1 -0
  66. package/dist/intelligence.js +145 -0
  67. package/dist/intelligence.js.map +1 -0
  68. package/dist/isolation.d.ts +129 -0
  69. package/dist/isolation.d.ts.map +1 -0
  70. package/dist/isolation.js +96 -0
  71. package/dist/isolation.js.map +1 -0
  72. package/dist/media.d.ts +344 -0
  73. package/dist/media.d.ts.map +1 -0
  74. package/dist/media.js +986 -0
  75. package/dist/media.js.map +1 -0
  76. package/dist/modes.d.ts +359 -0
  77. package/dist/modes.d.ts.map +1 -0
  78. package/dist/modes.js +134 -0
  79. package/dist/modes.js.map +1 -0
  80. package/dist/{library/public.d.ts → operations.d.ts} +15 -1
  81. package/dist/operations.d.ts.map +1 -0
  82. package/dist/{library/public.js → operations.js} +18 -7
  83. package/dist/operations.js.map +1 -0
  84. package/dist/orchestrator.d.ts +1339 -0
  85. package/dist/orchestrator.d.ts.map +1 -0
  86. package/dist/orchestrator.js +4308 -0
  87. package/dist/orchestrator.js.map +1 -0
  88. package/dist/performance.d.ts +416 -0
  89. package/dist/performance.d.ts.map +1 -0
  90. package/dist/performance.js +827 -0
  91. package/dist/performance.js.map +1 -0
  92. package/dist/quantum.d.ts +810 -0
  93. package/dist/quantum.d.ts.map +1 -0
  94. package/dist/quantum.js +2073 -0
  95. package/dist/quantum.js.map +1 -0
  96. package/dist/render.d.ts +381 -0
  97. package/dist/render.d.ts.map +1 -0
  98. package/dist/render.js +1073 -0
  99. package/dist/render.js.map +1 -0
  100. package/dist/scheduler.d.ts +596 -0
  101. package/dist/scheduler.d.ts.map +1 -0
  102. package/dist/scheduler.js +1259 -0
  103. package/dist/scheduler.js.map +1 -0
  104. package/dist/security.d.ts +501 -0
  105. package/dist/security.d.ts.map +1 -0
  106. package/dist/security.js +1170 -0
  107. package/dist/security.js.map +1 -0
  108. package/dist/server.d.ts +21 -0
  109. package/dist/server.d.ts.map +1 -0
  110. package/dist/{server/node.js → server.js} +16 -2
  111. package/dist/server.js.map +1 -0
  112. package/dist/tiers.d.ts +1002 -0
  113. package/dist/tiers.d.ts.map +1 -0
  114. package/dist/tiers.js +2165 -0
  115. package/dist/tiers.js.map +1 -0
  116. package/dist/virtual.d.ts +641 -0
  117. package/dist/virtual.d.ts.map +1 -0
  118. package/dist/virtual.js +1285 -0
  119. package/dist/virtual.js.map +1 -0
  120. package/dist/virtualcpu.d.ts +316 -0
  121. package/dist/virtualcpu.d.ts.map +1 -0
  122. package/dist/virtualcpu.js +1180 -0
  123. package/dist/virtualcpu.js.map +1 -0
  124. package/dist/virtualgpu.d.ts +345 -0
  125. package/dist/virtualgpu.d.ts.map +1 -0
  126. package/dist/virtualgpu.js +1000 -0
  127. package/dist/virtualgpu.js.map +1 -0
  128. package/dist/virtualization.d.ts +607 -0
  129. package/dist/virtualization.d.ts.map +1 -0
  130. package/dist/virtualization.js +1187 -0
  131. package/dist/virtualization.js.map +1 -0
  132. package/dist/virtualmemory.d.ts +818 -0
  133. package/dist/virtualmemory.d.ts.map +1 -0
  134. package/dist/virtualmemory.js +1796 -0
  135. package/dist/virtualmemory.js.map +1 -0
  136. package/dist/webscrape.d.ts +720 -0
  137. package/dist/webscrape.d.ts.map +1 -0
  138. package/dist/webscrape.js +2697 -0
  139. package/dist/webscrape.js.map +1 -0
  140. package/docker.config +769 -0
  141. package/docs/CONVERSA.txt +1523 -0
  142. package/docs/alternatives.md +499 -0
  143. package/docs/architecture-1.8.19.md +76 -0
  144. package/docs/architecture-1.8.19.mmd +33 -0
  145. package/docs/architecture-1.8.19.png +0 -0
  146. package/docs/architecture.md +355 -0
  147. package/docs/artifactavailability.md +6 -0
  148. package/docs/brancharchive-2026-08-18.md +18 -0
  149. package/docs/browser.md +7 -0
  150. package/docs/consolidation.md +70 -0
  151. package/docs/e2ugh-engine.md +528 -0
  152. package/{dist/examples/localjob.js → docs/example-localjob.ts} +3 -3
  153. package/{dist/examples/publicapi.js → docs/example-publicapi.ts} +2 -2
  154. package/docs/hardware.md +342 -0
  155. package/docs/optimization.md +126 -0
  156. package/docs/performance.md +499 -0
  157. package/docs/planning.1.8.18.md +94 -0
  158. package/docs/planning.1.8.19.md +49 -0
  159. package/docs/releasenotes-1.8.17.md +6 -0
  160. package/docs/releasenotes-1.8.18.md +32 -0
  161. package/docs/releasenotes-1.8.19.md +35 -0
  162. package/docs/research-1.8.18-isolation.md +43 -0
  163. package/docs/research-1.8.19-virtual-browser.md +27 -0
  164. package/docs/saddle.archive.1.8.17.tar.gz.gpg +0 -0
  165. package/docs/security.md +454 -0
  166. package/docs/todo-1.8.16.md +56 -1
  167. package/docs/todo-1.8.18.md +164 -0
  168. package/docs/todo-1.8.19.md +241 -0
  169. package/docs/viability.md +777 -0
  170. package/docs/virtualization.md +335 -0
  171. package/docs/web-duplicatecleanup.md +9 -0
  172. package/docs/web-ideas.md +106 -0
  173. package/docs/workflowimprovements-2026-08-19.md +70 -0
  174. package/gpumonitor.cpp +1543 -0
  175. package/gpus.json +2408 -0
  176. package/index.ts +1478 -0
  177. package/media.ts +1337 -0
  178. package/mttg.config +1549 -0
  179. package/orchestrator.ts +5462 -0
  180. package/package.json +304 -55
  181. package/passage.config +1331 -0
  182. package/performance.ts +1049 -0
  183. package/processors.json +2217 -0
  184. package/qemu.config +1062 -0
  185. package/qemubridge.py +1340 -0
  186. package/quantum.ts +2393 -0
  187. package/render.ts +1325 -0
  188. package/scheduler.ts +1600 -0
  189. package/security.ts +1449 -0
  190. package/tiers.ts +2871 -0
  191. package/tsconfig.json +29 -0
  192. package/virtualcpu.ts +1273 -0
  193. package/virtualgpu.ts +1181 -0
  194. package/virtualhardware.c +1011 -0
  195. package/virtualhardware.json +739 -0
  196. package/virtualization.ts +1628 -0
  197. package/virtualizationcore.cpp +4635 -0
  198. package/virtualmemory.ts +2303 -0
  199. package/vm.config.json +1910 -0
  200. package/{extension → web/extension}/manifest.json +1 -1
  201. package/web/readme.md +382 -0
  202. package/web/tsconfig.json +25 -0
  203. package/dist/adapters/forge.d.ts +0 -17
  204. package/dist/adapters/forge.d.ts.map +0 -1
  205. package/dist/adapters/forge.js +0 -19
  206. package/dist/adapters/forge.js.map +0 -1
  207. package/dist/adapters/forgejo.d.ts +0 -49
  208. package/dist/adapters/forgejo.d.ts.map +0 -1
  209. package/dist/adapters/forgejo.js +0 -8
  210. package/dist/adapters/forgejo.js.map +0 -1
  211. package/dist/adapters/github.d.ts +0 -12
  212. package/dist/adapters/github.d.ts.map +0 -1
  213. package/dist/adapters/github.js +0 -20
  214. package/dist/adapters/github.js.map +0 -1
  215. package/dist/adapters/gitlab.d.ts +0 -17
  216. package/dist/adapters/gitlab.d.ts.map +0 -1
  217. package/dist/adapters/gitlab.js +0 -10
  218. package/dist/adapters/gitlab.js.map +0 -1
  219. package/dist/adapters/huggingface.d.ts +0 -17
  220. package/dist/adapters/huggingface.d.ts.map +0 -1
  221. package/dist/adapters/huggingface.js +0 -6
  222. package/dist/adapters/huggingface.js.map +0 -1
  223. package/dist/adapters/socket.d.ts +0 -11
  224. package/dist/adapters/socket.d.ts.map +0 -1
  225. package/dist/adapters/socket.js +0 -17
  226. package/dist/adapters/socket.js.map +0 -1
  227. package/dist/adapters/transport.d.ts +0 -7
  228. package/dist/adapters/transport.d.ts.map +0 -1
  229. package/dist/adapters/transport.js +0 -36
  230. package/dist/adapters/transport.js.map +0 -1
  231. package/dist/ai/chunk.d.ts +0 -2
  232. package/dist/ai/chunk.d.ts.map +0 -1
  233. package/dist/ai/chunk.js +0 -36
  234. package/dist/ai/chunk.js.map +0 -1
  235. package/dist/ai/llmstxt.d.ts +0 -6
  236. package/dist/ai/llmstxt.d.ts.map +0 -1
  237. package/dist/ai/llmstxt.js +0 -12
  238. package/dist/ai/llmstxt.js.map +0 -1
  239. package/dist/ai/provenance.d.ts +0 -22
  240. package/dist/ai/provenance.d.ts.map +0 -1
  241. package/dist/ai/provenance.js +0 -25
  242. package/dist/ai/provenance.js.map +0 -1
  243. package/dist/ai/rag.d.ts +0 -26
  244. package/dist/ai/rag.d.ts.map +0 -1
  245. package/dist/ai/rag.js +0 -22
  246. package/dist/ai/rag.js.map +0 -1
  247. package/dist/ai/tokens.d.ts +0 -15
  248. package/dist/ai/tokens.d.ts.map +0 -1
  249. package/dist/ai/tokens.js +0 -9
  250. package/dist/ai/tokens.js.map +0 -1
  251. package/dist/api/auth.d.ts +0 -16
  252. package/dist/api/auth.d.ts.map +0 -1
  253. package/dist/api/auth.js +0 -19
  254. package/dist/api/auth.js.map +0 -1
  255. package/dist/api/contracts.d.ts +0 -28
  256. package/dist/api/contracts.d.ts.map +0 -1
  257. package/dist/api/contracts.js +0 -14
  258. package/dist/api/contracts.js.map +0 -1
  259. package/dist/api/control.d.ts +0 -10
  260. package/dist/api/control.d.ts.map +0 -1
  261. package/dist/api/control.js +0 -35
  262. package/dist/api/control.js.map +0 -1
  263. package/dist/api/http.d.ts +0 -7
  264. package/dist/api/http.d.ts.map +0 -1
  265. package/dist/api/http.js +0 -12
  266. package/dist/api/http.js.map +0 -1
  267. package/dist/api/rate.d.ts +0 -28
  268. package/dist/api/rate.d.ts.map +0 -1
  269. package/dist/api/rate.js +0 -36
  270. package/dist/api/rate.js.map +0 -1
  271. package/dist/api/security.d.ts +0 -12
  272. package/dist/api/security.d.ts.map +0 -1
  273. package/dist/api/security.js +0 -56
  274. package/dist/api/security.js.map +0 -1
  275. package/dist/api/service.d.ts +0 -5
  276. package/dist/api/service.d.ts.map +0 -1
  277. package/dist/api/service.js +0 -70
  278. package/dist/api/service.js.map +0 -1
  279. package/dist/apps/registry.d.ts +0 -33
  280. package/dist/apps/registry.d.ts.map +0 -1
  281. package/dist/apps/registry.js +0 -24
  282. package/dist/apps/registry.js.map +0 -1
  283. package/dist/binary/archive.d.ts +0 -27
  284. package/dist/binary/archive.d.ts.map +0 -1
  285. package/dist/binary/archive.js +0 -46
  286. package/dist/binary/archive.js.map +0 -1
  287. package/dist/binary/build.d.ts +0 -29
  288. package/dist/binary/build.d.ts.map +0 -1
  289. package/dist/binary/build.js +0 -17
  290. package/dist/binary/build.js.map +0 -1
  291. package/dist/binary/transform.d.ts +0 -70
  292. package/dist/binary/transform.d.ts.map +0 -1
  293. package/dist/binary/transform.js +0 -105
  294. package/dist/binary/transform.js.map +0 -1
  295. package/dist/bot/adapter.d.ts +0 -5
  296. package/dist/bot/adapter.d.ts.map +0 -1
  297. package/dist/bot/adapter.js +0 -11
  298. package/dist/bot/adapter.js.map +0 -1
  299. package/dist/bot/bot.d.ts +0 -28
  300. package/dist/bot/bot.d.ts.map +0 -1
  301. package/dist/bot/bot.js +0 -52
  302. package/dist/bot/bot.js.map +0 -1
  303. package/dist/bot/commands.d.ts +0 -8
  304. package/dist/bot/commands.d.ts.map +0 -1
  305. package/dist/bot/commands.js +0 -19
  306. package/dist/bot/commands.js.map +0 -1
  307. package/dist/bot/permissions.d.ts +0 -13
  308. package/dist/bot/permissions.d.ts.map +0 -1
  309. package/dist/bot/permissions.js +0 -16
  310. package/dist/bot/permissions.js.map +0 -1
  311. package/dist/browser/actions.d.ts +0 -33
  312. package/dist/browser/actions.d.ts.map +0 -1
  313. package/dist/browser/actions.js +0 -44
  314. package/dist/browser/actions.js.map +0 -1
  315. package/dist/browser/agent.d.ts +0 -21
  316. package/dist/browser/agent.d.ts.map +0 -1
  317. package/dist/browser/agent.js +0 -13
  318. package/dist/browser/agent.js.map +0 -1
  319. package/dist/browser/context.d.ts +0 -43
  320. package/dist/browser/context.d.ts.map +0 -1
  321. package/dist/browser/context.js +0 -54
  322. package/dist/browser/context.js.map +0 -1
  323. package/dist/browser/fingerprint.d.ts +0 -4
  324. package/dist/browser/fingerprint.d.ts.map +0 -1
  325. package/dist/browser/fingerprint.js +0 -12
  326. package/dist/browser/fingerprint.js.map +0 -1
  327. package/dist/browser/index.d.ts +0 -11
  328. package/dist/browser/index.d.ts.map +0 -1
  329. package/dist/browser/index.js +0 -11
  330. package/dist/browser/index.js.map +0 -1
  331. package/dist/browser/playwright.d.ts +0 -11
  332. package/dist/browser/playwright.d.ts.map +0 -1
  333. package/dist/browser/playwright.js +0 -24
  334. package/dist/browser/playwright.js.map +0 -1
  335. package/dist/browser/recorder.d.ts +0 -21
  336. package/dist/browser/recorder.d.ts.map +0 -1
  337. package/dist/browser/recorder.js +0 -58
  338. package/dist/browser/recorder.js.map +0 -1
  339. package/dist/browser/session.d.ts +0 -14
  340. package/dist/browser/session.d.ts.map +0 -1
  341. package/dist/browser/session.js +0 -21
  342. package/dist/browser/session.js.map +0 -1
  343. package/dist/browser/snapshot.d.ts +0 -69
  344. package/dist/browser/snapshot.d.ts.map +0 -1
  345. package/dist/browser/snapshot.js +0 -133
  346. package/dist/browser/snapshot.js.map +0 -1
  347. package/dist/captcha/contract.d.ts +0 -35
  348. package/dist/captcha/contract.d.ts.map +0 -1
  349. package/dist/captcha/contract.js +0 -17
  350. package/dist/captcha/contract.js.map +0 -1
  351. package/dist/captcha/evidence.d.ts +0 -10
  352. package/dist/captcha/evidence.d.ts.map +0 -1
  353. package/dist/captcha/evidence.js +0 -9
  354. package/dist/captcha/evidence.js.map +0 -1
  355. package/dist/captcha/guard.d.ts +0 -23
  356. package/dist/captcha/guard.d.ts.map +0 -1
  357. package/dist/captcha/guard.js +0 -12
  358. package/dist/captcha/guard.js.map +0 -1
  359. package/dist/cli/main.d.ts +0 -4
  360. package/dist/cli/main.d.ts.map +0 -1
  361. package/dist/cli/main.js +0 -46
  362. package/dist/cli/main.js.map +0 -1
  363. package/dist/core/errors.d.ts +0 -84
  364. package/dist/core/errors.d.ts.map +0 -1
  365. package/dist/core/errors.js +0 -54
  366. package/dist/core/errors.js.map +0 -1
  367. package/dist/core/events.d.ts +0 -9
  368. package/dist/core/events.d.ts.map +0 -1
  369. package/dist/core/events.js +0 -21
  370. package/dist/core/events.js.map +0 -1
  371. package/dist/core/hash.d.ts +0 -10
  372. package/dist/core/hash.d.ts.map +0 -1
  373. package/dist/core/hash.js +0 -82
  374. package/dist/core/hash.js.map +0 -1
  375. package/dist/core/ids.d.ts +0 -10
  376. package/dist/core/ids.d.ts.map +0 -1
  377. package/dist/core/ids.js +0 -15
  378. package/dist/core/ids.js.map +0 -1
  379. package/dist/delivery/manifest.d.ts +0 -35
  380. package/dist/delivery/manifest.d.ts.map +0 -1
  381. package/dist/delivery/manifest.js +0 -68
  382. package/dist/delivery/manifest.js.map +0 -1
  383. package/dist/deploy/index.d.ts +0 -6
  384. package/dist/deploy/index.d.ts.map +0 -1
  385. package/dist/deploy/index.js +0 -6
  386. package/dist/deploy/index.js.map +0 -1
  387. package/dist/dispatch/resumable.d.ts +0 -102
  388. package/dist/dispatch/resumable.d.ts.map +0 -1
  389. package/dist/dispatch/resumable.js +0 -64
  390. package/dist/dispatch/resumable.js.map +0 -1
  391. package/dist/dispatch/workflow.d.ts +0 -6
  392. package/dist/dispatch/workflow.d.ts.map +0 -1
  393. package/dist/dispatch/workflow.js +0 -37
  394. package/dist/dispatch/workflow.js.map +0 -1
  395. package/dist/domain/artifacts.d.ts +0 -12
  396. package/dist/domain/artifacts.d.ts.map +0 -1
  397. package/dist/domain/artifacts.js +0 -14
  398. package/dist/domain/artifacts.js.map +0 -1
  399. package/dist/domain/jobs.d.ts +0 -12
  400. package/dist/domain/jobs.d.ts.map +0 -1
  401. package/dist/domain/jobs.js +0 -20
  402. package/dist/domain/jobs.js.map +0 -1
  403. package/dist/domain/providers.d.ts +0 -10
  404. package/dist/domain/providers.d.ts.map +0 -1
  405. package/dist/domain/providers.js +0 -8
  406. package/dist/domain/providers.js.map +0 -1
  407. package/dist/domain/runtime.d.ts +0 -14
  408. package/dist/domain/runtime.d.ts.map +0 -1
  409. package/dist/domain/runtime.js +0 -10
  410. package/dist/domain/runtime.js.map +0 -1
  411. package/dist/domain/sessions.d.ts +0 -12
  412. package/dist/domain/sessions.d.ts.map +0 -1
  413. package/dist/domain/sessions.js +0 -57
  414. package/dist/domain/sessions.js.map +0 -1
  415. package/dist/examples/localjob.d.ts +0 -2
  416. package/dist/examples/localjob.d.ts.map +0 -1
  417. package/dist/examples/localjob.js.map +0 -1
  418. package/dist/examples/publicapi.d.ts +0 -2
  419. package/dist/examples/publicapi.d.ts.map +0 -1
  420. package/dist/examples/publicapi.js.map +0 -1
  421. package/dist/extension/build.d.ts +0 -9
  422. package/dist/extension/build.d.ts.map +0 -1
  423. package/dist/extension/build.js +0 -85
  424. package/dist/extension/build.js.map +0 -1
  425. package/dist/extension/content.d.ts +0 -6
  426. package/dist/extension/content.d.ts.map +0 -1
  427. package/dist/extension/content.js +0 -135
  428. package/dist/extension/content.js.map +0 -1
  429. package/dist/extension/index.d.ts +0 -7
  430. package/dist/extension/index.d.ts.map +0 -1
  431. package/dist/extension/index.js +0 -7
  432. package/dist/extension/index.js.map +0 -1
  433. package/dist/extension/pagebridge.d.ts +0 -6
  434. package/dist/extension/pagebridge.d.ts.map +0 -1
  435. package/dist/extension/pagebridge.js +0 -42
  436. package/dist/extension/pagebridge.js.map +0 -1
  437. package/dist/extension/permissions.d.ts +0 -26
  438. package/dist/extension/permissions.d.ts.map +0 -1
  439. package/dist/extension/permissions.js +0 -28
  440. package/dist/extension/permissions.js.map +0 -1
  441. package/dist/extension/popup.d.ts +0 -5
  442. package/dist/extension/popup.d.ts.map +0 -1
  443. package/dist/extension/popup.js +0 -26
  444. package/dist/extension/popup.js.map +0 -1
  445. package/dist/extension/protocol.d.ts +0 -58
  446. package/dist/extension/protocol.d.ts.map +0 -1
  447. package/dist/extension/protocol.js +0 -101
  448. package/dist/extension/protocol.js.map +0 -1
  449. package/dist/extension/serviceworker.d.ts +0 -25
  450. package/dist/extension/serviceworker.d.ts.map +0 -1
  451. package/dist/extension/serviceworker.js +0 -102
  452. package/dist/extension/serviceworker.js.map +0 -1
  453. package/dist/extension/worker.d.ts +0 -28
  454. package/dist/extension/worker.d.ts.map +0 -1
  455. package/dist/extension/worker.js +0 -22
  456. package/dist/extension/worker.js.map +0 -1
  457. package/dist/format/check.d.ts +0 -8
  458. package/dist/format/check.d.ts.map +0 -1
  459. package/dist/format/check.js +0 -38
  460. package/dist/format/check.js.map +0 -1
  461. package/dist/library/public.d.ts.map +0 -1
  462. package/dist/library/public.js.map +0 -1
  463. package/dist/mcp/browser.d.ts +0 -9
  464. package/dist/mcp/browser.d.ts.map +0 -1
  465. package/dist/mcp/browser.js +0 -13
  466. package/dist/mcp/browser.js.map +0 -1
  467. package/dist/mcp/server.d.ts +0 -35
  468. package/dist/mcp/server.d.ts.map +0 -1
  469. package/dist/mcp/server.js +0 -40
  470. package/dist/mcp/server.js.map +0 -1
  471. package/dist/mcp/transport.d.ts +0 -7
  472. package/dist/mcp/transport.d.ts.map +0 -1
  473. package/dist/mcp/transport.js +0 -17
  474. package/dist/mcp/transport.js.map +0 -1
  475. package/dist/memory/bridge.d.ts +0 -14
  476. package/dist/memory/bridge.d.ts.map +0 -1
  477. package/dist/memory/bridge.js +0 -16
  478. package/dist/memory/bridge.js.map +0 -1
  479. package/dist/memory/engine.d.ts +0 -38
  480. package/dist/memory/engine.d.ts.map +0 -1
  481. package/dist/memory/engine.js +0 -109
  482. package/dist/memory/engine.js.map +0 -1
  483. package/dist/memory/modes.d.ts +0 -43
  484. package/dist/memory/modes.d.ts.map +0 -1
  485. package/dist/memory/modes.js +0 -66
  486. package/dist/memory/modes.js.map +0 -1
  487. package/dist/memory/objects.d.ts +0 -21
  488. package/dist/memory/objects.d.ts.map +0 -1
  489. package/dist/memory/objects.js +0 -20
  490. package/dist/memory/objects.js.map +0 -1
  491. package/dist/memory/planner.d.ts +0 -66
  492. package/dist/memory/planner.d.ts.map +0 -1
  493. package/dist/memory/planner.js +0 -108
  494. package/dist/memory/planner.js.map +0 -1
  495. package/dist/memory/targets.d.ts +0 -78
  496. package/dist/memory/targets.d.ts.map +0 -1
  497. package/dist/memory/targets.js +0 -29
  498. package/dist/memory/targets.js.map +0 -1
  499. package/dist/memory/transforms.d.ts +0 -19
  500. package/dist/memory/transforms.d.ts.map +0 -1
  501. package/dist/memory/transforms.js +0 -14
  502. package/dist/memory/transforms.js.map +0 -1
  503. package/dist/modes/matrix.d.ts +0 -23
  504. package/dist/modes/matrix.d.ts.map +0 -1
  505. package/dist/modes/matrix.js +0 -18
  506. package/dist/modes/matrix.js.map +0 -1
  507. package/dist/modes/modes.d.ts +0 -33
  508. package/dist/modes/modes.d.ts.map +0 -1
  509. package/dist/modes/modes.js +0 -15
  510. package/dist/modes/modes.js.map +0 -1
  511. package/dist/modes/resolve.d.ts +0 -96
  512. package/dist/modes/resolve.d.ts.map +0 -1
  513. package/dist/modes/resolve.js +0 -48
  514. package/dist/modes/resolve.js.map +0 -1
  515. package/dist/observability/metrics.d.ts +0 -14
  516. package/dist/observability/metrics.d.ts.map +0 -1
  517. package/dist/observability/metrics.js +0 -18
  518. package/dist/observability/metrics.js.map +0 -1
  519. package/dist/packager/manifest.d.ts +0 -66
  520. package/dist/packager/manifest.d.ts.map +0 -1
  521. package/dist/packager/manifest.js +0 -94
  522. package/dist/packager/manifest.js.map +0 -1
  523. package/dist/packager/publish.d.ts +0 -51
  524. package/dist/packager/publish.d.ts.map +0 -1
  525. package/dist/packager/publish.js +0 -18
  526. package/dist/packager/publish.js.map +0 -1
  527. package/dist/packager/targetcli.d.ts +0 -7
  528. package/dist/packager/targetcli.d.ts.map +0 -1
  529. package/dist/packager/targetcli.js +0 -22
  530. package/dist/packager/targetcli.js.map +0 -1
  531. package/dist/persistence/adapter.d.ts +0 -5
  532. package/dist/persistence/adapter.d.ts.map +0 -1
  533. package/dist/persistence/adapter.js +0 -11
  534. package/dist/persistence/adapter.js.map +0 -1
  535. package/dist/persistence/drizzle.d.ts +0 -2
  536. package/dist/persistence/drizzle.d.ts.map +0 -1
  537. package/dist/persistence/drizzle.js +0 -12
  538. package/dist/persistence/drizzle.js.map +0 -1
  539. package/dist/persistence/memory.d.ts +0 -2
  540. package/dist/persistence/memory.d.ts.map +0 -1
  541. package/dist/persistence/memory.js +0 -27
  542. package/dist/persistence/memory.js.map +0 -1
  543. package/dist/persistence/migrations.d.ts +0 -19
  544. package/dist/persistence/migrations.d.ts.map +0 -1
  545. package/dist/persistence/migrations.js +0 -13
  546. package/dist/persistence/migrations.js.map +0 -1
  547. package/dist/persistence/prisma.d.ts +0 -2
  548. package/dist/persistence/prisma.d.ts.map +0 -1
  549. package/dist/persistence/prisma.js +0 -25
  550. package/dist/persistence/prisma.js.map +0 -1
  551. package/dist/persistence/schema.d.ts +0 -63
  552. package/dist/persistence/schema.d.ts.map +0 -1
  553. package/dist/persistence/schema.js +0 -28
  554. package/dist/persistence/schema.js.map +0 -1
  555. package/dist/persistence/sql.d.ts +0 -3
  556. package/dist/persistence/sql.d.ts.map +0 -1
  557. package/dist/persistence/sql.js +0 -37
  558. package/dist/persistence/sql.js.map +0 -1
  559. package/dist/protocol/blocks.d.ts +0 -10
  560. package/dist/protocol/blocks.d.ts.map +0 -1
  561. package/dist/protocol/blocks.js +0 -26
  562. package/dist/protocol/blocks.js.map +0 -1
  563. package/dist/protocol/json.d.ts +0 -6
  564. package/dist/protocol/json.d.ts.map +0 -1
  565. package/dist/protocol/json.js +0 -6
  566. package/dist/protocol/json.js.map +0 -1
  567. package/dist/protocol/ndjson.d.ts +0 -3
  568. package/dist/protocol/ndjson.d.ts.map +0 -1
  569. package/dist/protocol/ndjson.js +0 -20
  570. package/dist/protocol/ndjson.js.map +0 -1
  571. package/dist/protocol/sse.d.ts +0 -7
  572. package/dist/protocol/sse.d.ts.map +0 -1
  573. package/dist/protocol/sse.js +0 -25
  574. package/dist/protocol/sse.js.map +0 -1
  575. package/dist/proxy/pool.d.ts +0 -9
  576. package/dist/proxy/pool.d.ts.map +0 -1
  577. package/dist/proxy/pool.js +0 -27
  578. package/dist/proxy/pool.js.map +0 -1
  579. package/dist/queue/idempotency.d.ts +0 -10
  580. package/dist/queue/idempotency.d.ts.map +0 -1
  581. package/dist/queue/idempotency.js +0 -13
  582. package/dist/queue/idempotency.js.map +0 -1
  583. package/dist/queue/persistent.d.ts +0 -14
  584. package/dist/queue/persistent.d.ts.map +0 -1
  585. package/dist/queue/persistent.js +0 -66
  586. package/dist/queue/persistent.js.map +0 -1
  587. package/dist/queue/queue.d.ts +0 -7
  588. package/dist/queue/queue.d.ts.map +0 -1
  589. package/dist/queue/queue.js +0 -55
  590. package/dist/queue/queue.js.map +0 -1
  591. package/dist/queue/saga.d.ts +0 -5
  592. package/dist/queue/saga.d.ts.map +0 -1
  593. package/dist/queue/saga.js +0 -20
  594. package/dist/queue/saga.js.map +0 -1
  595. package/dist/release/assets.d.ts +0 -144
  596. package/dist/release/assets.d.ts.map +0 -1
  597. package/dist/release/assets.js +0 -157
  598. package/dist/release/assets.js.map +0 -1
  599. package/dist/release/evidence.d.ts +0 -92
  600. package/dist/release/evidence.d.ts.map +0 -1
  601. package/dist/release/evidence.js +0 -148
  602. package/dist/release/evidence.js.map +0 -1
  603. package/dist/release/verify.d.ts +0 -12
  604. package/dist/release/verify.d.ts.map +0 -1
  605. package/dist/release/verify.js +0 -112
  606. package/dist/release/verify.js.map +0 -1
  607. package/dist/runners/chain.d.ts +0 -125
  608. package/dist/runners/chain.d.ts.map +0 -1
  609. package/dist/runners/chain.js +0 -95
  610. package/dist/runners/chain.js.map +0 -1
  611. package/dist/runners/health.d.ts +0 -36
  612. package/dist/runners/health.d.ts.map +0 -1
  613. package/dist/runners/health.js +0 -26
  614. package/dist/runners/health.js.map +0 -1
  615. package/dist/runners/heartbeat.d.ts +0 -23
  616. package/dist/runners/heartbeat.d.ts.map +0 -1
  617. package/dist/runners/heartbeat.js +0 -29
  618. package/dist/runners/heartbeat.js.map +0 -1
  619. package/dist/runners/inprocess.d.ts +0 -17
  620. package/dist/runners/inprocess.d.ts.map +0 -1
  621. package/dist/runners/inprocess.js +0 -20
  622. package/dist/runners/inprocess.js.map +0 -1
  623. package/dist/runners/scheduler.d.ts +0 -5
  624. package/dist/runners/scheduler.d.ts.map +0 -1
  625. package/dist/runners/scheduler.js +0 -19
  626. package/dist/runners/scheduler.js.map +0 -1
  627. package/dist/runtime/abort.d.ts +0 -8
  628. package/dist/runtime/abort.d.ts.map +0 -1
  629. package/dist/runtime/abort.js +0 -13
  630. package/dist/runtime/abort.js.map +0 -1
  631. package/dist/runtime/compatibility.d.ts +0 -37
  632. package/dist/runtime/compatibility.d.ts.map +0 -1
  633. package/dist/runtime/compatibility.js +0 -10
  634. package/dist/runtime/compatibility.js.map +0 -1
  635. package/dist/runtime/detect.d.ts +0 -13
  636. package/dist/runtime/detect.d.ts.map +0 -1
  637. package/dist/runtime/detect.js +0 -18
  638. package/dist/runtime/detect.js.map +0 -1
  639. package/dist/runtime/engine.d.ts +0 -20
  640. package/dist/runtime/engine.d.ts.map +0 -1
  641. package/dist/runtime/engine.js +0 -61
  642. package/dist/runtime/engine.js.map +0 -1
  643. package/dist/runtime/retry.d.ts +0 -19
  644. package/dist/runtime/retry.d.ts.map +0 -1
  645. package/dist/runtime/retry.js +0 -54
  646. package/dist/runtime/retry.js.map +0 -1
  647. package/dist/runtime/worker.d.ts +0 -10
  648. package/dist/runtime/worker.d.ts.map +0 -1
  649. package/dist/runtime/worker.js +0 -25
  650. package/dist/runtime/worker.js.map +0 -1
  651. package/dist/scrape/cache.d.ts +0 -45
  652. package/dist/scrape/cache.d.ts.map +0 -1
  653. package/dist/scrape/cache.js +0 -45
  654. package/dist/scrape/cache.js.map +0 -1
  655. package/dist/scrape/crawl.d.ts +0 -47
  656. package/dist/scrape/crawl.d.ts.map +0 -1
  657. package/dist/scrape/crawl.js +0 -114
  658. package/dist/scrape/crawl.js.map +0 -1
  659. package/dist/scrape/extract.d.ts +0 -63
  660. package/dist/scrape/extract.d.ts.map +0 -1
  661. package/dist/scrape/extract.js +0 -47
  662. package/dist/scrape/extract.js.map +0 -1
  663. package/dist/scrape/normalize.d.ts +0 -52
  664. package/dist/scrape/normalize.d.ts.map +0 -1
  665. package/dist/scrape/normalize.js +0 -102
  666. package/dist/scrape/normalize.js.map +0 -1
  667. package/dist/scrape/robots.d.ts +0 -37
  668. package/dist/scrape/robots.d.ts.map +0 -1
  669. package/dist/scrape/robots.js +0 -71
  670. package/dist/scrape/robots.js.map +0 -1
  671. package/dist/scrape/schema.d.ts +0 -23
  672. package/dist/scrape/schema.d.ts.map +0 -1
  673. package/dist/scrape/schema.js +0 -92
  674. package/dist/scrape/schema.js.map +0 -1
  675. package/dist/scrape/scraper.d.ts +0 -6
  676. package/dist/scrape/scraper.d.ts.map +0 -1
  677. package/dist/scrape/scraper.js +0 -46
  678. package/dist/scrape/scraper.js.map +0 -1
  679. package/dist/scrape/semantic.d.ts +0 -25
  680. package/dist/scrape/semantic.d.ts.map +0 -1
  681. package/dist/scrape/semantic.js +0 -27
  682. package/dist/scrape/semantic.js.map +0 -1
  683. package/dist/server/node.d.ts +0 -7
  684. package/dist/server/node.d.ts.map +0 -1
  685. package/dist/server/node.js.map +0 -1
  686. package/dist/sessions/file.d.ts +0 -25
  687. package/dist/sessions/file.d.ts.map +0 -1
  688. package/dist/sessions/file.js +0 -13
  689. package/dist/sessions/file.js.map +0 -1
  690. package/dist/sessions/replay.d.ts +0 -9
  691. package/dist/sessions/replay.d.ts.map +0 -1
  692. package/dist/sessions/replay.js +0 -70
  693. package/dist/sessions/replay.js.map +0 -1
  694. package/dist/sessions/store.d.ts +0 -25
  695. package/dist/sessions/store.d.ts.map +0 -1
  696. package/dist/sessions/store.js +0 -13
  697. package/dist/sessions/store.js.map +0 -1
  698. package/dist/storage/adapter.d.ts +0 -5
  699. package/dist/storage/adapter.d.ts.map +0 -1
  700. package/dist/storage/adapter.js +0 -11
  701. package/dist/storage/adapter.js.map +0 -1
  702. package/dist/storage/cache.d.ts +0 -23
  703. package/dist/storage/cache.d.ts.map +0 -1
  704. package/dist/storage/cache.js +0 -79
  705. package/dist/storage/cache.js.map +0 -1
  706. package/dist/storage/checksum.d.ts +0 -3
  707. package/dist/storage/checksum.d.ts.map +0 -1
  708. package/dist/storage/checksum.js +0 -19
  709. package/dist/storage/checksum.js.map +0 -1
  710. package/dist/storage/chunked.d.ts +0 -21
  711. package/dist/storage/chunked.d.ts.map +0 -1
  712. package/dist/storage/chunked.js +0 -67
  713. package/dist/storage/chunked.js.map +0 -1
  714. package/dist/storage/content.d.ts +0 -24
  715. package/dist/storage/content.d.ts.map +0 -1
  716. package/dist/storage/content.js +0 -44
  717. package/dist/storage/content.js.map +0 -1
  718. package/dist/storage/filehosting.d.ts +0 -2
  719. package/dist/storage/filehosting.d.ts.map +0 -1
  720. package/dist/storage/filehosting.js +0 -24
  721. package/dist/storage/filehosting.js.map +0 -1
  722. package/dist/storage/githubcontents.d.ts +0 -2
  723. package/dist/storage/githubcontents.d.ts.map +0 -1
  724. package/dist/storage/githubcontents.js +0 -29
  725. package/dist/storage/githubcontents.js.map +0 -1
  726. package/dist/storage/index.d.ts +0 -12
  727. package/dist/storage/index.d.ts.map +0 -1
  728. package/dist/storage/index.js +0 -12
  729. package/dist/storage/index.js.map +0 -1
  730. package/dist/storage/local.d.ts +0 -2
  731. package/dist/storage/local.d.ts.map +0 -1
  732. package/dist/storage/local.js +0 -72
  733. package/dist/storage/local.js.map +0 -1
  734. package/dist/storage/memory.d.ts +0 -6
  735. package/dist/storage/memory.d.ts.map +0 -1
  736. package/dist/storage/memory.js +0 -36
  737. package/dist/storage/memory.js.map +0 -1
  738. package/dist/storage/pool.d.ts +0 -95
  739. package/dist/storage/pool.d.ts.map +0 -1
  740. package/dist/storage/pool.js +0 -202
  741. package/dist/storage/pool.js.map +0 -1
  742. package/dist/storage/s3compatible.d.ts +0 -3
  743. package/dist/storage/s3compatible.d.ts.map +0 -1
  744. package/dist/storage/s3compatible.js +0 -70
  745. package/dist/storage/s3compatible.js.map +0 -1
  746. package/dist/storage/sync.d.ts +0 -47
  747. package/dist/storage/sync.d.ts.map +0 -1
  748. package/dist/storage/sync.js +0 -75
  749. package/dist/storage/sync.js.map +0 -1
  750. package/dist/surfaces/adapters.d.ts +0 -97
  751. package/dist/surfaces/adapters.d.ts.map +0 -1
  752. package/dist/surfaces/adapters.js +0 -50
  753. package/dist/surfaces/adapters.js.map +0 -1
  754. package/dist/surfaces/controls.d.ts +0 -18
  755. package/dist/surfaces/controls.d.ts.map +0 -1
  756. package/dist/surfaces/controls.js +0 -45
  757. package/dist/surfaces/controls.js.map +0 -1
  758. package/dist/surfaces/manifest.d.ts +0 -68
  759. package/dist/surfaces/manifest.d.ts.map +0 -1
  760. package/dist/surfaces/manifest.js +0 -42
  761. package/dist/surfaces/manifest.js.map +0 -1
  762. package/dist/surfaces/n8n.d.ts +0 -25
  763. package/dist/surfaces/n8n.d.ts.map +0 -1
  764. package/dist/surfaces/n8n.js +0 -31
  765. package/dist/surfaces/n8n.js.map +0 -1
  766. package/dist/surfaces/operations.d.ts +0 -32
  767. package/dist/surfaces/operations.d.ts.map +0 -1
  768. package/dist/surfaces/operations.js +0 -53
  769. package/dist/surfaces/operations.js.map +0 -1
  770. package/dist/surfaces/requirements.d.ts +0 -29
  771. package/dist/surfaces/requirements.d.ts.map +0 -1
  772. package/dist/surfaces/requirements.js +0 -50
  773. package/dist/surfaces/requirements.js.map +0 -1
  774. package/dist/surfaces/targets.d.ts +0 -196
  775. package/dist/surfaces/targets.d.ts.map +0 -1
  776. package/dist/surfaces/targets.js +0 -27
  777. package/dist/surfaces/targets.js.map +0 -1
  778. package/dist/webhook/delivery.d.ts +0 -18
  779. package/dist/webhook/delivery.d.ts.map +0 -1
  780. package/dist/webhook/delivery.js +0 -36
  781. package/dist/webhook/delivery.js.map +0 -1
  782. package/dist/webhook/receiver.d.ts +0 -26
  783. package/dist/webhook/receiver.d.ts.map +0 -1
  784. package/dist/webhook/receiver.js +0 -24
  785. package/dist/webhook/receiver.js.map +0 -1
  786. package/dist/webhook/signature.d.ts +0 -3
  787. package/dist/webhook/signature.d.ts.map +0 -1
  788. package/dist/webhook/signature.js +0 -7
  789. package/dist/webhook/signature.js.map +0 -1
  790. package/dist/workflow/manifest.d.ts +0 -22
  791. package/dist/workflow/manifest.d.ts.map +0 -1
  792. package/dist/workflow/manifest.js +0 -21
  793. package/dist/workflow/manifest.js.map +0 -1
  794. package/dist/workflow/registry.d.ts +0 -7
  795. package/dist/workflow/registry.d.ts.map +0 -1
  796. package/dist/workflow/registry.js +0 -18
  797. package/dist/workflow/registry.js.map +0 -1
  798. package/dist/workflow/templates.d.ts +0 -6
  799. package/dist/workflow/templates.d.ts.map +0 -1
  800. package/dist/workflow/templates.js +0 -18
  801. package/dist/workflow/templates.js.map +0 -1
  802. package/dist/workflow/triggers.d.ts +0 -64
  803. package/dist/workflow/triggers.d.ts.map +0 -1
  804. package/dist/workflow/triggers.js +0 -99
  805. package/dist/workflow/triggers.js.map +0 -1
  806. package/docs/logs/.gitkeep +0 -0
  807. package/docs/plans/00.index.md +0 -50
  808. package/docs/plans/01.architecture.md +0 -86
  809. package/docs/plans/02.research.computer.use.md +0 -58
  810. package/docs/plans/03.research.captcha.bypass.md +0 -68
  811. package/docs/plans/04.research.sandbox.ai.md +0 -52
  812. package/docs/plans/05.capture.platform.md +0 -57
  813. package/docs/plans/06.dependencies.md +0 -97
  814. package/docs/plans/07.captcha.test.page.md +0 -41
  815. package/docs/plans/08.production.infra.md +0 -70
  816. package/docs/plans/09.database.schema.md +0 -121
  817. package/docs/plans/10.cloudinary.storage.md +0 -57
  818. package/docs/plans/11.movement.logs.json.md +0 -72
  819. package/docs/plans/12.research.atlas.agent.browser.md +0 -79
  820. package/docs/plans/13.research.anti.detection.md +0 -898
  821. package/docs/plans/14.research.proxy.md +0 -1495
  822. package/docs/plans/15.research.retry.rate.limit.md +0 -1958
  823. package/docs/plans/16.research.crawling.md +0 -1417
  824. package/docs/plans/17.research.caching.md +0 -1610
  825. package/docs/plans/18.research.content.extraction.md +0 -1952
  826. package/docs/plans/19.research.errors.events.md +0 -1523
  827. package/docs/plans/20.research.zod.validation.md +0 -1350
  828. package/docs/plans/21.research.batch.concurrency.md +0 -1888
  829. package/docs/plans/22.research.universal.runtime.md +0 -944
  830. package/docs/plans/23.research.ai.integration.md +0 -1465
  831. package/docs/plans/24.research.memory.persistence.md +0 -1979
  832. package/docs/plans/25.research.server.api.md +0 -342
  833. package/docs/plans/26.research.compilation.md +0 -249
  834. package/docs/plans/27.research.html.parsing.md +0 -251
  835. package/docs/plans/28.action.plan.md +0 -50
  836. package/docs/plans/29.api.reference.md +0 -174
  837. package/docs/plans/30.architecture.plan.md +0 -94
  838. package/docs/plans/31.auditoria.dados.md +0 -163
  839. package/docs/plans/32.bots.automacao.computacional.md +0 -214
  840. package/docs/plans/33.bots.codigo.revisao.md +0 -220
  841. package/docs/plans/34.bots.seguranca.cicd.md +0 -366
  842. package/docs/plans/35.comparativo.concorrencia.md +0 -464
  843. package/docs/plans/36.computational.memory.md +0 -340
  844. package/docs/plans/37.deploystrategy.md +0 -394
  845. package/docs/plans/38.flow.md +0 -155
  846. package/docs/plans/39.multi.platform.bot.md +0 -252
  847. package/docs/plans/40.npm.publish.md +0 -250
  848. package/docs/plans/41.o.que.falta.md +0 -407
  849. package/docs/plans/42.pesquisa.concorrencia.md +0 -721
  850. package/docs/plans/43.plan.universal.architecture.md +0 -496
  851. package/docs/plans/44.reference.md +0 -100
  852. package/docs/plans/45.robotarchitecture.md +0 -237
  853. package/docs/plans/46.scdnintegration.md +0 -284
  854. package/docs/plans/47.multiforge.readme.md +0 -129
  855. package/docs/plans/48.theory.v4.repo.os.md +0 -152
  856. package/docs/plans/49.third.party.infra.md +0 -12
  857. package/docs/plans/50.file.as.compute.md +0 -39
  858. package/docs/plans/51.architecture.virtual.processor.md +0 -80
  859. package/docs/plans/52.manifesto.v8.md +0 -11
  860. package/docs/plans/58.cdn.list.md +0 -23
  861. package/docs/plans/59.sql.frameworks.md +0 -33
  862. package/docs/plans/60.sql.thirdparty.md +0 -26
  863. package/docs/plans/61.objective.multiforge.md +0 -63
  864. package/docs/plans/62.huggingface.upload.md +0 -26
  865. package/docs/plans/63.kaggle.upload.md +0 -24
  866. package/docs/plans/64.npm.storage.md +0 -30
  867. package/docs/plans/65.rclone.terabox.md +0 -32
  868. package/docs/plans/66.buckets.and.models.todo.md +0 -14
  869. package/docs/plans/67.database.todo.md +0 -13
  870. package/docs/plans/68.deploy.packages.todo.md +0 -12
  871. package/docs/plans/69.report.human.operator.md +0 -133
  872. package/docs/plans/70.report.brain2qwerty.ems.md +0 -135
  873. package/docs/plans/71.report.hd.infinito.vram.md +0 -155
  874. package/docs/plans/72.plan.hd.infinito.node.md +0 -146
  875. package/docs/plans/73.plan.scifi.repos.md +0 -125
  876. package/docs/plans/74.000.manifesto.v8.flat.2..md +0 -11
  877. package/docs/plans/README.md +0 -489
  878. package/docs/plans/aggregate_platforms.mjs +0 -146
  879. package/docs/plans/examplesession.json +0 -36
  880. package/docs/plans/missing-facts.md +0 -192
  881. package/docs/plans/models.md +0 -64
  882. package/docs/plans/organize.cjs +0 -270
  883. package/docs/plans/platforms.md +0 -2887
  884. package/docs/plans/sites.md +0 -31322
  885. package/docs/sources/farm.py +0 -117
  886. package/docs/sources/html/saddle1.html +0 -132
  887. package/docs/sources/html/saddle2.html +0 -157
  888. package/docs/sources/html/saddle3.html +0 -119
  889. package/docs/sources/html/saddle4.html +0 -144
  890. package/docs/sources/html/saddle5.html +0 -72
  891. package/docs/sources/html/saddle6.html +0 -171
  892. package/docs/sources/html/saddle7.html +0 -236
  893. package/docs/sources/saddle.ts +0 -74
  894. package/docs/sources/schema.prisma +0 -88
  895. package/docs/sources/script.sh +0 -64
  896. package/docs/sources/workflows.yml +0 -458
  897. package/docs/talks1/_body.txt +0 -14
  898. package/docs/talks1/_index.md +0 -15
  899. package/docs/talks1/_screenshot.png +0 -0
  900. package/docs/talks1/assistant-01.md +0 -5
  901. package/docs/talks1/assistant-02.md +0 -5
  902. package/docs/talks1/assistant-03.md +0 -531
  903. package/docs/talks1/assistant-04.md +0 -26
  904. package/docs/talks1/assistant-05.md +0 -774
  905. package/docs/talks1/assistant-06.md +0 -1718
  906. package/docs/talks1/scrape-share.cjs +0 -185
  907. package/docs/talks1/scrape-share.ts +0 -183
  908. package/docs/talks1/user-01.md +0 -3
  909. package/docs/talks1/user-02.md +0 -3
  910. package/docs/talks1/user-03.md +0 -88
  911. package/docs/talks1/user-04.md +0 -3
  912. package/docs/talks1/user-05.md +0 -3
  913. package/docs/talks1/user-06.md +0 -88
  914. package/docs/talks1/user-07.md +0 -88
  915. package/docs/talks2/_body.txt +0 -14
  916. package/docs/talks2/_index.md +0 -16
  917. package/docs/talks2/_screenshot.png +0 -0
  918. package/docs/talks2/assistant-01.md +0 -5
  919. package/docs/talks2/assistant-02.md +0 -5
  920. package/docs/talks2/assistant-03.md +0 -424
  921. package/docs/talks2/assistant-04.md +0 -598
  922. package/docs/talks2/assistant-05.md +0 -1280
  923. package/docs/talks2/assistant-06.md +0 -1227
  924. package/docs/talks2/assistant-07.md +0 -1252
  925. package/docs/talks2/user-01.md +0 -3
  926. package/docs/talks2/user-02.md +0 -3
  927. package/docs/talks2/user-03.md +0 -88
  928. package/docs/talks2/user-04.md +0 -88
  929. package/docs/talks2/user-05.md +0 -88
  930. package/docs/talks2/user-06.md +0 -88
  931. package/docs/talks2/user-07.md +0 -3
  932. package/docs/talks3/_body.txt +0 -467
  933. package/docs/talks3/_index.md +0 -10
  934. package/docs/talks3/_screenshot.png +0 -0
  935. package/docs/talks3/assistant-01.md +0 -417
  936. package/docs/talks3/assistant-02.md +0 -417
  937. package/docs/talks3/assistant-03.md +0 -29
  938. package/docs/talks3/assistant-04.md +0 -727
  939. package/docs/talks3/user-01.md +0 -88
  940. package/docs/talks3/user-02.md +0 -88
  941. package/docs/talks3/user-03.md +0 -3
  942. package/docs/talks3/user-04.md +0 -3
  943. package/docs/talks4/_body.txt +0 -14
  944. package/docs/talks4/_index.md +0 -12
  945. package/docs/talks4/_screenshot.png +0 -0
  946. package/docs/talks4/assistant-01.md +0 -5
  947. package/docs/talks4/assistant-02.md +0 -5
  948. package/docs/talks4/assistant-03.md +0 -35
  949. package/docs/talks4/assistant-04.md +0 -512
  950. package/docs/talks4/assistant-05.md +0 -599
  951. package/docs/talks4/user-01.md +0 -3
  952. package/docs/talks4/user-02.md +0 -3
  953. package/docs/talks4/user-03.md +0 -88
  954. package/docs/talks4/user-04.md +0 -88
  955. package/docs/talks4/user-05.md +0 -7
  956. package/docs/talks5/_body.txt +0 -14
  957. package/docs/talks5/_index.md +0 -13
  958. package/docs/talks5/_screenshot.png +0 -0
  959. package/docs/talks5/assistant-01.md +0 -5
  960. package/docs/talks5/assistant-02.md +0 -5
  961. package/docs/talks5/assistant-03.md +0 -690
  962. package/docs/talks5/assistant-04.md +0 -758
  963. package/docs/talks5/assistant-05.md +0 -974
  964. package/docs/talks5/user-01.md +0 -3
  965. package/docs/talks5/user-02.md +0 -3
  966. package/docs/talks5/user-03.md +0 -105
  967. package/docs/talks5/user-04.md +0 -105
  968. package/docs/talks5/user-05.md +0 -63
  969. package/docs/talks5/user-06.md +0 -105
  970. package/docs/talks6/_body.txt +0 -14
  971. package/docs/talks6/_index.md +0 -9
  972. package/docs/talks6/_screenshot.png +0 -0
  973. package/docs/talks6/assistant-01.md +0 -5
  974. package/docs/talks6/assistant-02.md +0 -5
  975. package/docs/talks6/assistant-03.md +0 -1499
  976. package/docs/talks6/user-01.md +0 -3
  977. package/docs/talks6/user-02.md +0 -3
  978. package/docs/talks6/user-03.md +0 -88
  979. package/docs/talks6/user-04.md +0 -88
  980. package/docs/talks7/_body.txt +0 -14
  981. package/docs/talks7/_index.md +0 -10
  982. package/docs/talks7/_screenshot.png +0 -0
  983. package/docs/talks7/assistant-01.md +0 -5
  984. package/docs/talks7/assistant-02.md +0 -5
  985. package/docs/talks7/assistant-03.md +0 -523
  986. package/docs/talks7/assistant-04.md +0 -617
  987. package/docs/talks7/user-01.md +0 -3
  988. package/docs/talks7/user-02.md +0 -3
  989. package/docs/talks7/user-03.md +0 -105
  990. package/docs/talks7/user-04.md +0 -67
  991. package/docs/talks8/conversa1.txt +0 -1322
  992. package/docs/talks8/conversa2.txt +0 -237
  993. package/docs/talks9/Beyond the Obvious_ 50 Plataformas Auto-Hospedadas de Forja de C/303/263digo para Al/303/251m de Gitea e GitLab.md" +0 -174
  994. package/docs/talks9/De NPM a Multi-Linguagem_ Uma Arquitetura T/303/251cnica para a Execu/303/247/303/243o Integrada de C/303/263digo no Ecossistema Node.js.md" +0 -59
  995. package/docs/talks9/De NPM a VMs Virtuais_ Uma An/303/241lise Arquitet/303/264nica para a Realiza/303/247/303/243o do Ciclo de Vida do Projeto SADDLE.md" +0 -91
  996. package/docs/talks9/Mapeamento da Engrenagem Computacional_ Uma Arquitetura para Execu/303/247/303/243o Isolada e Persist/303/252ncia em Ambientes Distribu/303/255dos.md" +0 -116
  997. package/docs/talks9/O Cen/303/241rio Pr/303/241tico do SADDLE_ Uma An/303/241lise de Viabilidade e Modelo de Ciclo de Vida Integrado.md" +0 -128
  998. package/docs/talks9/README (2).md +0 -489
  999. package/docs/talks9/README.md +0 -198
  1000. package/docs/talks9/Viabilidade do Saddle_ Uma An/303/241lise T/303/251cnica da Transforma/303/247/303/243o de Armazenamento Remoto em Mem/303/263ria Computacional.md" +0 -80
  1001. package/docs/talks9/conversa.txt +0 -544
  1002. package/docs/talks9/other (2).md +0 -39
  1003. package/docs/talks9/other.md +0 -57
  1004. package/docs/talks9/outro.txt +0 -24
  1005. /package/{extension/README.md → docs/extension.md} +0 -0
@@ -1,1979 +0,0 @@
1
- # Pesquisa: Memory e Persistence Patterns (2026)
2
-
3
- > Padrões de armazenamento em memória, browser storage, server storage, edge KV, request deduplication, e persistência de estado para scraping.
4
-
5
- ---
6
-
7
- ## 1. In-Memory Storage
8
-
9
- ### 1.1. Map (nativo)
10
-
11
- ```typescript
12
- // Map simples — bom para uso básico
13
- const cache = new Map<string, unknown>();
14
-
15
- cache.set('key', { data: 'value' });
16
- cache.get('key'); // { data: 'value' }
17
- cache.has('key'); // true
18
- cache.delete('key'); // true
19
- cache.size; // 0
20
-
21
- // Iterar
22
- for (const [key, value] of cache) {
23
- console.log(key, value);
24
- }
25
-
26
- // Limitação: sem TTL, sem maxSize, sem cleanup automático
27
- ```
28
-
29
- ### 1.2. WeakMap (metadata, sem memory leak)
30
-
31
- ```typescript
32
- // WeakMap — keys devem ser objects, garbage collected automaticamente
33
- const metadata = new WeakMap<object, {
34
- scrapedAt: Date;
35
- accessCount: number;
36
- lastAccess: Date;
37
- }>();
38
-
39
- const pageData = { html: '<p>content</p>' };
40
- metadata.set(pageData, {
41
- scrapedAt: new Date(),
42
- accessCount: 0,
43
- lastAccess: new Date(),
44
- });
45
-
46
- // Quando pageData for GC'd, o metadata entry também é removido
47
- // Ideal para: metadata associada a objects que podem ser liberados
48
- ```
49
-
50
- ### 1.3. lru-cache v11 (bounded, TTL, fetchMethod)
51
-
52
- ```typescript
53
- import { LRUCache } from 'lru-cache';
54
-
55
- // Cache com limite de tamanho e TTL
56
- const cache = new LRUCache<string, unknown>({
57
- max: 500, // Máximo 500 entries
58
- ttl: 1000 * 60 * 5, // TTL: 5 minutos
59
- updateAgeOnGet: true, // Reset TTL ao acessar
60
- updateAgeOnHas: true, // Reset TTL ao verificar
61
-
62
- // fetchMethod: busca automática com cache
63
- fetchMethod: async (key: string) => {
64
- const response = await fetch(`https://api.example.com/data/${key}`);
65
- return response.json();
66
- },
67
- });
68
-
69
- // Uso com fetchMethod — busca e cacheia automaticamente
70
- const data = await cache.fetch('my-key');
71
- // Segunda vez: retorna do cache (sem fetch)
72
-
73
- // Operações básicas
74
- cache.set('key', 'value', { ttl: 1000 * 60 }); // TTL customizado
75
- cache.get('key'); // 'value'
76
- cache.has('key'); // true
77
- cache.delete('key'); // true
78
- cache.clear(); // Limpa tudo
79
-
80
- // Estatísticas
81
- cache.size; // Entries atuais
82
- cache.calculatedSize; // Tamanho estimado em bytes
83
- cache.dump(); // Serializar todo o cache
84
- cache.load(dumped); // Restaurar cache
85
-
86
- // Iterator
87
- for (const [key, value] of cache.entries()) {
88
- console.log(key, value);
89
- }
90
-
91
- // Cleanup periódico
92
- const interval = setInterval(() => {
93
- cache.purgeStale(); // Remove entries expiradas
94
- }, 60_000);
95
- ```
96
-
97
- **Exemplo: Cache de scraping com lru-cache**
98
-
99
- ```typescript
100
- interface ScrapeCacheEntry {
101
- html: string;
102
- markdown: string;
103
- metadata: Record<string, unknown>;
104
- scrapedAt: number;
105
- }
106
-
107
- const scrapeCache = new LRUCache<string, ScrapeCacheEntry>({
108
- max: 1000,
109
- ttl: 1000 * 60 * 30, // 30 minutos
110
- updateAgeOnGet: true,
111
- fetchMethod: async (url: string) => {
112
- console.log(`Fetching: ${url}`);
113
- const result = await scrape(url);
114
- return {
115
- html: result.content,
116
- markdown: result.text,
117
- metadata: result.metadata,
118
- scrapedAt: Date.now(),
119
- };
120
- },
121
- });
122
-
123
- // Transparent cache — busca se não existe
124
- const cached = await scrapeCache.fetch('https://example.com');
125
- // Segunda vez: retorna do cache instantaneamente
126
- ```
127
-
128
- ### 1.4. @keyv/bigmap (20M+ entries)
129
-
130
- ```typescript
131
- import { KeyvBigMap } from '@keyv/bigmap';
132
-
133
- // Para caches enormes — suporta 20M+ entries
134
- const bigCache = new KeyvBigMap({
135
- maxSize: 50_000_000, // 50M entries
136
- ttl: 1000 * 60 * 60, // 1 hora
137
- });
138
-
139
- await bigCache.set('key-1', { data: 'large payload' });
140
- const value = await bigCache.get('key-1');
141
-
142
- // Mais eficiente que Map para grandes volumes
143
- // Usa WeakRef para memória eficiente
144
- ```
145
-
146
- ---
147
-
148
- ## 2. Browser Storage
149
-
150
- ### 2.1. localStorage (5-10MB, síncrono)
151
-
152
- ```typescript
153
- // Limite: 5-10MB por origin
154
- // Síncrono: bloqueia a thread principal
155
- // Sem TTL nativo
156
-
157
- interface StoredItem<T> {
158
- value: T;
159
- timestamp: number;
160
- expires?: number;
161
- }
162
-
163
- class LocalStorageAdapter<T = unknown> {
164
- private prefix: string;
165
-
166
- constructor(prefix = 'ws_') {
167
- this.prefix = prefix;
168
- }
169
-
170
- set(key: string, value: T, ttlMs?: number): void {
171
- const item: StoredItem<T> = {
172
- value,
173
- timestamp: Date.now(),
174
- expires: ttlMs ? Date.now() + ttlMs : undefined,
175
- };
176
- localStorage.setItem(this.prefix + key, JSON.stringify(item));
177
- }
178
-
179
- get(key: string): T | null {
180
- const raw = localStorage.getItem(this.prefix + key);
181
- if (!raw) return null;
182
-
183
- try {
184
- const item: StoredItem<T> = JSON.parse(raw);
185
- if (item.expires && Date.now() > item.expires) {
186
- this.delete(key);
187
- return null;
188
- }
189
- return item.value;
190
- } catch {
191
- this.delete(key);
192
- return null;
193
- }
194
- }
195
-
196
- has(key: string): boolean {
197
- return this.get(key) !== null;
198
- }
199
-
200
- delete(key: string): void {
201
- localStorage.removeItem(this.prefix + key);
202
- }
203
-
204
- clear(): void {
205
- const keys = Object.keys(localStorage)
206
- .filter(k => k.startsWith(this.prefix));
207
- keys.forEach(k => localStorage.removeItem(k));
208
- }
209
-
210
- get size(): number {
211
- let count = 0;
212
- for (let i = 0; i < localStorage.length; i++) {
213
- const key = localStorage.key(i);
214
- if (key?.startsWith(this.prefix)) count++;
215
- }
216
- return count;
217
- }
218
- }
219
- ```
220
-
221
- ### 2.2. sessionStorage (tab-scoped)
222
-
223
- ```typescript
224
- // Mesmo API que localStorage, mas:
225
- // - Dados persistem só durante a sessão da tab
226
- // - Tab != Window (abrir link em nova tab = novo sessionStorage)
227
- // - Útil para: estado temporário de uma sessão de scraping
228
-
229
- const session = new LocalStorageAdapter('session_');
230
- // Mesmo adapter, diferente prefix e lifecycle
231
- ```
232
-
233
- ### 2.3. IndexedDB (async, large)
234
-
235
- ```typescript
236
- // Async, grande capacidade (GBs), estruturado
237
- // Não bloqueia a thread principal
238
-
239
- interface IDBConfig {
240
- dbName: string;
241
- version?: number;
242
- storeName: string;
243
- }
244
-
245
- class IndexedDBAdapter<T = unknown> {
246
- private db: IDBDatabase | null = null;
247
- private config: IDBConfig;
248
-
249
- constructor(config: IDBConfig) {
250
- this.config = { version: 1, ...config };
251
- }
252
-
253
- async init(): Promise<void> {
254
- return new Promise((resolve, reject) => {
255
- const request = indexedDB.open(
256
- this.config.dbName,
257
- this.config.version
258
- );
259
-
260
- request.onupgradeneeded = (event) => {
261
- const db = (event.target as IDBOpenDBRequest).result;
262
- if (!db.objectStoreNames.contains(this.config.storeName)) {
263
- db.createObjectStore(this.config.storeName, { keyPath: 'id' });
264
- }
265
- };
266
-
267
- request.onsuccess = (event) => {
268
- this.db = (event.target as IDBOpenDBRequest).result;
269
- resolve();
270
- };
271
-
272
- request.onerror = () => reject(request.error);
273
- });
274
- }
275
-
276
- async set(key: string, value: T): Promise<void> {
277
- if (!this.db) throw new Error('DB not initialized');
278
-
279
- return new Promise((resolve, reject) => {
280
- const tx = this.db!.transaction(this.config.storeName, 'readwrite');
281
- const store = tx.objectStore(this.config.storeName);
282
- const request = store.put({ id: key, value, timestamp: Date.now() });
283
-
284
- request.onsuccess = () => resolve();
285
- request.onerror = () => reject(request.error);
286
- });
287
- }
288
-
289
- async get(key: string): Promise<T | null> {
290
- if (!this.db) throw new Error('DB not initialized');
291
-
292
- return new Promise((resolve, reject) => {
293
- const tx = this.db!.transaction(this.config.storeName, 'readonly');
294
- const store = tx.objectStore(this.config.storeName);
295
- const request = store.get(key);
296
-
297
- request.onsuccess = () => {
298
- const result = request.result;
299
- resolve(result?.value ?? null);
300
- };
301
- request.onerror = () => reject(request.error);
302
- });
303
- }
304
-
305
- async delete(key: string): Promise<void> {
306
- if (!this.db) throw new Error('DB not initialized');
307
-
308
- return new Promise((resolve, reject) => {
309
- const tx = this.db!.transaction(this.config.storeName, 'readwrite');
310
- const store = tx.objectStore(this.config.storeName);
311
- const request = store.delete(key);
312
-
313
- request.onsuccess = () => resolve();
314
- request.onerror = () => reject(request.error);
315
- });
316
- }
317
-
318
- async getAll(): Promise<Map<string, T>> {
319
- if (!this.db) throw new Error('DB not initialized');
320
-
321
- return new Promise((resolve, reject) => {
322
- const tx = this.db!.transaction(this.config.storeName, 'readonly');
323
- const store = tx.objectStore(this.config.storeName);
324
- const request = store.openCursor();
325
-
326
- const map = new Map<string, T>();
327
-
328
- request.onsuccess = () => {
329
- const cursor = request.result;
330
- if (cursor) {
331
- map.set(cursor.key as string, cursor.value.value);
332
- cursor.continue();
333
- } else {
334
- resolve(map);
335
- }
336
- };
337
- request.onerror = () => reject(request.error);
338
- });
339
- }
340
- }
341
- ```
342
-
343
- ### 2.4. store2 2.14.3 (wrapper cross-browser)
344
-
345
- ```typescript
346
- import store from 'store2';
347
-
348
- // Abstração unificada sobre localStorage/sessionStorage/IndexedDB
349
- store.namespace('webscrape_'); // Prefix automático
350
-
351
- store.set('key', { data: 'value' }); // Auto-JSON
352
- store.get('key'); // { data: 'value' }
353
- store.has('key'); // true
354
- store.remove('key'); // true
355
-
356
- // TTL manual (store2 não tem nativo)
357
- store.set('key', 'value', { expire: 3600 }); // Não funciona nativamente
358
-
359
- // Área de session (sessionStorage)
360
- store.session.set('key', 'value');
361
- store.session.get('key');
362
-
363
- // Área de area (IndexedDB)
364
- store.area.set('key', 'value');
365
- store.area.get('key');
366
-
367
- // Limits
368
- store.set('key', 'value', false); // Não pode ser sobrescrito
369
- store.has('key'); // true
370
- // store.set('key', 'other'); // Silenciosamente falha
371
-
372
- // Callbacks
373
- store.on('set', (key: string, value: unknown) => {
374
- console.log(`Stored: ${key}`);
375
- });
376
-
377
- // Serialização customizada
378
- store.defaults({
379
- serialize: JSON.stringify,
380
- deserialize: JSON.parse,
381
- });
382
- ```
383
-
384
- ---
385
-
386
- ## 3. Server Storage
387
-
388
- ### 3.1. better-sqlite3 (síncrono, rápido)
389
-
390
- ```typescript
391
- import Database from 'better-sqlite3';
392
-
393
- const db = new Database(':memory:'); // Ou './data.db'
394
-
395
- // Criar tabela
396
- db.exec(`
397
- CREATE TABLE IF NOT EXISTS cache (
398
- key TEXT PRIMARY KEY,
399
- value TEXT NOT NULL,
400
- created_at INTEGER DEFAULT (unixepoch()),
401
- expires_at INTEGER,
402
- access_count INTEGER DEFAULT 0
403
- )
404
- `);
405
-
406
- // Inserir
407
- const insert = db.prepare(
408
- 'INSERT OR REPLACE INTO cache (key, value, expires_at) VALUES (?, ?, ?)'
409
- );
410
- insert.run('url:https://example.com', JSON.stringify({ html: '...' }), Date.now() + 300_000);
411
-
412
- // Buscar com TTL check
413
- const find = db.prepare(
414
- 'SELECT value FROM cache WHERE key = ? AND (expires_at IS NULL OR expires_at > ?)'
415
- );
416
- const row = find.get('url:https://example.com', Date.now()) as { value: string } | undefined;
417
- const data = row ? JSON.parse(row.value) : null;
418
-
419
- // Batch operations (transação)
420
- const insertMany = db.transaction((entries: [string, string, number][]) => {
421
- for (const [key, value, expires] of entries) {
422
- insert.run(key, value, expires);
423
- }
424
- });
425
-
426
- insertMany([
427
- ['url:https://a.com', '{"html":"..."}', Date.now() + 300_000],
428
- ['url:https://b.com', '{"html":"..."}', Date.now() + 300_000],
429
- ]);
430
-
431
- // Cleanup de entries expiradas
432
- const cleanup = db.prepare(
433
- 'DELETE FROM cache WHERE expires_at IS NOT NULL AND expires_at < ?'
434
- );
435
- cleanup.run(Date.now());
436
-
437
- // Estatísticas
438
- const count = db.prepare('SELECT COUNT(*) as count FROM cache').get();
439
- console.log(`Cache size: ${count.count}`);
440
- ```
441
-
442
- ### 3.2. @keyv/sqlite
443
-
444
- ```typescript
445
- import Keyv from 'keyv';
446
- import KeyvSqlite from '@keyv/sqlite';
447
-
448
- const keyv = new Keyv({
449
- store: new KeyvSqlite({
450
- uri: 'sqlite://./data.sqlite',
451
- busyTimeout: 5000,
452
- }),
453
- ttl: 1000 * 60 * 30, // 30 minutos
454
- });
455
-
456
- await keyv.set('key', { data: 'value' });
457
- await keyv.get('key'); // { data: 'value' }
458
- await keyv.has('key'); // true
459
- await keyv.delete('key'); // true
460
-
461
- // Iterator
462
- for await (const [key, value] of keyv.iterator()) {
463
- console.log(key, value);
464
- }
465
-
466
- // Namespace
467
- const keyvA = new Keyv({ namespace: 'a' });
468
- const keyvB = new Keyv({ namespace: 'b' });
469
- // keyvA e keyvB são independentes
470
- ```
471
-
472
- ### 3.3. @keyv/redis / ioredis
473
-
474
- ```typescript
475
- import Keyv from 'keyv';
476
- import KeyvRedis from '@keyv/redis';
477
-
478
- // Keyv com Redis adapter
479
- const keyv = new Keyv({
480
- store: new KeyvRedis('redis://localhost:6379'),
481
- ttl: 1000 * 60 * 60, // 1 hora
482
- });
483
-
484
- await keyv.set('session:abc', { userId: 123, role: 'admin' });
485
- await keyv.get('session:abc');
486
-
487
- // ioredis direto (mais controle)
488
- import Redis from 'ioredis';
489
-
490
- const redis = new Redis({
491
- host: 'localhost',
492
- port: 6379,
493
- maxRetriesPerRequest: 3,
494
- retryStrategy(times) {
495
- return Math.min(times * 50, 2000);
496
- },
497
- });
498
-
499
- // Comandos básicos
500
- await redis.set('key', JSON.stringify({ data: 'value' }), 'EX', 300); // TTL 5min
501
- const value = JSON.parse((await redis.get('key')) ?? 'null');
502
- await redis.del('key');
503
-
504
- // Pipeline (batch)
505
- const pipeline = redis.pipeline();
506
- pipeline.set('key1', 'value1', 'EX', 60);
507
- pipeline.set('key2', 'value2', 'EX', 60);
508
- pipeline.get('key1');
509
- const results = await pipeline.exec();
510
-
511
- // Pub/Sub para invalidation
512
- const subscriber = new Redis('redis://localhost:6379');
513
- const publisher = new Redis('redis://localhost:6379');
514
-
515
- subscriber.subscribe('cache:invalidate');
516
- subscriber.on('message', (channel, key) => {
517
- if (channel === 'cache:invalidate') {
518
- console.log(`Invalidating: ${key}`);
519
- }
520
- });
521
-
522
- publisher.publish('cache:invalidate', 'url:https://example.com');
523
- ```
524
-
525
- ---
526
-
527
- ## 4. Cross-Runtime KV: keyv 5.6.0
528
-
529
- ### Por que keyv?
530
-
531
- - **104M+ weekly downloads** — padrão de facto
532
- - **Pluggable adapters** — in-memory, SQLite, Redis, PostgreSQL, MongoDB
533
- - **TTL nativo** — suporte a expiração
534
- - **Iterator** — percorre todas as entries
535
- - **Namespace** — multi-tenant no mesmo store
536
- - **Universal** — funciona em Node, Deno, Bun, browsers
537
-
538
- ```typescript
539
- import Keyv from 'keyv';
540
-
541
- // Store padrão (in-memory com Map)
542
- const keyv = new Keyv();
543
- await keyv.set('key', 'value', 5000); // TTL 5s
544
- await keyv.get('key'); // 'value'
545
-
546
- // Com adapter Redis
547
- import KeyvRedis from '@keyv/redis';
548
- const redisKeyv = new Keyv({ store: new KeyvRedis('redis://localhost') });
549
-
550
- // Com adapter SQLite
551
- import KeyvSqlite from '@keyv/sqlite';
552
- const sqliteKeyv = new Keyv({ store: new KeyvSqlite('sqlite://data.db') });
553
-
554
- // Namespace
555
- const sessions = new Keyv({ namespace: 'sessions' });
556
- const cache = new Keyv({ namespace: 'cache' });
557
-
558
- // Multiple namespaces no mesmo Redis
559
- const keyvA = new Keyv({ store: new KeyvRedis('redis://localhost'), namespace: 'a' });
560
- const keyvB = new Keyv({ store: new KeyvRedis('redis://localhost'), namespace: 'b' });
561
- ```
562
-
563
- ### keyv patterns para scraping
564
-
565
- ```typescript
566
- // Pattern 1: Request cache
567
- const requestCache = new Keyv({
568
- namespace: 'requests',
569
- ttl: 1000 * 60 * 10, // 10 minutos
570
- });
571
-
572
- async function cachedFetch(url: string): Promise<Response> {
573
- const cached = await requestCache.get(url);
574
- if (cached) return new Response(cached.body, cached);
575
-
576
- const response = await fetch(url);
577
- if (response.ok) {
578
- const clone = response.clone();
579
- const body = await clone.text();
580
- await requestCache.set(url, {
581
- body,
582
- status: response.status,
583
- headers: Object.fromEntries(response.headers.entries()),
584
- });
585
- }
586
-
587
- return response;
588
- }
589
-
590
- // Pattern 2: Rate limiting
591
- const rateLimiter = new Keyv({
592
- namespace: 'ratelimit',
593
- ttl: 1000 * 60, // 1 minuto window
594
- });
595
-
596
- async function isRateLimited(domain: string, maxRequests = 10): Promise<boolean> {
597
- const count = (await rateLimiter.get(domain)) ?? 0;
598
- if (count >= maxRequests) return true;
599
- await rateLimiter.set(domain, count + 1);
600
- return false;
601
- }
602
-
603
- // Pattern 3: Deduplicação
604
- const dedup = new Keyv({
605
- namespace: 'dedup',
606
- ttl: 1000 * 60 * 60, // 1 hora
607
- });
608
-
609
- async function isDuplicate(content: string): Promise<boolean> {
610
- const hash = await contentHash(content);
611
- if (await dedup.has(hash)) return true;
612
- await dedup.set(hash, true);
613
- return false;
614
- }
615
- ```
616
-
617
- ---
618
-
619
- ## 5. Edge Runtime Storage
620
-
621
- ### 5.1. Cloudflare Workers KV
622
-
623
- ```typescript
624
- // Cloudflare Workers KV
625
- // API global em workers, TTL mínimo 60s
626
-
627
- export default {
628
- async fetch(request: Request, env: Env): Promise<Response> {
629
- const url = new URL(request.url);
630
- const key = url.pathname.slice(1);
631
-
632
- if (request.method === 'GET') {
633
- const value = await env.KV.get(key, 'json');
634
- if (!value) return new Response('Not found', { status: 404 });
635
- return Response.json(value);
636
- }
637
-
638
- if (request.method === 'PUT') {
639
- const body = await request.json();
640
- await env.KV.put(key, JSON.stringify(body), {
641
- expirationTtl: 3600, // 1 hora (mínimo 60s)
642
- });
643
- return new Response('OK');
644
- }
645
-
646
- return new Response('Method not allowed', { status: 405);
647
- },
648
- };
649
-
650
- // Interface
651
- interface Env {
652
- KV: KVNamespace;
653
- }
654
-
655
- // Batch operations
656
- async function batchGet(keys: string[], env: Env): Promise<Map<string, unknown>> {
657
- const results = new Map<string, unknown>();
658
-
659
- // KV não tem batch get nativo — fazer parallel
660
- await Promise.all(keys.map(async (key) => {
661
- const value = await env.KV.get(key, 'json');
662
- if (value) results.set(key, value);
663
- }));
664
-
665
- return results;
666
- }
667
-
668
- // List with prefix
669
- async function listByPrefix(prefix: string, env: Env): Promise<string[]> {
670
- const list = await env.KV.list({ prefix });
671
- return list.keys.map(k => k.name);
672
- }
673
- ```
674
-
675
- ### 5.2. Upstash Redis
676
-
677
- ```typescript
678
- import { Redis } from '@upstash/redis';
679
-
680
- const redis = new Redis({
681
- url: process.env.UPSTASH_REDIS_REST_URL!,
682
- token: process.env.UPSTASH_REDIS_REST_TOKEN!,
683
- });
684
-
685
- // Operações básicas
686
- await redis.set('key', { data: 'value' }, { ex: 300 }); // TTL 5min
687
- await redis.get('key');
688
- await redis.del('key');
689
-
690
- // Rate limiting
691
- async function upstashRateLimit(
692
- key: string,
693
- limit: number,
694
- windowSec: number
695
- ): Promise<{ allowed: boolean; remaining: number }> {
696
- const now = Date.now();
697
- const windowStart = now - windowSec * 1000;
698
-
699
- const pipeline = redis.pipeline();
700
- pipeline.zremrangebyscore(key, 0, windowStart);
701
- pipeline.zadd(key, { score: now, member: now.toString() });
702
- pipeline.zcard(key);
703
- pipeline.expire(key, windowSec);
704
-
705
- const results = await pipeline.exec();
706
- const count = results[2] as number;
707
-
708
- return {
709
- allowed: count <= limit,
710
- remaining: Math.max(0, limit - count),
711
- };
712
- }
713
-
714
- // Caching com TTL
715
- async function cached<T>(
716
- key: string,
717
- fn: () => Promise<T>,
718
- ttlSec = 300
719
- ): Promise<T> {
720
- const cached = await redis.get<T>(key);
721
- if (cached) return cached;
722
-
723
- const value = await fn();
724
- await redis.set(key, JSON.stringify(value), { ex: ttlSec });
725
- return value;
726
- }
727
- ```
728
-
729
- ### 5.3. Deno KV
730
-
731
- ```typescript
732
- // Deno KV — built-in no Deno 2.x
733
- const kv = await Deno.openKv();
734
-
735
- // Operações básicas
736
- await kv.set(['cache', 'url', 'https://example.com'], { html: '...' }, {
737
- expireIn: 1000 * 60 * 30, // 30 minutos
738
- });
739
-
740
- const entry = await kv.get(['cache', 'url', 'https://example.com']);
741
- if (entry.value) {
742
- console.log(entry.value);
743
- }
744
-
745
- // Delete
746
- await kv.delete(['cache', 'url', 'https://example.com']);
747
-
748
- // List com prefix
749
- const entries = await kv.list({ prefix: ['cache', 'url'] });
750
- for await (const entry of entries) {
751
- console.log(entry.key, entry.value);
752
- }
753
-
754
- // Atomic operations
755
- const res = await kv.atomic()
756
- .check({ key: ['counter'], versionstamp: null })
757
- .sum(['counter'], 1n)
758
- .commit();
759
-
760
- if (!res.ok) {
761
- console.log('Atomic operation failed');
762
- }
763
-
764
- // Transactions
765
- async function atomicCacheUpdate(
766
- key: string[],
767
- value: unknown,
768
- ttl: number
769
- ): Promise<boolean> {
770
- const result = await kv.atomic()
771
- .set(key, value, { expireIn: ttl })
772
- .commit();
773
- return result.ok;
774
- }
775
- ```
776
-
777
- ---
778
-
779
- ## 6. Request Deduplication
780
-
781
- ### 6.1. Keyv com TTL
782
-
783
- ```typescript
784
- import Keyv from 'keyv';
785
-
786
- const dedupStore = new Keyv({
787
- namespace: 'request-dedup',
788
- ttl: 1000 * 60 * 5, // 5 minutos
789
- });
790
-
791
- async function deduplicatedFetch(url: string): Promise<Response> {
792
- const key = `req:${url}`;
793
-
794
- // Verificar se já está em andamento
795
- if (await dedupStore.has(key)) {
796
- // Retornar do cache ou aguardar
797
- return cachedResponse(url);
798
- }
799
-
800
- // Marcar como em andamento
801
- await dedupStore.set(key, { status: 'pending', started: Date.now() });
802
-
803
- try {
804
- const response = await fetch(url);
805
- const body = await response.text();
806
-
807
- // Cache a resposta
808
- await dedupStore.set(key, {
809
- status: 'done',
810
- body,
811
- status_code: response.status,
812
- headers: Object.fromEntries(response.headers.entries()),
813
- });
814
-
815
- return new Response(body, {
816
- status: response.status,
817
- headers: response.headers,
818
- });
819
- } catch (error) {
820
- await dedupStore.delete(key);
821
- throw error;
822
- }
823
- }
824
- ```
825
-
826
- ### 6.2. Bloom Filter
827
-
828
- ```typescript
829
- // Bloom filter — probabilidade controlada de falsos positivos
830
- // Não tem falsos negativos — eficiente em memória
831
-
832
- class BloomFilter {
833
- private bits: Uint8Array;
834
- private size: number;
835
- private hashes: number;
836
-
837
- constructor(expectedItems: number, falsePositiveRate = 0.01) {
838
- // Calcular tamanho ótimo
839
- this.size = Math.ceil(
840
- -(expectedItems * Math.log(falsePositiveRate)) / (Math.log(2) ** 2)
841
- );
842
- this.hashes = Math.ceil((this.size / expectedItems) * Math.log(2));
843
- this.bits = new Uint8Array(Math.ceil(this.size / 8));
844
- }
845
-
846
- private hash(value: string, seed: number): number {
847
- let hash = 0;
848
- for (let i = 0; i < value.length; i++) {
849
- hash = ((hash << 5) - hash + value.charCodeAt(i) + seed) | 0;
850
- }
851
- return Math.abs(hash) % this.size;
852
- }
853
-
854
- add(value: string): void {
855
- for (let i = 0; i < this.hashes; i++) {
856
- const pos = this.hash(value, i);
857
- this.bits[Math.floor(pos / 8)] |= 1 << (pos % 8);
858
- }
859
- }
860
-
861
- mightContain(value: string): boolean {
862
- for (let i = 0; i < this.hashes; i++) {
863
- const pos = this.hash(value, i);
864
- if (!(this.bits[Math.floor(pos / 8)] & (1 << (pos % 8)))) {
865
- return false;
866
- }
867
- }
868
- return true;
869
- }
870
-
871
- // Serializar para armazenamento
872
- serialize(): Uint8Array {
873
- return this.bits;
874
- }
875
-
876
- // Restaurar de armazenamento
877
- static deserialize(data: Uint8Array, size: number, hashes: number): BloomFilter {
878
- const filter = new BloomFilter(1);
879
- filter.bits = data;
880
- filter.size = size;
881
- filter.hashes = hashes;
882
- return filter;
883
- }
884
- }
885
-
886
- // Uso
887
- const urlsSeen = new BloomFilter(100_000); // 100K URLs, 1% FPR
888
-
889
- async function shouldScrape(url: string): Promise<boolean> {
890
- if (urlsSeen.mightContain(url)) {
891
- console.log(`Probably seen: ${url}`);
892
- return false; // Pode ser falso positivo
893
- }
894
- urlsSeen.add(url);
895
- return true;
896
- }
897
- ```
898
-
899
- ### 6.3. ETag Conditional Requests
900
-
901
- ```typescript
902
- interface ETagCache {
903
- get(url: string): Promise<{ etag: string; lastModified: string } | null>;
904
- set(url: string, etag: string, lastModified: string): Promise<void>;
905
- }
906
-
907
- async function conditionalFetch(
908
- url: string,
909
- cache: ETagCache
910
- ): Promise<{ data: string; fromCache: boolean }> {
911
- const cached = await cache.get(url);
912
-
913
- const headers: Record<string, string> = {};
914
- if (cached?.etag) headers['If-None-Match'] = cached.etag;
915
- if (cached?.lastModified) headers['If-Modified-Since'] = cached.lastModified;
916
-
917
- const response = await fetch(url, { headers });
918
-
919
- if (response.status === 304) {
920
- // Não modificado — usar cache
921
- return { data: '', fromCache: true };
922
- }
923
-
924
- // Atualizar cache
925
- const etag = response.headers.get('etag');
926
- const lastModified = response.headers.get('last-modified');
927
- if (etag) await cache.set(url, etag, lastModified ?? '');
928
-
929
- const data = await response.text();
930
- return { data, fromCache: false };
931
- }
932
-
933
- // Implementação com Keyv
934
- class KeyvETagCache implements ETagCache {
935
- private store: Keyv;
936
-
937
- constructor() {
938
- this.store = new Keyv({ namespace: 'etag', ttl: 1000 * 60 * 60 * 24 });
939
- }
940
-
941
- async get(url: string) {
942
- return this.store.get(url) ?? null;
943
- }
944
-
945
- async set(url: string, etag: string, lastModified: string) {
946
- await this.store.set(url, { etag, lastModified });
947
- }
948
- }
949
- ```
950
-
951
- ---
952
-
953
- ## 7. Persistindo Estado
954
-
955
- ### 7.1. Cookies
956
-
957
- ```typescript
958
- // Cookies — persists across requests, limited to ~4KB
959
-
960
- // Definir cookie
961
- document.cookie = 'session=abc123; path=/; max-age=3600; SameSite=Lax; Secure';
962
-
963
- // Ler cookie
964
- function getCookie(name: string): string | null {
965
- const match = document.cookie.match(new RegExp(`(?:^|; )${name}=([^;]*)`));
966
- return match ? decodeURIComponent(match[1]) : null;
967
- }
968
-
969
- // Server-side (Set-Cookie header)
970
- // res.setHeader('Set-Cookie', 'session=abc123; Path=/; Max-Age=3600; HttpOnly; Secure');
971
-
972
- // Em fetch (não funciona cross-origin por padrão)
973
- await fetch(url, {
974
- credentials: 'include', // Enviar cookies
975
- });
976
- ```
977
-
978
- ### 7.2. Auth Tokens
979
-
980
- ```typescript
981
- interface AuthState {
982
- accessToken: string;
983
- refreshToken: string;
984
- expiresAt: number;
985
- }
986
-
987
- class AuthManager {
988
- private storage: Keyv<AuthState>;
989
-
990
- constructor() {
991
- this.storage = new Keyv({ namespace: 'auth', ttl: 1000 * 60 * 60 * 24 });
992
- }
993
-
994
- async saveTokens(url: string, tokens: AuthState): Promise<void> {
995
- await this.storage.set(this.key(url), tokens);
996
- }
997
-
998
- async getValidToken(url: string): Promise<string | null> {
999
- const tokens = await this.storage.get(this.key(url));
1000
- if (!tokens) return null;
1001
-
1002
- // Verificar se expirou
1003
- if (Date.now() > tokens.expiresAt) {
1004
- // Tentar refresh
1005
- const refreshed = await this.refresh(tokens.refreshToken);
1006
- if (refreshed) {
1007
- await this.saveTokens(url, refreshed);
1008
- return refreshed.accessToken;
1009
- }
1010
- await this.storage.delete(this.key(url));
1011
- return null;
1012
- }
1013
-
1014
- return tokens.accessToken;
1015
- }
1016
-
1017
- private key(url: string): string {
1018
- return new URL(url).origin;
1019
- }
1020
-
1021
- private async refresh(refreshToken: string): Promise<AuthState | null> {
1022
- try {
1023
- const response = await fetch('/auth/refresh', {
1024
- method: 'POST',
1025
- headers: { 'Content-Type': 'application/json' },
1026
- body: JSON.stringify({ refreshToken }),
1027
- });
1028
- return response.json();
1029
- } catch {
1030
- return null;
1031
- }
1032
- }
1033
- }
1034
- ```
1035
-
1036
- ### 7.3. Browser Fingerprints
1037
-
1038
- ```typescript
1039
- interface BrowserFingerprint {
1040
- canvas: string;
1041
- webgl: string;
1042
- audio: string;
1043
- fonts: string[];
1044
- screen: { width: number; height: number; colorDepth: number };
1045
- timezone: string;
1046
- language: string;
1047
- platform: string;
1048
- hash: string;
1049
- }
1050
-
1051
- async function generateFingerprint(): Promise<BrowserFingerprint> {
1052
- const components: string[] = [];
1053
-
1054
- // Canvas fingerprint
1055
- const canvas = document.createElement('canvas');
1056
- const ctx = canvas.getContext('2d')!;
1057
- ctx.textBaseline = 'top';
1058
- ctx.font = '14px Arial';
1059
- ctx.fillText('fingerprint', 2, 2);
1060
- const canvasHash = await hashString(canvas.toDataURL());
1061
- components.push(canvasHash);
1062
-
1063
- // WebGL fingerprint
1064
- const gl = document.createElement('canvas').getContext('webgl')!;
1065
- const debugInfo = gl.getExtension('WEBGL_debug_renderer_info');
1066
- const gpu = debugInfo
1067
- ? gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL)
1068
- : 'unknown';
1069
- const webglHash = await hashString(gpu);
1070
- components.push(webglHash);
1071
-
1072
- // Audio fingerprint
1073
- const audioCtx = new AudioContext();
1074
- const oscillator = audioCtx.createOscillator();
1075
- const analyser = audioCtx.createAnalyser();
1076
- const gain = audioCtx.createGain();
1077
- const scriptProcessor = audioCtx.createScriptProcessor(4096, 1, 1);
1078
-
1079
- oscillator.type = 'triangle';
1080
- oscillator.frequency.value = 10000;
1081
-
1082
- const buffer = new Float32Array(analyser.frequencyBinCount);
1083
- // ... coletar audio data
1084
- const audioHash = await hashString(buffer.toString());
1085
- components.push(audioHash);
1086
-
1087
- // Screen info
1088
- const screenInfo = `${screen.width}x${screen.height}x${screen.colorDepth}`;
1089
- components.push(screenInfo);
1090
-
1091
- // Timezone
1092
- const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
1093
- components.push(timezone);
1094
-
1095
- // Hash final
1096
- const hash = await hashString(components.join('|||'));
1097
-
1098
- return {
1099
- canvas: canvasHash,
1100
- webgl: webglHash,
1101
- audio: audioHash,
1102
- fonts: [],
1103
- screen: { width: screen.width, height: screen.height, colorDepth: screen.colorDepth },
1104
- timezone,
1105
- language: navigator.language,
1106
- platform: navigator.platform,
1107
- hash,
1108
- };
1109
- }
1110
-
1111
- async function hashString(str: string): Promise<string> {
1112
- const encoder = new TextEncoder();
1113
- const data = encoder.encode(str);
1114
- const hashBuffer = await crypto.subtle.digest('SHA-256', data);
1115
- return Array.from(new Uint8Array(hashBuffer))
1116
- .map(b => b.toString(16).padStart(2, '0'))
1117
- .join('');
1118
- }
1119
-
1120
- // Persistir fingerprint
1121
- class FingerprintManager {
1122
- private storage: Keyv<BrowserFingerprint>;
1123
-
1124
- constructor() {
1125
- this.storage = new Keyv({ namespace: 'fingerprint' });
1126
- }
1127
-
1128
- async getOrCreate(): Promise<BrowserFingerprint> {
1129
- const existing = await this.storage.get('current');
1130
- if (existing) return existing;
1131
-
1132
- const fp = await generateFingerprint();
1133
- await this.storage.set('current', fp);
1134
- return fp;
1135
- }
1136
-
1137
- async hasChanged(): Promise<boolean> {
1138
- const current = await this.storage.get('current');
1139
- if (!current) return false;
1140
-
1141
- const newFp = await generateFingerprint();
1142
- return current.hash !== newFp.hash;
1143
- }
1144
- }
1145
- ```
1146
-
1147
- ### 7.4. Checkpoints (resume scraping)
1148
-
1149
- ```typescript
1150
- interface ScrapeCheckpoint {
1151
- url: string;
1152
- pagesScraped: number;
1153
- lastPage: string;
1154
- extractedData: unknown[];
1155
- errors: string[];
1156
- timestamp: number;
1157
- }
1158
-
1159
- class CheckpointManager {
1160
- private storage: Keyv<ScrapeCheckpoint>;
1161
-
1162
- constructor() {
1163
- this.storage = new Keyv({
1164
- namespace: 'checkpoints',
1165
- ttl: 1000 * 60 * 60 * 24 * 7, // 7 dias
1166
- });
1167
- }
1168
-
1169
- async save(jobId: string, checkpoint: ScrapeCheckpoint): Promise<void> {
1170
- await this.storage.set(jobId, {
1171
- ...checkpoint,
1172
- timestamp: Date.now(),
1173
- });
1174
- console.log(`Checkpoint saved: ${checkpoint.pagesScraped} pages`);
1175
- }
1176
-
1177
- async resume(jobId: string): Promise<ScrapeCheckpoint | null> {
1178
- return this.storage.get(jobId) ?? null;
1179
- }
1180
-
1181
- async list(): Promise<ScrapeCheckpoint[]> {
1182
- const checkpoints: ScrapeCheckpoint[] = [];
1183
- for await (const [, value] of this.storage.iterator()) {
1184
- checkpoints.push(value);
1185
- }
1186
- return checkpoints.sort((a, b) => b.timestamp - a.timestamp);
1187
- }
1188
-
1189
- async delete(jobId: string): Promise<void> {
1190
- await this.storage.delete(jobId);
1191
- }
1192
- }
1193
-
1194
- // Uso: scraping com resume
1195
- async function scrapeWithResume(urls: string[]): Promise<void> {
1196
- const jobId = await hashString(urls.join('\n'));
1197
- const checkpointMgr = new CheckpointManager();
1198
-
1199
- // Tentar retomar de checkpoint
1200
- let checkpoint = await checkpointMgr.resume(jobId);
1201
- const startIdx = checkpoint?.pagesScraped ?? 0;
1202
- const data = checkpoint?.extractedData ?? [];
1203
-
1204
- for (let i = startIdx; i < urls.length; i++) {
1205
- try {
1206
- const result = await scrape(urls[i]);
1207
- data.push(result);
1208
-
1209
- // Save checkpoint a cada 10 páginas
1210
- if ((i + 1) % 10 === 0) {
1211
- await checkpointMgr.save(jobId, {
1212
- url: urls[i],
1213
- pagesScraped: i + 1,
1214
- lastPage: urls[i],
1215
- extractedData: data,
1216
- errors: checkpoint?.errors ?? [],
1217
- timestamp: Date.now(),
1218
- });
1219
- }
1220
- } catch (error) {
1221
- checkpoint = checkpoint ?? {
1222
- url: urls[i],
1223
- pagesScraped: i,
1224
- lastPage: urls[i],
1225
- extractedData: data,
1226
- errors: [],
1227
- timestamp: Date.now(),
1228
- };
1229
- checkpoint.errors.push(`${urls[i]}: ${error}`);
1230
- }
1231
- }
1232
-
1233
- // Limpar checkpoint após conclusão
1234
- await checkpointMgr.delete(jobId);
1235
- console.log(`Done! Scraped ${data.length} pages`);
1236
- }
1237
- ```
1238
-
1239
- ---
1240
-
1241
- ## 8. Browser-Compatible Storage Layer
1242
-
1243
- ### Runtime detection com fallback chain
1244
-
1245
- ```typescript
1246
- type StorageBackend = 'indexeddb' | 'localstorage' | 'memory';
1247
-
1248
- function detectStorageBackend(): StorageBackend {
1249
- // 1. IndexedDB (melhor para browsers)
1250
- if (typeof indexedDB !== 'undefined') return 'indexeddb';
1251
-
1252
- // 2. localStorage (fallback browsers)
1253
- if (typeof localStorage !== 'undefined') return 'localstorage';
1254
-
1255
- // 3. Memory (fallback universal)
1256
- return 'memory';
1257
- }
1258
-
1259
- // Universal storage interface
1260
- interface UniversalStorage {
1261
- get<T>(key: string): Promise<T | null>;
1262
- set<T>(key: string, value: T, ttlMs?: number): Promise<void>;
1263
- delete(key: string): Promise<void>;
1264
- has(key: string): Promise<boolean>;
1265
- clear(): Promise<void>;
1266
- }
1267
-
1268
- // Factory
1269
- function createStorage(namespace?: string): UniversalStorage {
1270
- const backend = detectStorageBackend();
1271
-
1272
- switch (backend) {
1273
- case 'indexeddb':
1274
- return new IndexedDBStorage(namespace ?? 'webscrape');
1275
- case 'localstorage':
1276
- return new LocalStorageStorage(namespace ?? 'ws_');
1277
- case 'memory':
1278
- return new MemoryStorage();
1279
- }
1280
- }
1281
-
1282
- // Memory storage
1283
- class MemoryStorage implements UniversalStorage {
1284
- private store = new Map<string, { value: unknown; expires?: number }>();
1285
-
1286
- async get<T>(key: string): Promise<T | null> {
1287
- const entry = this.store.get(key);
1288
- if (!entry) return null;
1289
- if (entry.expires && Date.now() > entry.expires) {
1290
- this.store.delete(key);
1291
- return null;
1292
- }
1293
- return entry.value as T;
1294
- }
1295
-
1296
- async set<T>(key: string, value: T, ttlMs?: number): Promise<void> {
1297
- this.store.set(key, {
1298
- value,
1299
- expires: ttlMs ? Date.now() + ttlMs : undefined,
1300
- });
1301
- }
1302
-
1303
- async delete(key: string): Promise<void> {
1304
- this.store.delete(key);
1305
- }
1306
-
1307
- async has(key: string): Promise<boolean> {
1308
- return (await this.get(key)) !== null;
1309
- }
1310
-
1311
- async clear(): Promise<void> {
1312
- this.store.clear();
1313
- }
1314
- }
1315
-
1316
- // LocalStorage storage
1317
- class LocalStorageStorage implements UniversalStorage {
1318
- private prefix: string;
1319
-
1320
- constructor(prefix: string) {
1321
- this.prefix = prefix;
1322
- }
1323
-
1324
- async get<T>(key: string): Promise<T | null> {
1325
- const raw = localStorage.getItem(this.prefix + key);
1326
- if (!raw) return null;
1327
- try {
1328
- const item = JSON.parse(raw);
1329
- if (item.expires && Date.now() > item.expires) {
1330
- this.delete(key);
1331
- return null;
1332
- }
1333
- return item.value;
1334
- } catch {
1335
- return null;
1336
- }
1337
- }
1338
-
1339
- async set<T>(key: string, value: T, ttlMs?: number): Promise<void> {
1340
- localStorage.setItem(this.prefix + key, JSON.stringify({
1341
- value,
1342
- expires: ttlMs ? Date.now() + ttlMs : undefined,
1343
- }));
1344
- }
1345
-
1346
- async delete(key: string): Promise<void> {
1347
- localStorage.removeItem(this.prefix + key);
1348
- }
1349
-
1350
- async has(key: string): Promise<boolean> {
1351
- return (await this.get(key)) !== null;
1352
- }
1353
-
1354
- async clear(): Promise<void> {
1355
- const keys = Object.keys(localStorage)
1356
- .filter(k => k.startsWith(this.prefix));
1357
- keys.forEach(k => localStorage.removeItem(k));
1358
- }
1359
- }
1360
-
1361
- // IndexedDB storage
1362
- class IndexedDBStorage implements UniversalStorage {
1363
- private dbName: string;
1364
- private db: IDBDatabase | null = null;
1365
-
1366
- constructor(dbName: string) {
1367
- this.dbName = dbName;
1368
- }
1369
-
1370
- private async init(): Promise<IDBDatabase> {
1371
- if (this.db) return this.db;
1372
-
1373
- return new Promise((resolve, reject) => {
1374
- const request = indexedDB.open(this.dbName, 1);
1375
-
1376
- request.onupgradeneeded = () => {
1377
- const db = request.result;
1378
- if (!db.objectStoreNames.contains('store')) {
1379
- db.createObjectStore('store', { keyPath: 'key' });
1380
- }
1381
- };
1382
-
1383
- request.onsuccess = () => {
1384
- this.db = request.result;
1385
- resolve(this.db);
1386
- };
1387
-
1388
- request.onerror = () => reject(request.error);
1389
- });
1390
- }
1391
-
1392
- async get<T>(key: string): Promise<T | null> {
1393
- const db = await this.init();
1394
- return new Promise((resolve) => {
1395
- const tx = db.transaction('store', 'readonly');
1396
- const store = tx.objectStore('store');
1397
- const request = store.get(key);
1398
-
1399
- request.onsuccess = () => {
1400
- const result = request.result;
1401
- if (!result) return resolve(null);
1402
- if (result.expires && Date.now() > result.expires) {
1403
- this.delete(key);
1404
- return resolve(null);
1405
- }
1406
- resolve(result.value);
1407
- };
1408
-
1409
- request.onerror = () => resolve(null);
1410
- });
1411
- }
1412
-
1413
- async set<T>(key: string, value: T, ttlMs?: number): Promise<void> {
1414
- const db = await this.init();
1415
- return new Promise((resolve, reject) => {
1416
- const tx = db.transaction('store', 'readwrite');
1417
- const store = tx.objectStore('store');
1418
- const request = store.put({
1419
- key,
1420
- value,
1421
- expires: ttlMs ? Date.now() + ttlMs : undefined,
1422
- });
1423
-
1424
- request.onsuccess = () => resolve();
1425
- request.onerror = () => reject(request.error);
1426
- });
1427
- }
1428
-
1429
- async delete(key: string): Promise<void> {
1430
- const db = await this.init();
1431
- return new Promise((resolve, reject) => {
1432
- const tx = db.transaction('store', 'readwrite');
1433
- const store = tx.objectStore('store');
1434
- const request = store.delete(key);
1435
-
1436
- request.onsuccess = () => resolve();
1437
- request.onerror = () => reject(request.error);
1438
- });
1439
- }
1440
-
1441
- async has(key: string): Promise<boolean> {
1442
- return (await this.get(key)) !== null;
1443
- }
1444
-
1445
- async clear(): Promise<void> {
1446
- const db = await this.init();
1447
- return new Promise((resolve, reject) => {
1448
- const tx = db.transaction('store', 'readwrite');
1449
- const store = tx.objectStore('store');
1450
- const request = store.clear();
1451
-
1452
- request.onsuccess = () => resolve();
1453
- request.onerror = () => reject(request.error);
1454
- });
1455
- }
1456
- }
1457
- ```
1458
-
1459
- ---
1460
-
1461
- ## 9. Storage Lifecycle
1462
-
1463
- ### 9.1. TTL Strategies
1464
-
1465
- #### Eager eviction (remove on access)
1466
-
1467
- ```typescript
1468
- // Remove item imediatamente quando expirado
1469
- class EagerTTLCache<T> {
1470
- private store = new Map<string, { value: T; expires: number }>();
1471
-
1472
- get(key: string): T | null {
1473
- const entry = this.store.get(key);
1474
- if (!entry) return null;
1475
-
1476
- if (Date.now() > entry.expires) {
1477
- this.store.delete(key); // Eager: remove agora
1478
- return null;
1479
- }
1480
-
1481
- return entry.value;
1482
- }
1483
-
1484
- set(key: string, value: T, ttlMs: number): void {
1485
- this.store.set(key, {
1486
- value,
1487
- expires: Date.now() + ttlMs,
1488
- });
1489
- }
1490
- }
1491
- ```
1492
-
1493
- #### Lazy eviction (remove in background)
1494
-
1495
- ```typescript
1496
- // Remove expiradas em background interval
1497
- class LazyTTLCache<T> {
1498
- private store = new Map<string, { value: T; expires: number }>();
1499
- private cleanupInterval: ReturnType<typeof setInterval>;
1500
-
1501
- constructor(cleanupMs = 60_000) {
1502
- this.cleanupInterval = setInterval(() => {
1503
- this.cleanup();
1504
- }, cleanupMs);
1505
- }
1506
-
1507
- private cleanup(): void {
1508
- const now = Date.now();
1509
- for (const [key, entry] of this.store) {
1510
- if (now > entry.expires) {
1511
- this.store.delete(key);
1512
- }
1513
- }
1514
- }
1515
-
1516
- destroy(): void {
1517
- clearInterval(this.cleanupInterval);
1518
- }
1519
-
1520
- get(key: string): T | null {
1521
- const entry = this.store.get(key);
1522
- return entry?.value ?? null;
1523
- }
1524
-
1525
- set(key: string, value: T, ttlMs: number): void {
1526
- this.store.set(key, {
1527
- value,
1528
- expires: Date.now() + ttlMs,
1529
- });
1530
- }
1531
- }
1532
- ```
1533
-
1534
- #### Scheduled eviction (event-driven)
1535
-
1536
- ```typescript
1537
- // Agenda limpeza baseada no próximo item que expira
1538
- class ScheduledTTLCache<T> {
1539
- private store = new Map<string, { value: T; expires: number }>();
1540
- private nextCleanup: ReturnType<typeof setTimeout> | null = null;
1541
-
1542
- set(key: string, value: T, ttlMs: number): void {
1543
- const expires = Date.now() + ttlMs;
1544
- this.store.set(key, { value, expires });
1545
-
1546
- // Agendar cleanup se este expira antes do próximo
1547
- if (!this.nextCleanup || expires < this.nextCleanupAt) {
1548
- this.scheduleCleanup(expires);
1549
- }
1550
- }
1551
-
1552
- get(key: string): T | null {
1553
- return this.store.get(key)?.value ?? null;
1554
- }
1555
-
1556
- private nextCleanupAt = 0;
1557
-
1558
- private scheduleCleanup(at: number): void {
1559
- if (this.nextCleanup) clearTimeout(this.nextCleanup);
1560
- this.nextCleanupAt = at;
1561
- this.nextCleanup = setTimeout(() => {
1562
- this.cleanup();
1563
- }, Math.max(0, at - Date.now()));
1564
- }
1565
-
1566
- private cleanup(): void {
1567
- const now = Date.now();
1568
- for (const [key, entry] of this.store) {
1569
- if (now > entry.expires) {
1570
- this.store.delete(key);
1571
- }
1572
- }
1573
-
1574
- // Reagendar baseado no próximo item
1575
- let nextExpiry = Infinity;
1576
- for (const entry of this.store.values()) {
1577
- if (entry.expires < nextExpiry) {
1578
- nextExpiry = entry.expires;
1579
- }
1580
- }
1581
- if (nextExpiry < Infinity) {
1582
- this.scheduleCleanup(nextExpiry);
1583
- }
1584
- }
1585
- }
1586
- ```
1587
-
1588
- ### 9.2. Eviction Strategies
1589
-
1590
- ```typescript
1591
- // LRU — Least Recently Used
1592
- class LRUCache<T> {
1593
- private store = new Map<string, { value: T; lastAccess: number }>();
1594
- private maxSize: number;
1595
-
1596
- constructor(maxSize = 100) {
1597
- this.maxSize = maxSize;
1598
- }
1599
-
1600
- get(key: string): T | null {
1601
- const entry = this.store.get(key);
1602
- if (!entry) return null;
1603
- entry.lastAccess = Date.now();
1604
- return entry.value;
1605
- }
1606
-
1607
- set(key: string, value: T): void {
1608
- if (this.store.size >= this.maxSize) {
1609
- this.evict();
1610
- }
1611
- this.store.set(key, { value, lastAccess: Date.now() });
1612
- }
1613
-
1614
- private evict(): void {
1615
- let oldest: string | null = null;
1616
- let oldestTime = Infinity;
1617
-
1618
- for (const [key, entry] of this.store) {
1619
- if (entry.lastAccess < oldestTime) {
1620
- oldestTime = entry.lastAccess;
1621
- oldest = key;
1622
- }
1623
- }
1624
-
1625
- if (oldest) this.store.delete(oldest);
1626
- }
1627
- }
1628
-
1629
- // LFU — Least Frequently Used
1630
- class LFUCache<T> {
1631
- private store = new Map<string, { value: T; count: number }>();
1632
- private maxSize: number;
1633
-
1634
- constructor(maxSize = 100) {
1635
- this.maxSize = maxSize;
1636
- }
1637
-
1638
- get(key: string): T | null {
1639
- const entry = this.store.get(key);
1640
- if (!entry) return null;
1641
- entry.count++;
1642
- return entry.value;
1643
- }
1644
-
1645
- set(key: string, value: T): void {
1646
- if (this.store.size >= this.maxSize) {
1647
- this.evict();
1648
- }
1649
- this.store.set(key, { value, count: 0 });
1650
- }
1651
-
1652
- private evict(): void {
1653
- let leastUsed: string | null = null;
1654
- let minCount = Infinity;
1655
-
1656
- for (const [key, entry] of this.store) {
1657
- if (entry.count < minCount) {
1658
- minCount = entry.count;
1659
- leastUsed = key;
1660
- }
1661
- }
1662
-
1663
- if (leastUsed) this.store.delete(leastUsed);
1664
- }
1665
- }
1666
-
1667
- // FIFO — First In First Out
1668
- class FIFOCache<T> {
1669
- private store = new Map<string, T>();
1670
- private maxSize: number;
1671
-
1672
- constructor(maxSize = 100) {
1673
- this.maxSize = maxSize;
1674
- }
1675
-
1676
- get(key: string): T | null {
1677
- return this.store.get(key) ?? null;
1678
- }
1679
-
1680
- set(key: string, value: T): void {
1681
- if (this.store.size >= this.maxSize) {
1682
- // Remove a primeira entrada (mais antiga)
1683
- const firstKey = this.store.keys().next().value;
1684
- if (firstKey) this.store.delete(firstKey);
1685
- }
1686
- this.store.set(key, value);
1687
- }
1688
- }
1689
-
1690
- // Size-based eviction
1691
- class SizeBasedCache<T> {
1692
- private store = new Map<string, { value: T; size: number }>();
1693
- private maxSize: number;
1694
- private currentSize = 0;
1695
-
1696
- constructor(maxSize: number) {
1697
- this.maxSize = maxSize;
1698
- }
1699
-
1700
- get(key: string): T | null {
1701
- return this.store.get(key)?.value ?? null;
1702
- }
1703
-
1704
- set(key: string, value: T, sizeFn: (v: T) => number): void {
1705
- const size = sizeFn(value);
1706
-
1707
- // Remove entradas até caber
1708
- while (this.currentSize + size > this.maxSize && this.store.size > 0) {
1709
- const firstKey = this.store.keys().next().value;
1710
- if (firstKey) {
1711
- this.currentSize -= this.store.get(firstKey)!.size;
1712
- this.store.delete(firstKey);
1713
- }
1714
- }
1715
-
1716
- this.store.set(key, { value, size });
1717
- this.currentSize += size;
1718
- }
1719
- }
1720
- ```
1721
-
1722
- ---
1723
-
1724
- ## 10. Best Practices
1725
-
1726
- ### Separar stores por volatilidade
1727
-
1728
- ```
1729
- Cache Architecture:
1730
- ├── Hot Cache (in-memory, LRU, 1K items, TTL 5min)
1731
- │ └── Requisições frequentes, dados que mudam rápido
1732
- ├── Warm Cache (SQLite/Redis, 100K items, TTL 1 hora)
1733
- │ └── Dados semi-estáveis, sessões de usuário
1734
- ├── Cold Cache (IndexedDB/KV, unlimited, TTL 7 dias)
1735
- │ └── Dados estáticos, checkpoints, logs
1736
- └── Archive (Disk, unlimited, sem TTL)
1737
- └── Resultados de scraping permanentes
1738
- ```
1739
-
1740
- ### Compression
1741
-
1742
- ```typescript
1743
- // Comprimir dados grandes antes de armazenar
1744
- async function compress(data: string): Promise<Uint8Array> {
1745
- const stream = new Blob([data]).stream();
1746
- const compressed = stream.pipeThrough(new CompressionStream('gzip'));
1747
- return new Uint8Array(await new Response(compressed).arrayBuffer());
1748
- }
1749
-
1750
- async function decompress(data: Uint8Array): Promise<string> {
1751
- const stream = new Blob([data]).stream();
1752
- const decompressed = stream.pipeThrough(new DecompressionStream('gzip'));
1753
- return new Response(decompressed).text();
1754
- }
1755
-
1756
- // Uso
1757
- const largeData = JSON.stringify(bigObject);
1758
- const compressed = await compress(largeData);
1759
- await keyv.set('key', compressed);
1760
- // Reduz 60-80% do tamanho
1761
- ```
1762
-
1763
- ### Atomic writes
1764
-
1765
- ```typescript
1766
- // Usar transações para writes atômicos
1767
- async function atomicWrite<T>(
1768
- key: string,
1769
- value: T,
1770
- storage: UniversalStorage
1771
- ): Promise<void> {
1772
- // Write temporário
1773
- const tempKey = `${key}:temp:${Date.now()}`;
1774
- await storage.set(tempKey, value);
1775
-
1776
- // Rename atômico (não existe em todos os storages)
1777
- // Para Keyv: usar transação do adapter
1778
- const existing = await storage.get(key);
1779
- try {
1780
- await storage.set(key, value);
1781
- await storage.delete(tempKey);
1782
- } catch (error) {
1783
- // Rollback
1784
- if (existing) await storage.set(key, existing);
1785
- await storage.delete(tempKey);
1786
- throw error;
1787
- }
1788
- }
1789
- ```
1790
-
1791
- ### Graceful degradation
1792
-
1793
- ```typescript
1794
- async function resilientGet<T>(key: string): Promise<T | null> {
1795
- const stores: UniversalStorage[] = [
1796
- memoryCache, // Rápido, volatile
1797
- sqliteCache, // Persistente, síncrono
1798
- redisCache, // Persistente, async
1799
- ];
1800
-
1801
- for (const store of stores) {
1802
- try {
1803
- const value = await store.get<T>(key);
1804
- if (value !== null) return value;
1805
- } catch (error) {
1806
- console.warn(`Storage ${store.constructor.name} failed:`, error);
1807
- continue;
1808
- }
1809
- }
1810
-
1811
- return null;
1812
- }
1813
-
1814
- async function resilientSet<T>(key: string, value: T, ttlMs?: number): Promise<void> {
1815
- const stores: UniversalStorage[] = [
1816
- memoryCache,
1817
- sqliteCache,
1818
- redisCache,
1819
- ];
1820
-
1821
- for (const store of stores) {
1822
- try {
1823
- await store.set(key, value, ttlMs);
1824
- } catch (error) {
1825
- console.warn(`Storage ${store.constructor.name} write failed:`, error);
1826
- continue;
1827
- }
1828
- }
1829
- }
1830
- ```
1831
-
1832
- ---
1833
-
1834
- ## 11. Exemplos Completos
1835
-
1836
- ### 11.1. Multi-tier scraping cache
1837
-
1838
- ```typescript
1839
- class MultiTierCache<T> {
1840
- private hot: LRUCache<T>;
1841
- private warm: Keyv<T>;
1842
- private cold: UniversalStorage;
1843
-
1844
- constructor() {
1845
- this.hot = new LRUCache<T>({ max: 1000, ttl: 1000 * 60 * 5 });
1846
- this.warm = new Keyv({
1847
- namespace: 'warm',
1848
- ttl: 1000 * 60 * 60,
1849
- });
1850
- this.cold = createStorage('cold');
1851
- }
1852
-
1853
- async get(key: string): Promise<T | null> {
1854
- // Hot
1855
- const hotValue = this.hot.get(key);
1856
- if (hotValue) return hotValue;
1857
-
1858
- // Warm
1859
- const warmValue = await this.warm.get(key);
1860
- if (warmValue) {
1861
- this.hot.set(key, warmValue);
1862
- return warmValue;
1863
- }
1864
-
1865
- // Cold
1866
- const coldValue = await this.cold.get<T>(key);
1867
- if (coldValue) {
1868
- this.hot.set(key, coldValue);
1869
- await this.warm.set(key, coldValue);
1870
- return coldValue;
1871
- }
1872
-
1873
- return null;
1874
- }
1875
-
1876
- async set(key: string, value: T): Promise<void> {
1877
- this.hot.set(key, value);
1878
- await this.warm.set(key, value);
1879
- await this.cold.set(key, value);
1880
- }
1881
-
1882
- async invalidate(key: string): Promise<void> {
1883
- this.hot.delete(key);
1884
- await this.warm.delete(key);
1885
- await this.cold.delete(key);
1886
- }
1887
- }
1888
- ```
1889
-
1890
- ### 11.2. Persistent scraping session
1891
-
1892
- ```typescript
1893
- class ScrapingSession {
1894
- private checkpointMgr: CheckpointManager;
1895
- private authMgr: AuthManager;
1896
- private cache: MultiTierCache<unknown>;
1897
-
1898
- constructor() {
1899
- this.checkpointMgr = new CheckpointManager();
1900
- this.authMgr = new AuthManager();
1901
- this.cache = new MultiTierCache();
1902
- }
1903
-
1904
- async scrape(urls: string[]): Promise<void> {
1905
- const jobId = await this.generateJobId(urls);
1906
-
1907
- // Restaurar de checkpoint
1908
- const checkpoint = await this.checkpointMgr.resume(jobId);
1909
- const startIdx = checkpoint?.pagesScraped ?? 0;
1910
-
1911
- for (let i = startIdx; i < urls.length; i++) {
1912
- const url = urls[i];
1913
-
1914
- // Verificar cache
1915
- const cached = await this.cache.get(`scrape:${url}`);
1916
- if (cached) {
1917
- console.log(`Cache hit: ${url}`);
1918
- continue;
1919
- }
1920
-
1921
- // Obter auth se necessário
1922
- const token = await this.authMgr.getValidToken(url);
1923
-
1924
- // Scrape
1925
- try {
1926
- const result = await fetch(url, {
1927
- headers: token ? { Authorization: `Bearer ${token}` } : {},
1928
- });
1929
-
1930
- const data = await result.json();
1931
- await this.cache.set(`scrape:${url}`, data);
1932
-
1933
- // Checkpoint a cada 10 páginas
1934
- if ((i + 1) % 10 === 0) {
1935
- await this.checkpointMgr.save(jobId, {
1936
- url,
1937
- pagesScraped: i + 1,
1938
- lastPage: url,
1939
- extractedData: [],
1940
- errors: [],
1941
- timestamp: Date.now(),
1942
- });
1943
- }
1944
- } catch (error) {
1945
- console.error(`Failed: ${url}`, error);
1946
- }
1947
- }
1948
- }
1949
-
1950
- private async generateJobId(urls: string[]): Promise<string> {
1951
- return crypto.subtle
1952
- .digest('SHA-256', new TextEncoder().encode(urls.join('\n')))
1953
- .then(buf => Array.from(new Uint8Array(buf))
1954
- .map(b => b.toString(16).padStart(2, '0'))
1955
- .join('')
1956
- .slice(0, 16));
1957
- }
1958
- }
1959
- ```
1960
-
1961
- ---
1962
-
1963
- ## 12. Checklist
1964
-
1965
- - [ ] Usar Map para caches simples, lru-cache para bounded
1966
- - [ ] WeakMap para metadata associada a objects
1967
- - [ ] Browser: IndexedDB para dados grandes, localStorage para pequenos
1968
- - [ ] Server: better-sqlite3 para síncrono, Redis para distribuído
1969
- - [ ] Cross-runtime: keyv 5.6.0 com adapters pluggable
1970
- - [ ] Edge: Cloudflare KV, Upstash Redis, Deno KV
1971
- - [ ] Request dedup com Keyv TTL ou Bloom filter
1972
- - [ ] ETag conditional requests para freshness
1973
- - [ ] Persistir: auth tokens, checkpoints, fingerprints
1974
- - [ ] Separar stores por volatilidade (hot/warm/cold)
1975
- - [ ] TTL: eager para hot, lazy para warm, scheduled para cold
1976
- - [ ] Eviction: LRU para hot, size-based para warm
1977
- - [ ] Compression para dados > 10KB
1978
- - [ ] Graceful degradation: memory → SQLite → Redis
1979
- - [ ] Atomic writes para consistência