@nitpicker/archive 0.0.0 → 0.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (739) hide show
  1. package/LICENSE +191 -0
  2. package/README.md +106 -0
  3. package/lib/apply-connection-pragmas.d.ts +18 -0
  4. package/lib/apply-connection-pragmas.js +40 -0
  5. package/lib/archive-accessor.d.ts +265 -0
  6. package/lib/archive-accessor.js +477 -0
  7. package/lib/archive-lock.d.ts +45 -0
  8. package/lib/archive-lock.js +154 -0
  9. package/lib/archive.d.ts +736 -0
  10. package/lib/archive.js +1055 -0
  11. package/lib/body-hash/compute-body-hash.d.ts +32 -0
  12. package/lib/body-hash/compute-body-hash.js +35 -0
  13. package/lib/cache/clear-archive-cache-entry.d.ts +18 -0
  14. package/lib/cache/clear-archive-cache-entry.js +24 -0
  15. package/lib/cache/clear-archive-cache-root.d.ts +26 -0
  16. package/lib/cache/clear-archive-cache-root.js +32 -0
  17. package/lib/cache/compute-archive-cache-key.d.ts +39 -0
  18. package/lib/cache/compute-archive-cache-key.js +95 -0
  19. package/lib/cache/extract-archive-to-cache.d.ts +57 -0
  20. package/lib/cache/extract-archive-to-cache.js +326 -0
  21. package/lib/cache/get-archive-cache-root.d.ts +20 -0
  22. package/lib/cache/get-archive-cache-root.js +53 -0
  23. package/lib/cache/is-archive-cache-disabled.d.ts +24 -0
  24. package/lib/cache/is-archive-cache-disabled.js +34 -0
  25. package/lib/cache/list-archive-cache-entries.d.ts +23 -0
  26. package/lib/cache/list-archive-cache-entries.js +130 -0
  27. package/lib/cache/path-exists.d.ts +9 -0
  28. package/lib/cache/path-exists.js +18 -0
  29. package/lib/cache/resolve-archive-cache-dir.d.ts +26 -0
  30. package/lib/cache/resolve-archive-cache-dir.js +32 -0
  31. package/lib/cache/types.d.ts +33 -0
  32. package/lib/cache/types.js +1 -0
  33. package/lib/content-type/is-html-content-type.d.ts +17 -0
  34. package/lib/content-type/is-html-content-type.js +19 -0
  35. package/lib/content-type/normalize-content-type.d.ts +14 -0
  36. package/lib/content-type/normalize-content-type.js +20 -0
  37. package/lib/create-adjunct-tables.d.ts +66 -0
  38. package/lib/create-adjunct-tables.js +510 -0
  39. package/lib/create-entity-tables.d.ts +191 -0
  40. package/lib/create-entity-tables.js +387 -0
  41. package/lib/create-progress-reporter.d.ts +30 -0
  42. package/lib/create-progress-reporter.js +38 -0
  43. package/lib/create-ref-tables.d.ts +35 -0
  44. package/lib/create-ref-tables.js +185 -0
  45. package/lib/create-template-tables.d.ts +28 -0
  46. package/lib/create-template-tables.js +92 -0
  47. package/lib/database.d.ts +549 -0
  48. package/lib/database.js +803 -0
  49. package/lib/db-ops/_shared/clear-write-ref-caches.d.ts +27 -0
  50. package/lib/db-ops/_shared/clear-write-ref-caches.js +35 -0
  51. package/lib/db-ops/_shared/compress-payload.d.ts +26 -0
  52. package/lib/db-ops/_shared/compress-payload.js +30 -0
  53. package/lib/db-ops/_shared/create-write-ref-caches.d.ts +17 -0
  54. package/lib/db-ops/_shared/create-write-ref-caches.js +27 -0
  55. package/lib/db-ops/_shared/decode-json-ref.d.ts +17 -0
  56. package/lib/db-ops/_shared/decode-json-ref.js +31 -0
  57. package/lib/db-ops/_shared/keyset-paginate-by-id.d.ts +49 -0
  58. package/lib/db-ops/_shared/keyset-paginate-by-id.js +69 -0
  59. package/lib/db-ops/_shared/load-response-headers-by-set-ids.d.ts +20 -0
  60. package/lib/db-ops/_shared/load-response-headers-by-set-ids.js +53 -0
  61. package/lib/db-ops/_shared/resolve-content-item-id.d.ts +62 -0
  62. package/lib/db-ops/_shared/resolve-content-item-id.js +143 -0
  63. package/lib/db-ops/_shared/resolve-url-or-blob.d.ts +23 -0
  64. package/lib/db-ops/_shared/resolve-url-or-blob.js +29 -0
  65. package/lib/db-ops/_shared/retry-setting.d.ts +16 -0
  66. package/lib/db-ops/_shared/retry-setting.js +18 -0
  67. package/lib/db-ops/_shared/safe-parse-json.d.ts +11 -0
  68. package/lib/db-ops/_shared/safe-parse-json.js +18 -0
  69. package/lib/db-ops/_shared/types.d.ts +94 -0
  70. package/lib/db-ops/_shared/types.js +1 -0
  71. package/lib/db-ops/_shared/upsert-blob-ref.d.ts +25 -0
  72. package/lib/db-ops/_shared/upsert-blob-ref.js +48 -0
  73. package/lib/db-ops/_shared/upsert-content-type-ref.d.ts +30 -0
  74. package/lib/db-ops/_shared/upsert-content-type-ref.js +45 -0
  75. package/lib/db-ops/_shared/upsert-json-ref.d.ts +22 -0
  76. package/lib/db-ops/_shared/upsert-json-ref.js +41 -0
  77. package/lib/db-ops/_shared/upsert-response-headers.d.ts +35 -0
  78. package/lib/db-ops/_shared/upsert-response-headers.js +49 -0
  79. package/lib/db-ops/_shared/upsert-url-ref.d.ts +39 -0
  80. package/lib/db-ops/_shared/upsert-url-ref.js +62 -0
  81. package/lib/db-ops/anchors/get-anchors-on-page.d.ts +10 -0
  82. package/lib/db-ops/anchors/get-anchors-on-page.js +21 -0
  83. package/lib/db-ops/config/get-base-url.d.ts +8 -0
  84. package/lib/db-ops/config/get-base-url.js +14 -0
  85. package/lib/db-ops/config/get-config.d.ts +10 -0
  86. package/lib/db-ops/config/get-config.js +29 -0
  87. package/lib/db-ops/config/get-created-cwd.d.ts +11 -0
  88. package/lib/db-ops/config/get-created-cwd.js +13 -0
  89. package/lib/db-ops/config/get-name.d.ts +8 -0
  90. package/lib/db-ops/config/get-name.js +14 -0
  91. package/lib/db-ops/config/info-column-allowlist.d.ts +7 -0
  92. package/lib/db-ops/config/info-column-allowlist.js +29 -0
  93. package/lib/db-ops/config/info-json-columns.d.ts +5 -0
  94. package/lib/db-ops/config/info-json-columns.js +11 -0
  95. package/lib/db-ops/config/set-config.d.ts +12 -0
  96. package/lib/db-ops/config/set-config.js +21 -0
  97. package/lib/db-ops/config/update-config.d.ts +17 -0
  98. package/lib/db-ops/config/update-config.js +36 -0
  99. package/lib/db-ops/console-logs/compute-console-log-hash.d.ts +19 -0
  100. package/lib/db-ops/console-logs/compute-console-log-hash.js +30 -0
  101. package/lib/db-ops/console-logs/replace-console-logs.d.ts +34 -0
  102. package/lib/db-ops/console-logs/replace-console-logs.js +123 -0
  103. package/lib/db-ops/console-logs/stringify-console-log-args.d.ts +16 -0
  104. package/lib/db-ops/console-logs/stringify-console-log-args.js +26 -0
  105. package/lib/db-ops/console-logs/types.d.ts +38 -0
  106. package/lib/db-ops/console-logs/types.js +1 -0
  107. package/lib/db-ops/console-logs/upsert-console-log-item.d.ts +29 -0
  108. package/lib/db-ops/console-logs/upsert-console-log-item.js +52 -0
  109. package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts +18 -0
  110. package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js +23 -0
  111. package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts +12 -0
  112. package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.js +15 -0
  113. package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts +14 -0
  114. package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.js +30 -0
  115. package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.d.ts +57 -0
  116. package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.js +87 -0
  117. package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts +21 -0
  118. package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js +27 -0
  119. package/lib/db-ops/errors/insert-crawl-error.d.ts +15 -0
  120. package/lib/db-ops/errors/insert-crawl-error.js +21 -0
  121. package/lib/db-ops/errors/insert-page-error.d.ts +21 -0
  122. package/lib/db-ops/errors/insert-page-error.js +30 -0
  123. package/lib/db-ops/errors/list-dns-burned-host-candidates.d.ts +33 -0
  124. package/lib/db-ops/errors/list-dns-burned-host-candidates.js +158 -0
  125. package/lib/db-ops/html/read-page-html-blob.d.ts +17 -0
  126. package/lib/db-ops/html/read-page-html-blob.js +20 -0
  127. package/lib/db-ops/lifecycle/checkpoint.d.ts +8 -0
  128. package/lib/db-ops/lifecycle/checkpoint.js +9 -0
  129. package/lib/db-ops/lifecycle/destroy.d.ts +6 -0
  130. package/lib/db-ops/lifecycle/destroy.js +7 -0
  131. package/lib/db-ops/lifecycle/init.d.ts +58 -0
  132. package/lib/db-ops/lifecycle/init.js +122 -0
  133. package/lib/db-ops/list-reconcile/record-list-reconcile-run.d.ts +22 -0
  134. package/lib/db-ops/list-reconcile/record-list-reconcile-run.js +41 -0
  135. package/lib/db-ops/meta/get-audios-of-page.d.ts +11 -0
  136. package/lib/db-ops/meta/get-audios-of-page.js +15 -0
  137. package/lib/db-ops/meta/get-buttons-of-page.d.ts +11 -0
  138. package/lib/db-ops/meta/get-buttons-of-page.js +15 -0
  139. package/lib/db-ops/meta/get-canvases-of-page.d.ts +11 -0
  140. package/lib/db-ops/meta/get-canvases-of-page.js +15 -0
  141. package/lib/db-ops/meta/get-custom-elements-of-page.d.ts +11 -0
  142. package/lib/db-ops/meta/get-custom-elements-of-page.js +15 -0
  143. package/lib/db-ops/meta/get-headings-of-page.d.ts +11 -0
  144. package/lib/db-ops/meta/get-headings-of-page.js +15 -0
  145. package/lib/db-ops/meta/get-iframes-of-page.d.ts +11 -0
  146. package/lib/db-ops/meta/get-iframes-of-page.js +15 -0
  147. package/lib/db-ops/meta/get-jsonld-of-page.d.ts +13 -0
  148. package/lib/db-ops/meta/get-jsonld-of-page.js +27 -0
  149. package/lib/db-ops/meta/get-main-content-images-of-page.d.ts +11 -0
  150. package/lib/db-ops/meta/get-main-content-images-of-page.js +15 -0
  151. package/lib/db-ops/meta/get-main-content-tables-of-page.d.ts +11 -0
  152. package/lib/db-ops/meta/get-main-content-tables-of-page.js +15 -0
  153. package/lib/db-ops/meta/get-page-technologies-of-page.d.ts +11 -0
  154. package/lib/db-ops/meta/get-page-technologies-of-page.js +15 -0
  155. package/lib/db-ops/meta/get-technology-signals-of-page.d.ts +11 -0
  156. package/lib/db-ops/meta/get-technology-signals-of-page.js +15 -0
  157. package/lib/db-ops/meta/get-videos-of-page.d.ts +11 -0
  158. package/lib/db-ops/meta/get-videos-of-page.js +15 -0
  159. package/lib/db-ops/outages/close-network-outage.d.ts +12 -0
  160. package/lib/db-ops/outages/close-network-outage.js +15 -0
  161. package/lib/db-ops/outages/close-stale-open-network-outages.d.ts +26 -0
  162. package/lib/db-ops/outages/close-stale-open-network-outages.js +42 -0
  163. package/lib/db-ops/outages/compute-outage-clamp-timestamp.d.ts +26 -0
  164. package/lib/db-ops/outages/compute-outage-clamp-timestamp.js +31 -0
  165. package/lib/db-ops/outages/insert-network-outage.d.ts +14 -0
  166. package/lib/db-ops/outages/insert-network-outage.js +29 -0
  167. package/lib/db-ops/outages/list-network-outages.d.ts +28 -0
  168. package/lib/db-ops/outages/list-network-outages.js +42 -0
  169. package/lib/db-ops/pages/order/set-url-order.d.ts +13 -0
  170. package/lib/db-ops/pages/order/set-url-order.js +38 -0
  171. package/lib/db-ops/pages/read/build-page-query.d.ts +18 -0
  172. package/lib/db-ops/pages/read/build-page-query.js +40 -0
  173. package/lib/db-ops/pages/read/get-crawling-state.d.ts +117 -0
  174. package/lib/db-ops/pages/read/get-crawling-state.js +161 -0
  175. package/lib/db-ops/pages/read/get-existing-page-urls.d.ts +15 -0
  176. package/lib/db-ops/pages/read/get-existing-page-urls.js +30 -0
  177. package/lib/db-ops/pages/read/get-page-count.d.ts +12 -0
  178. package/lib/db-ops/pages/read/get-page-count.js +21 -0
  179. package/lib/db-ops/pages/read/get-page-source-by-url.d.ts +24 -0
  180. package/lib/db-ops/pages/read/get-page-source-by-url.js +28 -0
  181. package/lib/db-ops/pages/read/get-pages-with-rels.d.ts +38 -0
  182. package/lib/db-ops/pages/read/get-pages-with-rels.js +107 -0
  183. package/lib/db-ops/pages/read/get-pages.d.ts +11 -0
  184. package/lib/db-ops/pages/read/get-pages.js +51 -0
  185. package/lib/db-ops/pages/read/get-scraped-html-page-count.d.ts +18 -0
  186. package/lib/db-ops/pages/read/get-scraped-html-page-count.js +25 -0
  187. package/lib/db-ops/pages/read/reconstruct-page-rows.d.ts +31 -0
  188. package/lib/db-ops/pages/read/reconstruct-page-rows.js +32 -0
  189. package/lib/db-ops/pages/reset/clear-page-derived-rows.d.ts +20 -0
  190. package/lib/db-ops/pages/reset/clear-page-derived-rows.js +37 -0
  191. package/lib/db-ops/pages/reset/repromote-external-pages.d.ts +30 -0
  192. package/lib/db-ops/pages/reset/repromote-external-pages.js +91 -0
  193. package/lib/db-ops/pages/reset/reset-failed-pages.d.ts +88 -0
  194. package/lib/db-ops/pages/reset/reset-failed-pages.js +216 -0
  195. package/lib/db-ops/pages/reset/reset-pages-by-urls.d.ts +80 -0
  196. package/lib/db-ops/pages/reset/reset-pages-by-urls.js +151 -0
  197. package/lib/db-ops/pages/reset/retryable-image-scan-codes.d.ts +24 -0
  198. package/lib/db-ops/pages/reset/retryable-image-scan-codes.js +24 -0
  199. package/lib/db-ops/pages/write/insert-audios.d.ts +14 -0
  200. package/lib/db-ops/pages/write/insert-audios.js +25 -0
  201. package/lib/db-ops/pages/write/insert-buttons.d.ts +14 -0
  202. package/lib/db-ops/pages/write/insert-buttons.js +29 -0
  203. package/lib/db-ops/pages/write/insert-canvases.d.ts +14 -0
  204. package/lib/db-ops/pages/write/insert-canvases.js +26 -0
  205. package/lib/db-ops/pages/write/insert-custom-elements.d.ts +27 -0
  206. package/lib/db-ops/pages/write/insert-custom-elements.js +40 -0
  207. package/lib/db-ops/pages/write/insert-headings.d.ts +15 -0
  208. package/lib/db-ops/pages/write/insert-headings.js +27 -0
  209. package/lib/db-ops/pages/write/insert-iframes.d.ts +14 -0
  210. package/lib/db-ops/pages/write/insert-iframes.js +28 -0
  211. package/lib/db-ops/pages/write/insert-inventory-content-items.d.ts +38 -0
  212. package/lib/db-ops/pages/write/insert-inventory-content-items.js +60 -0
  213. package/lib/db-ops/pages/write/insert-inventory-seeds.d.ts +36 -0
  214. package/lib/db-ops/pages/write/insert-inventory-seeds.js +48 -0
  215. package/lib/db-ops/pages/write/insert-inventory-skipped-pages.d.ts +42 -0
  216. package/lib/db-ops/pages/write/insert-inventory-skipped-pages.js +56 -0
  217. package/lib/db-ops/pages/write/insert-jsonld.d.ts +17 -0
  218. package/lib/db-ops/pages/write/insert-jsonld.js +49 -0
  219. package/lib/db-ops/pages/write/insert-main-content-images.d.ts +20 -0
  220. package/lib/db-ops/pages/write/insert-main-content-images.js +32 -0
  221. package/lib/db-ops/pages/write/insert-main-content-tables.d.ts +14 -0
  222. package/lib/db-ops/pages/write/insert-main-content-tables.js +29 -0
  223. package/lib/db-ops/pages/write/insert-page.d.ts +56 -0
  224. package/lib/db-ops/pages/write/insert-page.js +259 -0
  225. package/lib/db-ops/pages/write/insert-technologies.d.ts +25 -0
  226. package/lib/db-ops/pages/write/insert-technologies.js +45 -0
  227. package/lib/db-ops/pages/write/insert-videos.d.ts +14 -0
  228. package/lib/db-ops/pages/write/insert-videos.js +28 -0
  229. package/lib/db-ops/pages/write/link-redirect-sources.d.ts +42 -0
  230. package/lib/db-ops/pages/write/link-redirect-sources.js +101 -0
  231. package/lib/db-ops/pages/write/record-redirect.d.ts +35 -0
  232. package/lib/db-ops/pages/write/record-redirect.js +100 -0
  233. package/lib/db-ops/pages/write/set-skipped-page.d.ts +13 -0
  234. package/lib/db-ops/pages/write/set-skipped-page.js +24 -0
  235. package/lib/db-ops/pages/write/update-page.d.ts +40 -0
  236. package/lib/db-ops/pages/write/update-page.js +419 -0
  237. package/lib/db-ops/pages/write/write-page-html-blob.d.ts +19 -0
  238. package/lib/db-ops/pages/write/write-page-html-blob.js +41 -0
  239. package/lib/db-ops/referrers/get-redirects-for-pages.d.ts +9 -0
  240. package/lib/db-ops/referrers/get-redirects-for-pages.js +15 -0
  241. package/lib/db-ops/referrers/get-referrers-of-page.d.ts +17 -0
  242. package/lib/db-ops/referrers/get-referrers-of-page.js +32 -0
  243. package/lib/db-ops/referrers/get-referrers-of-resource.d.ts +8 -0
  244. package/lib/db-ops/referrers/get-referrers-of-resource.js +15 -0
  245. package/lib/db-ops/resources/build-resource-query.d.ts +25 -0
  246. package/lib/db-ops/resources/build-resource-query.js +29 -0
  247. package/lib/db-ops/resources/get-existing-resource-urls.d.ts +9 -0
  248. package/lib/db-ops/resources/get-existing-resource-urls.js +24 -0
  249. package/lib/db-ops/resources/get-resource-by-url.d.ts +13 -0
  250. package/lib/db-ops/resources/get-resource-by-url.js +22 -0
  251. package/lib/db-ops/resources/get-resource-url-list.d.ts +17 -0
  252. package/lib/db-ops/resources/get-resource-url-list.js +26 -0
  253. package/lib/db-ops/resources/get-resources.d.ts +8 -0
  254. package/lib/db-ops/resources/get-resources.js +11 -0
  255. package/lib/db-ops/resources/insert-inventory-resources.d.ts +24 -0
  256. package/lib/db-ops/resources/insert-inventory-resources.js +64 -0
  257. package/lib/db-ops/resources/insert-resource-referrers.d.ts +15 -0
  258. package/lib/db-ops/resources/insert-resource-referrers.js +54 -0
  259. package/lib/db-ops/resources/insert-resource.d.ts +34 -0
  260. package/lib/db-ops/resources/insert-resource.js +73 -0
  261. package/lib/db-ops/resources/reconstruct-resource-rows.d.ts +26 -0
  262. package/lib/db-ops/resources/reconstruct-resource-rows.js +30 -0
  263. package/lib/db-ops/templates/assign-template-labels.d.ts +60 -0
  264. package/lib/db-ops/templates/assign-template-labels.js +187 -0
  265. package/lib/db-ops/templates/replace-page-templates.d.ts +44 -0
  266. package/lib/db-ops/templates/replace-page-templates.js +174 -0
  267. package/lib/db-ops/templates/serialize-template-cluster-reason.d.ts +16 -0
  268. package/lib/db-ops/templates/serialize-template-cluster-reason.js +17 -0
  269. package/lib/db-ops/templates/types.d.ts +103 -0
  270. package/lib/db-ops/templates/types.js +1 -0
  271. package/lib/debug.d.ts +8 -0
  272. package/lib/debug.js +9 -0
  273. package/lib/decode-html-blob.d.ts +20 -0
  274. package/lib/decode-html-blob.js +23 -0
  275. package/lib/decode-page-html-blob-bytes.d.ts +25 -0
  276. package/lib/decode-page-html-blob-bytes.js +29 -0
  277. package/lib/decode-stored-blob-bytes.d.ts +25 -0
  278. package/lib/decode-stored-blob-bytes.js +47 -0
  279. package/lib/derive-lineage-from-parent.d.ts +37 -0
  280. package/lib/derive-lineage-from-parent.js +42 -0
  281. package/lib/drop-legacy-tables.d.ts +45 -0
  282. package/lib/drop-legacy-tables.js +56 -0
  283. package/lib/error-kind/classify-error-kind.d.ts +20 -0
  284. package/lib/error-kind/classify-error-kind.js +136 -0
  285. package/lib/error-kind/network-related-error-kinds.d.ts +23 -0
  286. package/lib/error-kind/network-related-error-kinds.js +28 -0
  287. package/lib/error-kind/permanent-error-kinds.d.ts +48 -0
  288. package/lib/error-kind/permanent-error-kinds.js +54 -0
  289. package/lib/error-kind/types.d.ts +54 -0
  290. package/lib/error-kind/types.js +1 -0
  291. package/lib/filesystem/append-text.d.ts +9 -0
  292. package/lib/filesystem/append-text.js +14 -0
  293. package/lib/filesystem/copy-dir-sync.d.ts +6 -0
  294. package/lib/filesystem/copy-dir-sync.js +9 -0
  295. package/lib/filesystem/copy-dir.d.ts +7 -0
  296. package/lib/filesystem/copy-dir.js +13 -0
  297. package/lib/filesystem/copy-file-with-progress.d.ts +20 -0
  298. package/lib/filesystem/copy-file-with-progress.js +34 -0
  299. package/lib/filesystem/exists.d.ts +6 -0
  300. package/lib/filesystem/exists.js +9 -0
  301. package/lib/filesystem/get-file-list.d.ts +8 -0
  302. package/lib/filesystem/get-file-list.js +12 -0
  303. package/lib/filesystem/is-dir.d.ts +6 -0
  304. package/lib/filesystem/is-dir.js +10 -0
  305. package/lib/filesystem/mkdir.d.ts +8 -0
  306. package/lib/filesystem/mkdir.js +15 -0
  307. package/lib/filesystem/output-binary.d.ts +12 -0
  308. package/lib/filesystem/output-binary.js +17 -0
  309. package/lib/filesystem/output-json.d.ts +9 -0
  310. package/lib/filesystem/output-json.js +14 -0
  311. package/lib/filesystem/output-text.d.ts +11 -0
  312. package/lib/filesystem/output-text.js +32 -0
  313. package/lib/filesystem/parse-pax-path.d.ts +18 -0
  314. package/lib/filesystem/parse-pax-path.js +45 -0
  315. package/lib/filesystem/parse-tar-size-field.d.ts +13 -0
  316. package/lib/filesystem/parse-tar-size-field.js +37 -0
  317. package/lib/filesystem/peek-tar-top-dir.d.ts +35 -0
  318. package/lib/filesystem/peek-tar-top-dir.js +215 -0
  319. package/lib/filesystem/read-json.d.ts +7 -0
  320. package/lib/filesystem/read-json.js +11 -0
  321. package/lib/filesystem/read-text.d.ts +6 -0
  322. package/lib/filesystem/read-text.js +10 -0
  323. package/lib/filesystem/readline.d.ts +11 -0
  324. package/lib/filesystem/readline.js +26 -0
  325. package/lib/filesystem/remove.d.ts +5 -0
  326. package/lib/filesystem/remove.js +10 -0
  327. package/lib/filesystem/rename.d.ts +16 -0
  328. package/lib/filesystem/rename.js +48 -0
  329. package/lib/filesystem/tar.d.ts +23 -0
  330. package/lib/filesystem/tar.js +78 -0
  331. package/lib/filesystem/untar.d.ts +33 -0
  332. package/lib/filesystem/untar.js +84 -0
  333. package/lib/get-failed-page-messages.d.ts +56 -0
  334. package/lib/get-failed-page-messages.js +146 -0
  335. package/lib/get-json.d.ts +10 -0
  336. package/lib/get-json.js +25 -0
  337. package/lib/html-snapshot/match-selector-html.d.ts +25 -0
  338. package/lib/html-snapshot/match-selector-html.js +28 -0
  339. package/lib/html-snapshot/match-selector-snapshots.d.ts +33 -0
  340. package/lib/html-snapshot/match-selector-snapshots.js +33 -0
  341. package/lib/html-snapshot/search-html-document.d.ts +29 -0
  342. package/lib/html-snapshot/search-html-document.js +27 -0
  343. package/lib/html-snapshot/search-html-snapshots.d.ts +36 -0
  344. package/lib/html-snapshot/search-html-snapshots.js +35 -0
  345. package/lib/html-snapshot/types.d.ts +111 -0
  346. package/lib/html-snapshot/types.js +7 -0
  347. package/lib/html-snapshot/validate-search-pattern.d.ts +15 -0
  348. package/lib/html-snapshot/validate-search-pattern.js +17 -0
  349. package/lib/init-schema.d.ts +48 -0
  350. package/lib/init-schema.js +124 -0
  351. package/lib/is-inventory-source.d.ts +21 -0
  352. package/lib/is-inventory-source.js +22 -0
  353. package/lib/is-within-outage-window.d.ts +49 -0
  354. package/lib/is-within-outage-window.js +33 -0
  355. package/lib/isolated-cluster/create-isolated-cluster-run.d.ts +24 -0
  356. package/lib/isolated-cluster/create-isolated-cluster-run.js +26 -0
  357. package/lib/libsql-dialect.d.ts +115 -0
  358. package/lib/libsql-dialect.js +116 -0
  359. package/lib/limited-page-ids.d.ts +9 -0
  360. package/lib/limited-page-ids.js +18 -0
  361. package/lib/meta/assert-compatible-version.d.ts +60 -0
  362. package/lib/meta/assert-compatible-version.js +93 -0
  363. package/lib/meta/classify-jsonld-type.d.ts +23 -0
  364. package/lib/meta/classify-jsonld-type.js +43 -0
  365. package/lib/meta/compare-semver.d.ts +23 -0
  366. package/lib/meta/compare-semver.js +51 -0
  367. package/lib/meta/compute-main-contents-denormalized.d.ts +41 -0
  368. package/lib/meta/compute-main-contents-denormalized.js +88 -0
  369. package/lib/meta/compute-page-denormalized.d.ts +20 -0
  370. package/lib/meta/compute-page-denormalized.js +35 -0
  371. package/lib/meta/derive-flat-from-meta.d.ts +34 -0
  372. package/lib/meta/derive-flat-from-meta.js +158 -0
  373. package/lib/meta/derive-meta-extras.d.ts +20 -0
  374. package/lib/meta/derive-meta-extras.js +23 -0
  375. package/lib/meta/extract-tags-for-archive.d.ts +17 -0
  376. package/lib/meta/extract-tags-for-archive.js +36 -0
  377. package/lib/meta/parse-main-content-class-list.d.ts +19 -0
  378. package/lib/meta/parse-main-content-class-list.js +34 -0
  379. package/lib/meta/summarize-jsonld.d.ts +17 -0
  380. package/lib/meta/summarize-jsonld.js +29 -0
  381. package/lib/meta/technologies/combine-technology-confidence.d.ts +46 -0
  382. package/lib/meta/technologies/combine-technology-confidence.js +93 -0
  383. package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.d.ts +45 -0
  384. package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.js +53 -0
  385. package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.d.ts +33 -0
  386. package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.js +46 -0
  387. package/lib/meta/technologies/extract-technologies-for-archive.d.ts +33 -0
  388. package/lib/meta/technologies/extract-technologies-for-archive.js +30 -0
  389. package/lib/meta/technologies/match-js-license-comment-signal.d.ts +21 -0
  390. package/lib/meta/technologies/match-js-license-comment-signal.js +23 -0
  391. package/lib/meta/technologies/match-structural-technology-signals.d.ts +26 -0
  392. package/lib/meta/technologies/match-structural-technology-signals.js +48 -0
  393. package/lib/meta/technologies/normalize-wappalyzer-entries.d.ts +25 -0
  394. package/lib/meta/technologies/normalize-wappalyzer-entries.js +35 -0
  395. package/lib/meta/technologies/technology-signal-definitions.d.ts +44 -0
  396. package/lib/meta/technologies/technology-signal-definitions.js +60 -0
  397. package/lib/meta/technologies/types.d.ts +70 -0
  398. package/lib/meta/technologies/types.js +13 -0
  399. package/lib/meta/types.d.ts +1490 -0
  400. package/lib/meta/types.js +65 -0
  401. package/lib/migrate-content-items-alias-of-id.d.ts +52 -0
  402. package/lib/migrate-content-items-alias-of-id.js +69 -0
  403. package/lib/migrate-content-items-dedupe-cap-event-id.d.ts +46 -0
  404. package/lib/migrate-content-items-dedupe-cap-event-id.js +62 -0
  405. package/lib/migrate-content-items-is-metadata-only.d.ts +44 -0
  406. package/lib/migrate-content-items-is-metadata-only.js +60 -0
  407. package/lib/migrate-entity-tables.d.ts +55 -0
  408. package/lib/migrate-entity-tables.js +66 -0
  409. package/lib/migrate-info-created-cwd.d.ts +17 -0
  410. package/lib/migrate-info-created-cwd.js +36 -0
  411. package/lib/migrate-info-main-content-selector.d.ts +16 -0
  412. package/lib/migrate-info-main-content-selector.js +35 -0
  413. package/lib/migrate-info-request-header-names.d.ts +16 -0
  414. package/lib/migrate-info-request-header-names.js +35 -0
  415. package/lib/migrate-info-roots.d.ts +20 -0
  416. package/lib/migrate-info-roots.js +52 -0
  417. package/lib/migrate-inventory-runs-to-list-reconcile-runs.d.ts +34 -0
  418. package/lib/migrate-inventory-runs-to-list-reconcile-runs.js +51 -0
  419. package/lib/migrate-list-reconcile-runs-exclude-skipped.d.ts +25 -0
  420. package/lib/migrate-list-reconcile-runs-exclude-skipped.js +44 -0
  421. package/lib/migrate-list-reconcile-runs-invalid-skipped.d.ts +16 -0
  422. package/lib/migrate-list-reconcile-runs-invalid-skipped.js +35 -0
  423. package/lib/migrate-main-contents-columns.d.ts +26 -0
  424. package/lib/migrate-main-contents-columns.js +61 -0
  425. package/lib/migrate-page-meta-body-hash.d.ts +44 -0
  426. package/lib/migrate-page-meta-body-hash.js +63 -0
  427. package/lib/migrate-page-meta-console-error-count.d.ts +27 -0
  428. package/lib/migrate-page-meta-console-error-count.js +45 -0
  429. package/lib/migrate-page-meta-custom-element-count.d.ts +33 -0
  430. package/lib/migrate-page-meta-custom-element-count.js +51 -0
  431. package/lib/migrate-page-meta-image-scan.d.ts +27 -0
  432. package/lib/migrate-page-meta-image-scan.js +52 -0
  433. package/lib/migrate-page-tags-to-page-technologies.d.ts +39 -0
  434. package/lib/migrate-page-tags-to-page-technologies.js +79 -0
  435. package/lib/migrate-ref-tables.d.ts +25 -0
  436. package/lib/migrate-ref-tables.js +38 -0
  437. package/lib/native-executor/check-native-executor-config.d.ts +17 -0
  438. package/lib/native-executor/check-native-executor-config.js +20 -0
  439. package/lib/page-meta-column-maps.d.ts +32 -0
  440. package/lib/page-meta-column-maps.js +43 -0
  441. package/lib/page.d.ts +546 -0
  442. package/lib/page.js +764 -0
  443. package/lib/peek-archive-lock.d.ts +40 -0
  444. package/lib/peek-archive-lock.js +62 -0
  445. package/lib/populate-entity-tables/collapse-anchor-rows.d.ts +41 -0
  446. package/lib/populate-entity-tables/collapse-anchor-rows.js +87 -0
  447. package/lib/populate-entity-tables/derive-dom-path.d.ts +35 -0
  448. package/lib/populate-entity-tables/derive-dom-path.js +72 -0
  449. package/lib/populate-entity-tables/is-blob-ref-value.d.ts +16 -0
  450. package/lib/populate-entity-tables/is-blob-ref-value.js +19 -0
  451. package/lib/populate-entity-tables/match-images-to-dom-paths.d.ts +66 -0
  452. package/lib/populate-entity-tables/match-images-to-dom-paths.js +96 -0
  453. package/lib/populate-entity-tables/populate-anchor-edges.d.ts +33 -0
  454. package/lib/populate-entity-tables/populate-anchor-edges.js +153 -0
  455. package/lib/populate-entity-tables/populate-content-items.d.ts +40 -0
  456. package/lib/populate-entity-tables/populate-content-items.js +141 -0
  457. package/lib/populate-entity-tables/populate-entities.d.ts +81 -0
  458. package/lib/populate-entity-tables/populate-entities.js +110 -0
  459. package/lib/populate-entity-tables/populate-image-items.d.ts +91 -0
  460. package/lib/populate-entity-tables/populate-image-items.js +226 -0
  461. package/lib/populate-entity-tables/populate-page-meta.d.ts +33 -0
  462. package/lib/populate-entity-tables/populate-page-meta.js +267 -0
  463. package/lib/populate-entity-tables/populate-resource-items.d.ts +22 -0
  464. package/lib/populate-entity-tables/populate-resource-items.js +114 -0
  465. package/lib/populate-entity-tables/populate-resource-ref-edges.d.ts +31 -0
  466. package/lib/populate-entity-tables/populate-resource-ref-edges.js +33 -0
  467. package/lib/populate-entity-tables/resolve-blob-refs.d.ts +31 -0
  468. package/lib/populate-entity-tables/resolve-blob-refs.js +100 -0
  469. package/lib/populate-entity-tables/resolve-content-type-refs.d.ts +22 -0
  470. package/lib/populate-entity-tables/resolve-content-type-refs.js +27 -0
  471. package/lib/populate-entity-tables/resolve-header-sets.d.ts +49 -0
  472. package/lib/populate-entity-tables/resolve-header-sets.js +122 -0
  473. package/lib/populate-entity-tables/resolve-json-refs.d.ts +25 -0
  474. package/lib/populate-entity-tables/resolve-json-refs.js +67 -0
  475. package/lib/populate-entity-tables/resolve-text-refs.d.ts +30 -0
  476. package/lib/populate-entity-tables/resolve-text-refs.js +61 -0
  477. package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.d.ts +21 -0
  478. package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.js +27 -0
  479. package/lib/populate-entity-tables/resolve-url-refs.d.ts +33 -0
  480. package/lib/populate-entity-tables/resolve-url-refs.js +60 -0
  481. package/lib/populate-entity-tables/test-utils/count-rows.d.ts +17 -0
  482. package/lib/populate-entity-tables/test-utils/count-rows.js +20 -0
  483. package/lib/populate-entity-tables/test-utils/seed-content-items.d.ts +25 -0
  484. package/lib/populate-entity-tables/test-utils/seed-content-items.js +42 -0
  485. package/lib/populate-entity-tables/test-utils/setup-entities-db.d.ts +30 -0
  486. package/lib/populate-entity-tables/test-utils/setup-entities-db.js +175 -0
  487. package/lib/populate-entity-tables/types.d.ts +157 -0
  488. package/lib/populate-entity-tables/types.js +12 -0
  489. package/lib/populate-entity-tables/upsert-text-refs.d.ts +38 -0
  490. package/lib/populate-entity-tables/upsert-text-refs.js +78 -0
  491. package/lib/populate-ref-tables/classify-content-type.d.ts +16 -0
  492. package/lib/populate-ref-tables/classify-content-type.js +52 -0
  493. package/lib/populate-ref-tables/compute-content-hash.d.ts +22 -0
  494. package/lib/populate-ref-tables/compute-content-hash.js +26 -0
  495. package/lib/populate-ref-tables/compute-header-flags.d.ts +16 -0
  496. package/lib/populate-ref-tables/compute-header-flags.js +70 -0
  497. package/lib/populate-ref-tables/content-type-rules.d.ts +38 -0
  498. package/lib/populate-ref-tables/content-type-rules.js +133 -0
  499. package/lib/populate-ref-tables/create-header-table-caches.d.ts +25 -0
  500. package/lib/populate-ref-tables/create-header-table-caches.js +49 -0
  501. package/lib/populate-ref-tables/data-uri-url-refs-limit.d.ts +15 -0
  502. package/lib/populate-ref-tables/data-uri-url-refs-limit.js +15 -0
  503. package/lib/populate-ref-tables/decode-data-uri.d.ts +21 -0
  504. package/lib/populate-ref-tables/decode-data-uri.js +126 -0
  505. package/lib/populate-ref-tables/decompose-header-set.d.ts +29 -0
  506. package/lib/populate-ref-tables/decompose-header-set.js +157 -0
  507. package/lib/populate-ref-tables/decompose-url.d.ts +25 -0
  508. package/lib/populate-ref-tables/decompose-url.js +70 -0
  509. package/lib/populate-ref-tables/header-stability.d.ts +19 -0
  510. package/lib/populate-ref-tables/header-stability.js +22 -0
  511. package/lib/populate-ref-tables/header-value-cache-key.d.ts +17 -0
  512. package/lib/populate-ref-tables/header-value-cache-key.js +19 -0
  513. package/lib/populate-ref-tables/normalize-mime.d.ts +24 -0
  514. package/lib/populate-ref-tables/normalize-mime.js +36 -0
  515. package/lib/populate-ref-tables/populate-blob-refs.d.ts +38 -0
  516. package/lib/populate-ref-tables/populate-blob-refs.js +134 -0
  517. package/lib/populate-ref-tables/populate-content-type-refs.d.ts +27 -0
  518. package/lib/populate-ref-tables/populate-content-type-refs.js +70 -0
  519. package/lib/populate-ref-tables/populate-header-tables.d.ts +35 -0
  520. package/lib/populate-ref-tables/populate-header-tables.js +80 -0
  521. package/lib/populate-ref-tables/populate-json-refs.d.ts +29 -0
  522. package/lib/populate-ref-tables/populate-json-refs.js +101 -0
  523. package/lib/populate-ref-tables/populate-refs.d.ts +51 -0
  524. package/lib/populate-ref-tables/populate-refs.js +62 -0
  525. package/lib/populate-ref-tables/populate-text-refs.d.ts +32 -0
  526. package/lib/populate-ref-tables/populate-text-refs.js +133 -0
  527. package/lib/populate-ref-tables/populate-url-refs.d.ts +28 -0
  528. package/lib/populate-ref-tables/populate-url-refs.js +148 -0
  529. package/lib/populate-ref-tables/test-utils/count-rows.d.ts +15 -0
  530. package/lib/populate-ref-tables/test-utils/count-rows.js +17 -0
  531. package/lib/populate-ref-tables/types.d.ts +197 -0
  532. package/lib/populate-ref-tables/types.js +7 -0
  533. package/lib/populate-ref-tables/upsert-one-header-set.d.ts +34 -0
  534. package/lib/populate-ref-tables/upsert-one-header-set.js +208 -0
  535. package/lib/populate-ref-tables/volatile-header-names.d.ts +20 -0
  536. package/lib/populate-ref-tables/volatile-header-names.js +33 -0
  537. package/lib/redirect-table.d.ts +9 -0
  538. package/lib/redirect-table.js +28 -0
  539. package/lib/release-finished-statements.d.ts +42 -0
  540. package/lib/release-finished-statements.js +45 -0
  541. package/lib/resolve-redirect-chain.d.ts +33 -0
  542. package/lib/resolve-redirect-chain.js +27 -0
  543. package/lib/resource.d.ts +46 -0
  544. package/lib/resource.js +62 -0
  545. package/lib/retarget-legacy-fk-tables.d.ts +61 -0
  546. package/lib/retarget-legacy-fk-tables.js +137 -0
  547. package/lib/safe-path.d.ts +9 -0
  548. package/lib/safe-path.js +17 -0
  549. package/lib/scope/build-scope-map.d.ts +21 -0
  550. package/lib/scope/build-scope-map.js +32 -0
  551. package/lib/scope/find-scope-entry.d.ts +25 -0
  552. package/lib/scope/find-scope-entry.js +45 -0
  553. package/lib/template-classification/classify-archive-page-templates.d.ts +42 -0
  554. package/lib/template-classification/classify-archive-page-templates.js +70 -0
  555. package/lib/template-classification/classify-page-templates.d.ts +33 -0
  556. package/lib/template-classification/classify-page-templates.js +60 -0
  557. package/lib/template-classification/collect-page-ids-with-html.d.ts +24 -0
  558. package/lib/template-classification/collect-page-ids-with-html.js +30 -0
  559. package/lib/template-classification/collect-page-stylesheet-urls.d.ts +32 -0
  560. package/lib/template-classification/collect-page-stylesheet-urls.js +132 -0
  561. package/lib/template-classification/create-content-root-hint.d.ts +41 -0
  562. package/lib/template-classification/create-content-root-hint.js +51 -0
  563. package/lib/template-classification/create-page-cluster-factory.d.ts +36 -0
  564. package/lib/template-classification/create-page-cluster-factory.js +52 -0
  565. package/lib/template-classification/load-page-cluster-candidates.d.ts +32 -0
  566. package/lib/template-classification/load-page-cluster-candidates.js +93 -0
  567. package/lib/template-classification/page-cluster/resolve-page-cluster-keys.d.ts +34 -0
  568. package/lib/template-classification/page-cluster/resolve-page-cluster-keys.js +236 -0
  569. package/lib/template-classification/page-cluster/types.d.ts +123 -0
  570. package/lib/template-classification/page-cluster/types.js +1 -0
  571. package/lib/template-classification/types.d.ts +234 -0
  572. package/lib/template-classification/types.js +1 -0
  573. package/lib/test-utils/build-full-schema-test-db.d.ts +34 -0
  574. package/lib/test-utils/build-full-schema-test-db.js +74 -0
  575. package/lib/test-utils/compare-strings.d.ts +12 -0
  576. package/lib/test-utils/compare-strings.js +14 -0
  577. package/lib/test-utils/create-legacy-analysis-tables.d.ts +13 -0
  578. package/lib/test-utils/create-legacy-analysis-tables.js +37 -0
  579. package/lib/test-utils/fk-parent-tables.d.ts +15 -0
  580. package/lib/test-utils/fk-parent-tables.js +19 -0
  581. package/lib/test-utils/seed-content-item.d.ts +35 -0
  582. package/lib/test-utils/seed-content-item.js +42 -0
  583. package/lib/test-utils/setup-legacy-fk-db.d.ts +34 -0
  584. package/lib/test-utils/setup-legacy-fk-db.js +271 -0
  585. package/lib/transfer/attach-source-database.d.ts +31 -0
  586. package/lib/transfer/attach-source-database.js +40 -0
  587. package/lib/transfer/build-split-config.d.ts +30 -0
  588. package/lib/transfer/build-split-config.js +42 -0
  589. package/lib/transfer/clear-replaced-page-rows.d.ts +23 -0
  590. package/lib/transfer/clear-replaced-page-rows.js +61 -0
  591. package/lib/transfer/concat-archives.d.ts +55 -0
  592. package/lib/transfer/concat-archives.js +95 -0
  593. package/lib/transfer/concat-source-transfer-phases.d.ts +10 -0
  594. package/lib/transfer/concat-source-transfer-phases.js +17 -0
  595. package/lib/transfer/content-item-rank-sql.d.ts +21 -0
  596. package/lib/transfer/content-item-rank-sql.js +23 -0
  597. package/lib/transfer/copy-anchor-edges.d.ts +28 -0
  598. package/lib/transfer/copy-anchor-edges.js +42 -0
  599. package/lib/transfer/copy-dictionaries-for-concat.d.ts +39 -0
  600. package/lib/transfer/copy-dictionaries-for-concat.js +81 -0
  601. package/lib/transfer/copy-dictionaries-for-split.d.ts +29 -0
  602. package/lib/transfer/copy-dictionaries-for-split.js +224 -0
  603. package/lib/transfer/copy-header-set-children.d.ts +27 -0
  604. package/lib/transfer/copy-header-set-children.js +51 -0
  605. package/lib/transfer/copy-image-items.d.ts +21 -0
  606. package/lib/transfer/copy-image-items.js +62 -0
  607. package/lib/transfer/copy-inventory-source-lists.d.ts +30 -0
  608. package/lib/transfer/copy-inventory-source-lists.js +61 -0
  609. package/lib/transfer/copy-journal-tables.d.ts +30 -0
  610. package/lib/transfer/copy-journal-tables.js +56 -0
  611. package/lib/transfer/copy-page-console-logs.d.ts +16 -0
  612. package/lib/transfer/copy-page-console-logs.js +27 -0
  613. package/lib/transfer/copy-page-html-blobs.d.ts +31 -0
  614. package/lib/transfer/copy-page-html-blobs.js +43 -0
  615. package/lib/transfer/copy-page-meta.d.ts +27 -0
  616. package/lib/transfer/copy-page-meta.js +68 -0
  617. package/lib/transfer/copy-page-template-clusters.d.ts +31 -0
  618. package/lib/transfer/copy-page-template-clusters.js +56 -0
  619. package/lib/transfer/copy-page-template-labels.d.ts +34 -0
  620. package/lib/transfer/copy-page-template-labels.js +88 -0
  621. package/lib/transfer/copy-resource-items.d.ts +29 -0
  622. package/lib/transfer/copy-resource-items.js +89 -0
  623. package/lib/transfer/copy-resource-ref-edges.d.ts +20 -0
  624. package/lib/transfer/copy-resource-ref-edges.js +33 -0
  625. package/lib/transfer/copy-simple-page-scoped-tables.d.ts +15 -0
  626. package/lib/transfer/copy-simple-page-scoped-tables.js +34 -0
  627. package/lib/transfer/count-external-pages-in-scope.d.ts +26 -0
  628. package/lib/transfer/count-external-pages-in-scope.js +40 -0
  629. package/lib/transfer/create-transfer-temp-tables.d.ts +30 -0
  630. package/lib/transfer/create-transfer-temp-tables.js +59 -0
  631. package/lib/transfer/dictionary-copy-specs.d.ts +32 -0
  632. package/lib/transfer/dictionary-copy-specs.js +28 -0
  633. package/lib/transfer/drop-transfer-temp-tables.d.ts +9 -0
  634. package/lib/transfer/drop-transfer-temp-tables.js +18 -0
  635. package/lib/transfer/flatten-redirect-chains.d.ts +44 -0
  636. package/lib/transfer/flatten-redirect-chains.js +74 -0
  637. package/lib/transfer/insert-content-items.d.ts +35 -0
  638. package/lib/transfer/insert-content-items.js +116 -0
  639. package/lib/transfer/list-transfer-columns.d.ts +35 -0
  640. package/lib/transfer/list-transfer-columns.js +45 -0
  641. package/lib/transfer/mark-in-scope-content-items.d.ts +31 -0
  642. package/lib/transfer/mark-in-scope-content-items.js +54 -0
  643. package/lib/transfer/merge-archive-configs.d.ts +51 -0
  644. package/lib/transfer/merge-archive-configs.js +85 -0
  645. package/lib/transfer/plan-content-items-for-concat.d.ts +33 -0
  646. package/lib/transfer/plan-content-items-for-concat.js +51 -0
  647. package/lib/transfer/plan-content-items-for-split.d.ts +42 -0
  648. package/lib/transfer/plan-content-items-for-split.js +83 -0
  649. package/lib/transfer/plan-resource-items.d.ts +40 -0
  650. package/lib/transfer/plan-resource-items.js +69 -0
  651. package/lib/transfer/quote-transfer-identifier.d.ts +18 -0
  652. package/lib/transfer/quote-transfer-identifier.js +20 -0
  653. package/lib/transfer/reclassify-resource-externality.d.ts +37 -0
  654. package/lib/transfer/reclassify-resource-externality.js +66 -0
  655. package/lib/transfer/remap-redirect-dest-ids.d.ts +34 -0
  656. package/lib/transfer/remap-redirect-dest-ids.js +51 -0
  657. package/lib/transfer/replace-content-items.d.ts +39 -0
  658. package/lib/transfer/replace-content-items.js +83 -0
  659. package/lib/transfer/simple-page-scoped-tables.d.ts +25 -0
  660. package/lib/transfer/simple-page-scoped-tables.js +38 -0
  661. package/lib/transfer/split-archive.d.ts +55 -0
  662. package/lib/transfer/split-archive.js +98 -0
  663. package/lib/transfer/split-source-transfer-phases.d.ts +11 -0
  664. package/lib/transfer/split-source-transfer-phases.js +18 -0
  665. package/lib/transfer/transfer-action.d.ts +34 -0
  666. package/lib/transfer/transfer-action.js +32 -0
  667. package/lib/transfer/transfer-archive-rows.d.ts +65 -0
  668. package/lib/transfer/transfer-archive-rows.js +156 -0
  669. package/lib/transfer/transfer-dictionary-tables.d.ts +12 -0
  670. package/lib/transfer/transfer-dictionary-tables.js +22 -0
  671. package/lib/transfer/transfer-post-phases.d.ts +10 -0
  672. package/lib/transfer/transfer-post-phases.js +13 -0
  673. package/lib/transfer/transfer-source-alias.d.ts +9 -0
  674. package/lib/transfer/transfer-source-alias.js +9 -0
  675. package/lib/transfer/types.d.ts +111 -0
  676. package/lib/transfer/types.js +35 -0
  677. package/lib/types.d.ts +772 -0
  678. package/lib/types.js +1 -0
  679. package/lib/url-alias/compute-tier-a-alias-key.d.ts +44 -0
  680. package/lib/url-alias/compute-tier-a-alias-key.js +47 -0
  681. package/lib/url-alias/compute-tier-b-alias-key.d.ts +23 -0
  682. package/lib/url-alias/compute-tier-b-alias-key.js +26 -0
  683. package/lib/url-alias/group-alias-candidates.d.ts +27 -0
  684. package/lib/url-alias/group-alias-candidates.js +29 -0
  685. package/lib/url-alias/types.d.ts +12 -0
  686. package/lib/url-alias/types.js +1 -0
  687. package/lib/url-pattern/compute-shape-key.d.ts +58 -0
  688. package/lib/url-pattern/compute-shape-key.js +129 -0
  689. package/lib/url-pattern/decompose-url.d.ts +33 -0
  690. package/lib/url-pattern/decompose-url.js +79 -0
  691. package/lib/url-pattern/reconstruct-url.d.ts +15 -0
  692. package/lib/url-pattern/reconstruct-url.js +33 -0
  693. package/lib/url-sort/compute-url-sort-ranks.d.ts +30 -0
  694. package/lib/url-sort/compute-url-sort-ranks.js +33 -0
  695. package/lib/utils/array/each-splitted.d.ts +10 -0
  696. package/lib/utils/array/each-splitted.js +14 -0
  697. package/lib/utils/debug.d.ts +8 -0
  698. package/lib/utils/debug.js +8 -0
  699. package/lib/utils/error/emit-error-with-retry.d.ts +40 -0
  700. package/lib/utils/error/emit-error-with-retry.js +44 -0
  701. package/lib/utils/error/emit-error.d.ts +39 -0
  702. package/lib/utils/error/emit-error.js +41 -0
  703. package/lib/utils/object/parse-response-headers.d.ts +12 -0
  704. package/lib/utils/object/parse-response-headers.js +26 -0
  705. package/lib/utils/object/redact-request-headers.d.ts +15 -0
  706. package/lib/utils/object/redact-request-headers.js +27 -0
  707. package/lib/utils/types/types.d.ts +383 -0
  708. package/lib/utils/types/types.js +1 -0
  709. package/lib/verify-migration/capture-rejection.d.ts +24 -0
  710. package/lib/verify-migration/capture-rejection.js +31 -0
  711. package/lib/verify-migration/check-anchor-edges-count.d.ts +34 -0
  712. package/lib/verify-migration/check-anchor-edges-count.js +72 -0
  713. package/lib/verify-migration/check-anchor-edges-sum.d.ts +13 -0
  714. package/lib/verify-migration/check-anchor-edges-sum.js +27 -0
  715. package/lib/verify-migration/check-content-items-count.d.ts +16 -0
  716. package/lib/verify-migration/check-content-items-count.js +30 -0
  717. package/lib/verify-migration/check-content-type-preservation.d.ts +22 -0
  718. package/lib/verify-migration/check-content-type-preservation.js +40 -0
  719. package/lib/verify-migration/check-foreign-key-integrity.d.ts +31 -0
  720. package/lib/verify-migration/check-foreign-key-integrity.js +47 -0
  721. package/lib/verify-migration/check-image-items-count.d.ts +12 -0
  722. package/lib/verify-migration/check-image-items-count.js +26 -0
  723. package/lib/verify-migration/check-page-meta-count.d.ts +15 -0
  724. package/lib/verify-migration/check-page-meta-count.js +31 -0
  725. package/lib/verify-migration/check-reader-parity.d.ts +23 -0
  726. package/lib/verify-migration/check-reader-parity.js +182 -0
  727. package/lib/verify-migration/check-resource-items-count.d.ts +17 -0
  728. package/lib/verify-migration/check-resource-items-count.js +33 -0
  729. package/lib/verify-migration/check-url-round-trip.d.ts +43 -0
  730. package/lib/verify-migration/check-url-round-trip.js +112 -0
  731. package/lib/verify-migration/types.d.ts +70 -0
  732. package/lib/verify-migration/types.js +63 -0
  733. package/lib/verify-migration/verify-migration.d.ts +41 -0
  734. package/lib/verify-migration/verify-migration.js +120 -0
  735. package/lib/with-bulk-write-pragmas.d.ts +51 -0
  736. package/lib/with-bulk-write-pragmas.js +132 -0
  737. package/lib/write-queue.d.ts +32 -0
  738. package/lib/write-queue.js +55 -0
  739. package/package.json +350 -3
@@ -0,0 +1,477 @@
1
+ import { raceWithTimeout } from '@d-zero/shared/race-with-timeout';
2
+ import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
3
+ import { log } from './debug.js';
4
+ import { readJSON } from './filesystem/read-json.js';
5
+ import { readText } from './filesystem/read-text.js';
6
+ import Page from './page.js';
7
+ import Resource from './resource.js';
8
+ import { safePath } from './safe-path.js';
9
+ /**
10
+ * Provides read-only access to an archive's database and stored data files.
11
+ *
12
+ * This class is the base for the `Archive` class and is also returned
13
+ * by `Archive.connect` for read-only access to an existing archive.
14
+ * It supports querying pages, anchors, referrers, resources, and custom data.
15
+ */
16
+ /**
17
+ * Default timeout for {@link ArchiveAccessor.close}'s `db.destroy()` step.
18
+ *
19
+ * `knex.destroy()` will otherwise wait the full `acquireTimeoutMillis`
20
+ * (10 minutes in this repo) for in-flight queries to drain. For a viewer
21
+ * shut down by Ctrl-C while the live crawler holds a long write lock that
22
+ * is an unacceptable user experience, so we bound the wait and treat the
23
+ * accessor as closed after the timeout regardless.
24
+ */
25
+ const DEFAULT_CLOSE_TIMEOUT_MS = 5000;
26
+ export class ArchiveAccessor extends EventEmitter {
27
+ /**
28
+ * Promise tracking an in-progress (or completed) close. `null` means the
29
+ * accessor is open and idle; a settled promise means we are closed (the
30
+ * accessor stays "closed" even if `db.destroy()` rejected, because there
31
+ * is nothing safe to retry — see {@link ArchiveAccessor.close}).
32
+ */
33
+ #closeOnce = null;
34
+ /** The SQLite database instance for querying archived data. */
35
+ #db;
36
+ /**
37
+ * Whether this accessor was opened in read-only mode. With HTML stored
38
+ * as a SQLite BLOB, this no longer toggles any code path — the SELECT
39
+ * is identical for writer- and reader-mode accessors. Kept on the
40
+ * accessor so callers like the viewer can still surface "this archive
41
+ * is being read read-only" UI hints without re-deriving it.
42
+ */
43
+ #readOnly;
44
+ /** Absolute path to the temporary working directory containing the database and files. */
45
+ #tmpDir;
46
+ /**
47
+ * Whether this accessor was opened in read-only mode (no filesystem
48
+ * mutation on `tmpDir`).
49
+ */
50
+ get readOnly() {
51
+ return this.#readOnly;
52
+ }
53
+ /**
54
+ * The absolute path to the temporary working directory used by this accessor.
55
+ */
56
+ get tmpDir() {
57
+ return this.#tmpDir;
58
+ }
59
+ /**
60
+ * Creates a new ArchiveAccessor instance.
61
+ * @param tmpDir - The path to the temporary directory containing the archive data.
62
+ * @param db - The Database instance for querying the SQLite database.
63
+ * @param options - Construction options.
64
+ * @param options.readOnly - When `true`, helpers must not mutate the
65
+ * filesystem under `tmpDir` (used for live-crawl / stub-mode opens
66
+ * where any write would race the crawler).
67
+ */
68
+ constructor(tmpDir, db, options = {}) {
69
+ super();
70
+ this.#tmpDir = tmpDir;
71
+ this.#db = db;
72
+ this.#readOnly = options.readOnly ?? false;
73
+ this.#db.on('error', (e) => {
74
+ void this.emit('error', e);
75
+ });
76
+ }
77
+ /**
78
+ * Enables `await using accessor = ...`. Delegates to {@link close} with
79
+ * the default timeout — callers that need a non-default `timeoutMs`
80
+ * must call `close` explicitly instead of relying on disposal.
81
+ *
82
+ * Dispatches through the instance's own `close`, so an `Archive`
83
+ * (which overrides `close`) gets its full teardown here too — this
84
+ * method does not need to be re-implemented on subclasses.
85
+ */
86
+ async [Symbol.asyncDispose]() {
87
+ await this.close();
88
+ }
89
+ /**
90
+ * Closes the underlying database connection.
91
+ *
92
+ * This is the **read-only** close path: it releases the SQLite handle and
93
+ * does nothing else. The temporary working directory is left untouched and
94
+ * no `.nitpicker` archive is produced. This makes it safe to call from
95
+ * read-only consumers (e.g. the viewer attached to an in-progress crawl's
96
+ * tmpDir), where touching the filesystem would race with — or destroy —
97
+ * the live crawler's working state.
98
+ *
99
+ * Subclasses that own the archive's lifecycle (notably `Archive`)
100
+ * override this to add write/cleanup steps.
101
+ *
102
+ * **Idempotent and concurrent-safe**: the first invocation captures the
103
+ * close promise; later invocations (from the same caller, a shutdown
104
+ * signal handler, or a parallel manager teardown) await the same
105
+ * promise and resolve together. If `db.destroy()` rejects, the
106
+ * rejection propagates to *all* awaiters and the accessor stays
107
+ * latched closed — a hung knex pool is not safe to "retry close".
108
+ *
109
+ * **Bounded**: when the optional `timeoutMs` (default {@link
110
+ * DEFAULT_CLOSE_TIMEOUT_MS}) elapses before `db.destroy()` settles, the
111
+ * call resolves with a warning. This prevents a viewer shutdown from
112
+ * being held for the underlying pool's 10-minute `acquireTimeoutMillis`
113
+ * when the live crawler holds the SQLite write lock.
114
+ * @param options - Close options.
115
+ * @param options.timeoutMs - Milliseconds to wait for `db.destroy()`
116
+ * before giving up. Use `Infinity` to wait indefinitely (only
117
+ * advisable in tests and batch jobs that own the DB exclusively).
118
+ */
119
+ async close(options = {}) {
120
+ if (this.#closeOnce) {
121
+ return this.#closeOnce;
122
+ }
123
+ const timeoutMs = options.timeoutMs ?? DEFAULT_CLOSE_TIMEOUT_MS;
124
+ this.#closeOnce = this.#runClose(timeoutMs);
125
+ return this.#closeOnce;
126
+ }
127
+ /**
128
+ * Retrieves anchor (link) data for a specific page by its database ID.
129
+ * @param pageId - The database ID of the page whose anchors to retrieve.
130
+ * @returns An array of anchor records found on the page.
131
+ */
132
+ async getAnchorsOnPage(pageId) {
133
+ const refs = await this.#db.getAnchorsOnPage(pageId);
134
+ return refs;
135
+ }
136
+ /**
137
+ * Retrieves the audios within the given page's detected main content
138
+ * region, from `page_main_content_audios`.
139
+ * @param pageId - The database id of the page.
140
+ * @returns Ordered audio rows.
141
+ */
142
+ async getAudiosOfPage(pageId) {
143
+ return this.#db.getAudiosOfPage(pageId);
144
+ }
145
+ /**
146
+ * Retrieves the button-like elements within the given page's detected
147
+ * main content region, from `page_main_content_buttons`.
148
+ * @param pageId - The database id of the page.
149
+ * @returns Ordered button rows.
150
+ */
151
+ async getButtonsOfPage(pageId) {
152
+ return this.#db.getButtonsOfPage(pageId);
153
+ }
154
+ /**
155
+ * Retrieves the canvases within the given page's detected main content
156
+ * region, from `page_main_content_canvases`.
157
+ * @param pageId - The database id of the page.
158
+ * @returns Ordered canvas rows.
159
+ */
160
+ async getCanvasesOfPage(pageId) {
161
+ return this.#db.getCanvasesOfPage(pageId);
162
+ }
163
+ /**
164
+ * Retrieves the crawl configuration stored in the archive database.
165
+ * @returns The parsed {@link Config} object.
166
+ */
167
+ async getConfig() {
168
+ return this.#db.getConfig();
169
+ }
170
+ /**
171
+ * Retrieves the Web Components (custom elements) within the given page's
172
+ * detected main content region, from `page_main_content_custom_elements`.
173
+ * @param pageId - The database id of the page.
174
+ * @returns Ordered custom-element rows.
175
+ */
176
+ async getCustomElementsOfPage(pageId) {
177
+ return this.#db.getCustomElementsOfPage(pageId);
178
+ }
179
+ async getData(name, format = 'json') {
180
+ const filePath = safePath(this.#tmpDir, `${name}.${format}`);
181
+ if (format === 'json') {
182
+ return await readJSON(filePath);
183
+ }
184
+ return await readText(filePath);
185
+ }
186
+ /**
187
+ * Retrieves the headings within the given page's detected main content
188
+ * region, from `page_main_content_headings`.
189
+ * @param pageId - The database id of the page.
190
+ * @returns Ordered heading rows.
191
+ */
192
+ async getHeadingsOfPage(pageId) {
193
+ return this.#db.getHeadingsOfPage(pageId);
194
+ }
195
+ /**
196
+ * Reads the HTML snapshot of a page from the archive.
197
+ *
198
+ * HTML is stored as zstd-compressed BLOBs in `page_html_blobs` (keyed by
199
+ * SHA-256 of the raw bytes) with `page_html_ref` linking `page_id → hash`.
200
+ * The read is a straight join + decompress; writer- and read-only (stub)
201
+ * accessors take the same code path because nothing here touches the
202
+ * filesystem.
203
+ *
204
+ * Returns `null` when the page row exists but has no stored body — for
205
+ * example a redirect source, a non-HTML resource (PDF), a page that
206
+ * failed to render, or an external page (whose row is metadata-only).
207
+ * Distinguishing "no body stored" from "empty body" is preserved: an
208
+ * empty HTML string returns `''`, not `null`.
209
+ *
210
+ * Throws if the page's referenced blob is missing, its codec marker is
211
+ * unrecognised, its zstd data is corrupt, or it decompresses past
212
+ * `MAX_DECODED_SNAPSHOT_BYTES` — each indicates an archive that was
213
+ * truncated, damaged or written by a future tool, none of which we
214
+ * silently paper over. A decode failure names the page and the blob hash
215
+ * (`HTML snapshot of page <id> (page_html_blobs hash <hex>) could not be decoded: …`).
216
+ * @param pageId - The database id of the page to read.
217
+ * @returns The HTML content as a UTF-8 string, or `null` when no body
218
+ * is stored for `pageId`.
219
+ * @example
220
+ * const html = await accessor.getHtmlOfPage(pageId);
221
+ * if (html === null) {
222
+ * // page has no stored body — redirect source / non-HTML / failed render
223
+ * } else {
224
+ * processHtml(html);
225
+ * }
226
+ */
227
+ async getHtmlOfPage(pageId) {
228
+ const html = await this.#db.getHtmlOfPageById(pageId);
229
+ if (html === null) {
230
+ log('No HTML body stored for page id=%d', pageId);
231
+ }
232
+ return html;
233
+ }
234
+ /**
235
+ * Retrieves the iframes within the given page's detected main content
236
+ * region, from `page_main_content_iframes`.
237
+ * @param pageId - The database id of the page.
238
+ * @returns Ordered iframe rows.
239
+ */
240
+ async getIframesOfPage(pageId) {
241
+ return this.#db.getIframesOfPage(pageId);
242
+ }
243
+ /**
244
+ * Retrieves the JSON-LD / SpeculationRules entries for the given page,
245
+ * parsed back from the `page_jsonld` table.
246
+ * @param pageId - The database id of the page.
247
+ * @returns Ordered entries with `kind`, `type`, `raw`, `parsed`, `parseError`.
248
+ */
249
+ async getJsonLdOfPage(pageId) {
250
+ return this.#db.getJsonLdOfPage(pageId);
251
+ }
252
+ /**
253
+ * Returns the underlying Knex query builder instance for direct SQL access.
254
+ * Enables advanced queries (GROUP BY, HAVING, JOINs) at the database layer
255
+ * for performance-critical operations on large datasets.
256
+ * @returns The Knex instance connected to the SQLite database.
257
+ */
258
+ getKnex() {
259
+ return this.#db.getKnex();
260
+ }
261
+ /**
262
+ * Retrieves the images within the given page's detected main content
263
+ * region, from `page_main_content_images`.
264
+ * @param pageId - The database id of the page.
265
+ * @returns Ordered image rows.
266
+ */
267
+ async getMainContentImagesOfPage(pageId) {
268
+ return this.#db.getMainContentImagesOfPage(pageId);
269
+ }
270
+ /**
271
+ * Retrieves the tables within the given page's detected main content
272
+ * region, from `page_main_content_tables`.
273
+ * @param pageId - The database id of the page.
274
+ * @returns Ordered table rows.
275
+ */
276
+ async getMainContentTablesOfPage(pageId) {
277
+ return this.#db.getMainContentTablesOfPage(pageId);
278
+ }
279
+ /**
280
+ * Retrieves all pages from the archive, optionally filtered by type.
281
+ * Eagerly loads redirect relationships (`redirectFrom`) but does NOT load
282
+ * anchor or referrer relationships.
283
+ * Use {@link getPagesWithRefs} if you need those relationships.
284
+ * @param filter - An optional filter to narrow the results (e.g., `'internal-page'`, `'external-page'`).
285
+ * @returns An array of {@link Page} instances.
286
+ */
287
+ async getPages(filter) {
288
+ const pages = await this.#db.getPages(filter);
289
+ if (pages.length === 0)
290
+ return [];
291
+ const pageIds = pages.map((p) => p.id);
292
+ const redirects = await this.#db.getRedirectsForPages(pageIds);
293
+ const redirectMap = new Map();
294
+ for (const redirect of redirects) {
295
+ const current = redirectMap.get(redirect.pageId);
296
+ if (current) {
297
+ current.push(redirect);
298
+ continue;
299
+ }
300
+ redirectMap.set(redirect.pageId, [redirect]);
301
+ }
302
+ return pages.map((page) => new Page(this, page, redirectMap.get(page.id) || []));
303
+ }
304
+ /**
305
+ * Retrieves pages with their related data (redirects, anchors, referrers) in batches.
306
+ * Processes pages in chunks of `limit` size, calling the callback for each batch.
307
+ * @param limit - The maximum number of pages to load per batch.
308
+ * @param callback - A function called for each batch of pages with the current offset and total count.
309
+ * @param options - Optional URL parsing options and whether to include referrer relationships.
310
+ */
311
+ async getPagesWithRefs(limit, callback, options) {
312
+ const max = await this.#getPageCount();
313
+ let times = 0;
314
+ while (true) {
315
+ const offset = times * limit;
316
+ log('%d times loop: %o', times, {
317
+ offset,
318
+ limit,
319
+ max,
320
+ });
321
+ const pages = await this.#getPagesWithRels(offset, limit, options);
322
+ if (pages.length === 0) {
323
+ break;
324
+ }
325
+ await callback(pages, offset, max);
326
+ times++;
327
+ }
328
+ }
329
+ /**
330
+ * Retrieves the confidence-combined technology roll-up for the given
331
+ * page, from the `page_technologies` table.
332
+ * @param pageId - The database id of the page.
333
+ * @returns Technology rows with category, version, confidence, signalCount.
334
+ */
335
+ async getPageTechnologiesOfPage(pageId) {
336
+ return this.#db.getPageTechnologiesOfPage(pageId);
337
+ }
338
+ /**
339
+ * Retrieves pages that link to the specified page (incoming links).
340
+ * @param pageId - The database ID of the target page.
341
+ * @returns An array of referrer records.
342
+ */
343
+ async getReferrersOfPage(pageId) {
344
+ const refs = await this.#db.getReferrersOfPage(pageId);
345
+ return refs;
346
+ }
347
+ /**
348
+ * Retrieves page URLs that reference the specified resource.
349
+ * @param pageId - The database ID of the resource.
350
+ * @returns An array of page URL strings that reference this resource.
351
+ */
352
+ async getReferrersOfResource(pageId) {
353
+ const refs = await this.#db.getReferrersOfResource(pageId);
354
+ return refs;
355
+ }
356
+ /**
357
+ * Retrieves all sub-resources (CSS, JS, images, etc.) stored in the archive.
358
+ * @returns An array of {@link Resource} instances.
359
+ */
360
+ async getResources() {
361
+ const resources = await this.#db.getResources();
362
+ return resources.map((r) => new Resource(this, r));
363
+ }
364
+ /**
365
+ * Retrieves a flat list of all resource URLs stored in the archive.
366
+ * @param onProgress - Forwarded to {@link Database.getResourceUrlList} —
367
+ * see that method's docs.
368
+ * @returns An array of resource URL strings.
369
+ */
370
+ async getResourceUrlList(onProgress) {
371
+ return this.#db.getResourceUrlList(onProgress);
372
+ }
373
+ /**
374
+ * Retrieves the raw technology-detection signals for the given page,
375
+ * from the `technology_signals` table.
376
+ * @param pageId - The database id of the page.
377
+ * @returns Ordered signal rows with technology, signalType, evidence, weight.
378
+ */
379
+ async getTechnologySignalsOfPage(pageId) {
380
+ return this.#db.getTechnologySignalsOfPage(pageId);
381
+ }
382
+ /**
383
+ * Retrieves the videos within the given page's detected main content
384
+ * region, from `page_main_content_videos`.
385
+ * @param pageId - The database id of the page.
386
+ * @returns Ordered video rows.
387
+ */
388
+ async getVideosOfPage(pageId) {
389
+ return this.#db.getVideosOfPage(pageId);
390
+ }
391
+ /**
392
+ * Returns the total number of internal pages in the archive.
393
+ */
394
+ async #getPageCount() {
395
+ return this.#db.getPageCount();
396
+ }
397
+ /**
398
+ * Loads a batch of pages with their related data (redirects, anchors, referrers).
399
+ * When `withRefs` is false, loads only pages without relationships for better performance.
400
+ * @param offset - The number of pages to skip
401
+ * @param limit - The maximum number of pages to return
402
+ * @param options - URL parsing and referrer loading options
403
+ */
404
+ async #getPagesWithRels(offset, limit, options) {
405
+ if (options?.withRefs === false) {
406
+ const pages = await this.#db.getPages('internal-page', offset, limit);
407
+ return pages.map((page) => new Page(this, page));
408
+ }
409
+ const { pages, redirects, anchors, referrers } = await this.#db.getPagesWithRels(offset, limit);
410
+ const redirectMap = new Map();
411
+ const anchorMap = new Map();
412
+ const refersMap = new Map();
413
+ log('Mapping redirects');
414
+ for (const redirect of redirects) {
415
+ const current = redirectMap.get(redirect.pageId);
416
+ if (current) {
417
+ current.push(redirect);
418
+ continue;
419
+ }
420
+ redirectMap.set(redirect.pageId, [redirect]);
421
+ }
422
+ log('Mapping anchors');
423
+ for (const anchor of anchors) {
424
+ const current = anchorMap.get(anchor.pageId);
425
+ if (current) {
426
+ current.push(anchor);
427
+ continue;
428
+ }
429
+ anchorMap.set(anchor.pageId, [anchor]);
430
+ }
431
+ log('Mapping referrers');
432
+ for (const referrer of referrers) {
433
+ const current = refersMap.get(referrer.pageId);
434
+ if (current) {
435
+ current.push(referrer);
436
+ continue;
437
+ }
438
+ refersMap.set(referrer.pageId, [referrer]);
439
+ }
440
+ log('Create Page Data');
441
+ const pPages = [];
442
+ for (const page of pages) {
443
+ const pRedirects = redirectMap.get(page.id) || [];
444
+ const pAnchors = anchorMap.get(page.id) || [];
445
+ const pRefers = refersMap.get(page.id) || [];
446
+ pPages.push(new Page(this, page, pRedirects, pAnchors, pRefers, options?.disableQueries));
447
+ }
448
+ log('Create Page Data: Done');
449
+ return pPages;
450
+ }
451
+ /**
452
+ * Actual close worker — invoked exactly once per accessor via
453
+ * {@link close}'s shared promise. Races `db.destroy()` against the
454
+ * caller-supplied timeout; on timeout we log and resolve so the
455
+ * consumer (typically a process shutting down) is not blocked, even
456
+ * though the underlying knex pool may still be draining in the
457
+ * background.
458
+ * @param timeoutMs - Maximum time to wait for `db.destroy()`.
459
+ */
460
+ async #runClose(timeoutMs) {
461
+ if (!Number.isFinite(timeoutMs)) {
462
+ await this.#db.destroy();
463
+ return;
464
+ }
465
+ // Track destroy() so we can attach an error-suppressing handler if we
466
+ // give up waiting — otherwise a late rejection becomes an unhandled
467
+ // promise rejection on the process.
468
+ const destroy = this.#db.destroy().then(() => 'done');
469
+ const { timeout } = await raceWithTimeout(() => destroy, timeoutMs);
470
+ if (timeout) {
471
+ log('ArchiveAccessor.close: db.destroy() did not settle within %dms — giving up', timeoutMs);
472
+ destroy.catch((error) => {
473
+ log('ArchiveAccessor.close: late db.destroy() rejection (post-timeout): %O', error);
474
+ });
475
+ }
476
+ }
477
+ }
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Lock failure surfaced when an archive's tmpDir is already in use by another
3
+ * live process.
4
+ *
5
+ * The lock path and PID of the holder are exposed so callers (CLI / orchestrator)
6
+ * can render a precise error message and let operators recover manually.
7
+ */
8
+ export declare class ArchiveLockError extends Error {
9
+ /** PID of the live process currently holding the lock, or `null` if unknown. */
10
+ readonly holderPid: number | null;
11
+ /** The lock directory path that could not be acquired. */
12
+ readonly lockPath: string;
13
+ /**
14
+ * @param lockPath - Absolute path of the lock directory.
15
+ * @param holderPid - PID stored in the lock's `pid.txt`, or `null` if unreadable.
16
+ */
17
+ constructor(lockPath: string, holderPid: number | null);
18
+ }
19
+ /**
20
+ * Acquire an advisory lock on the given tmpDir.
21
+ *
22
+ * Uses `fs.mkdir(lockPath, { recursive: false })` as the atomic primitive so
23
+ * that two concurrent processes targeting the same archive cannot both believe
24
+ * they own it. The lock is published as a sibling directory named
25
+ * `{tmpDir}.lock` containing a `pid.txt` file with the holder's PID.
26
+ *
27
+ * On `EEXIST` the function tries once to detect a stale lock: if the recorded
28
+ * PID is not alive any more (`process.kill(pid, 0)` throws `ESRCH`), the stale
29
+ * directory is removed and a single retry is issued. If acquisition still
30
+ * fails — or the holder is alive — an {@link ArchiveLockError} is thrown.
31
+ *
32
+ * The returned function releases the lock; it is idempotent so callers can put
33
+ * it in a `finally` block without worrying about double-release.
34
+ * @param tmpDir - Absolute path to the archive's temporary working directory.
35
+ * @returns A release function to be called when the work is done.
36
+ * @throws {ArchiveLockError} When the lock cannot be acquired even after a stale-lock retry.
37
+ * @example
38
+ * const releaseLock = await acquireArchiveLock(tmpDir);
39
+ * try {
40
+ * // ... exclusive work against tmpDir ...
41
+ * } finally {
42
+ * await releaseLock();
43
+ * }
44
+ */
45
+ export declare function acquireArchiveLock(tmpDir: string): Promise<() => Promise<void>>;
@@ -0,0 +1,154 @@
1
+ import fs from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ /**
4
+ * Lock failure surfaced when an archive's tmpDir is already in use by another
5
+ * live process.
6
+ *
7
+ * The lock path and PID of the holder are exposed so callers (CLI / orchestrator)
8
+ * can render a precise error message and let operators recover manually.
9
+ */
10
+ export class ArchiveLockError extends Error {
11
+ /** PID of the live process currently holding the lock, or `null` if unknown. */
12
+ holderPid;
13
+ /** The lock directory path that could not be acquired. */
14
+ lockPath;
15
+ /**
16
+ * @param lockPath - Absolute path of the lock directory.
17
+ * @param holderPid - PID stored in the lock's `pid.txt`, or `null` if unreadable.
18
+ */
19
+ constructor(lockPath, holderPid) {
20
+ const suffix = holderPid === null ? '' : ` (PID ${holderPid})`;
21
+ super(`Archive is being used by another process${suffix}: ${lockPath}`);
22
+ this.name = 'ArchiveLockError';
23
+ this.lockPath = lockPath;
24
+ this.holderPid = holderPid;
25
+ }
26
+ }
27
+ /**
28
+ * Acquire an advisory lock on the given tmpDir.
29
+ *
30
+ * Uses `fs.mkdir(lockPath, { recursive: false })` as the atomic primitive so
31
+ * that two concurrent processes targeting the same archive cannot both believe
32
+ * they own it. The lock is published as a sibling directory named
33
+ * `{tmpDir}.lock` containing a `pid.txt` file with the holder's PID.
34
+ *
35
+ * On `EEXIST` the function tries once to detect a stale lock: if the recorded
36
+ * PID is not alive any more (`process.kill(pid, 0)` throws `ESRCH`), the stale
37
+ * directory is removed and a single retry is issued. If acquisition still
38
+ * fails — or the holder is alive — an {@link ArchiveLockError} is thrown.
39
+ *
40
+ * The returned function releases the lock; it is idempotent so callers can put
41
+ * it in a `finally` block without worrying about double-release.
42
+ * @param tmpDir - Absolute path to the archive's temporary working directory.
43
+ * @returns A release function to be called when the work is done.
44
+ * @throws {ArchiveLockError} When the lock cannot be acquired even after a stale-lock retry.
45
+ * @example
46
+ * const releaseLock = await acquireArchiveLock(tmpDir);
47
+ * try {
48
+ * // ... exclusive work against tmpDir ...
49
+ * } finally {
50
+ * await releaseLock();
51
+ * }
52
+ */
53
+ export async function acquireArchiveLock(tmpDir) {
54
+ const lockPath = `${tmpDir}.lock`;
55
+ await tryAcquire(lockPath);
56
+ let released = false;
57
+ return async () => {
58
+ if (released) {
59
+ return;
60
+ }
61
+ released = true;
62
+ await releaseLock(lockPath);
63
+ };
64
+ }
65
+ /**
66
+ * Attempt to create the lock directory and write the holder PID, handling one
67
+ * round of stale-lock recovery.
68
+ * @param lockPath - The absolute lock directory path.
69
+ */
70
+ async function tryAcquire(lockPath) {
71
+ try {
72
+ await fs.mkdir(lockPath, { recursive: false });
73
+ await fs.writeFile(path.join(lockPath, 'pid.txt'), String(process.pid), 'utf8');
74
+ return;
75
+ }
76
+ catch (error) {
77
+ if (!isEexist(error)) {
78
+ throw error;
79
+ }
80
+ }
81
+ const holderPid = await readHolderPid(lockPath);
82
+ if (holderPid !== null && isProcessAlive(holderPid)) {
83
+ throw new ArchiveLockError(lockPath, holderPid);
84
+ }
85
+ // Stale lock — clean up and retry once. A concurrent acquirer may have
86
+ // already reclaimed the directory, so a second EEXIST is treated as a real
87
+ // collision and surfaced to the caller.
88
+ await fs.rm(lockPath, { recursive: true, force: true });
89
+ try {
90
+ await fs.mkdir(lockPath, { recursive: false });
91
+ await fs.writeFile(path.join(lockPath, 'pid.txt'), String(process.pid), 'utf8');
92
+ }
93
+ catch (error) {
94
+ if (isEexist(error)) {
95
+ const pid = await readHolderPid(lockPath);
96
+ throw new ArchiveLockError(lockPath, pid);
97
+ }
98
+ throw error;
99
+ }
100
+ }
101
+ /**
102
+ * Remove the lock directory. `ENOENT` is swallowed so `finally`-style callers
103
+ * tolerate an externally-cleaned lock, but any other failure (permissions,
104
+ * disk full, etc.) is propagated so the caller can react.
105
+ * @param lockPath - The absolute lock directory path.
106
+ */
107
+ async function releaseLock(lockPath) {
108
+ try {
109
+ await fs.rm(lockPath, { recursive: true, force: true });
110
+ }
111
+ catch (error) {
112
+ if (error.code !== 'ENOENT') {
113
+ throw error;
114
+ }
115
+ }
116
+ }
117
+ /**
118
+ * Read the PID stored in the lock directory's `pid.txt`.
119
+ * @param lockPath - The absolute lock directory path.
120
+ * @returns The parsed PID, or `null` if the file is missing or malformed.
121
+ */
122
+ async function readHolderPid(lockPath) {
123
+ try {
124
+ const raw = await fs.readFile(path.join(lockPath, 'pid.txt'), 'utf8');
125
+ const pid = Number.parseInt(raw.trim(), 10);
126
+ return Number.isFinite(pid) && pid > 0 ? pid : null;
127
+ }
128
+ catch {
129
+ return null;
130
+ }
131
+ }
132
+ /**
133
+ * Check whether the given PID is still alive using a signal-0 probe.
134
+ * @param pid - The process id to probe.
135
+ * @returns `true` if the process exists (regardless of permission).
136
+ */
137
+ function isProcessAlive(pid) {
138
+ try {
139
+ process.kill(pid, 0);
140
+ return true;
141
+ }
142
+ catch (error) {
143
+ // EPERM means the process exists but is owned by another user — still alive.
144
+ return error.code === 'EPERM';
145
+ }
146
+ }
147
+ /**
148
+ * Type-guard for an `EEXIST` filesystem error.
149
+ * @param error - The error value to inspect.
150
+ * @returns `true` when the error is a Node `EEXIST` from a filesystem call.
151
+ */
152
+ function isEexist(error) {
153
+ return error?.code === 'EEXIST';
154
+ }