@nitpicker/archive 0.0.0 → 0.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (739) hide show
  1. package/LICENSE +191 -0
  2. package/README.md +106 -0
  3. package/lib/apply-connection-pragmas.d.ts +18 -0
  4. package/lib/apply-connection-pragmas.js +40 -0
  5. package/lib/archive-accessor.d.ts +265 -0
  6. package/lib/archive-accessor.js +477 -0
  7. package/lib/archive-lock.d.ts +45 -0
  8. package/lib/archive-lock.js +154 -0
  9. package/lib/archive.d.ts +736 -0
  10. package/lib/archive.js +1055 -0
  11. package/lib/body-hash/compute-body-hash.d.ts +32 -0
  12. package/lib/body-hash/compute-body-hash.js +35 -0
  13. package/lib/cache/clear-archive-cache-entry.d.ts +18 -0
  14. package/lib/cache/clear-archive-cache-entry.js +24 -0
  15. package/lib/cache/clear-archive-cache-root.d.ts +26 -0
  16. package/lib/cache/clear-archive-cache-root.js +32 -0
  17. package/lib/cache/compute-archive-cache-key.d.ts +39 -0
  18. package/lib/cache/compute-archive-cache-key.js +95 -0
  19. package/lib/cache/extract-archive-to-cache.d.ts +57 -0
  20. package/lib/cache/extract-archive-to-cache.js +326 -0
  21. package/lib/cache/get-archive-cache-root.d.ts +20 -0
  22. package/lib/cache/get-archive-cache-root.js +53 -0
  23. package/lib/cache/is-archive-cache-disabled.d.ts +24 -0
  24. package/lib/cache/is-archive-cache-disabled.js +34 -0
  25. package/lib/cache/list-archive-cache-entries.d.ts +23 -0
  26. package/lib/cache/list-archive-cache-entries.js +130 -0
  27. package/lib/cache/path-exists.d.ts +9 -0
  28. package/lib/cache/path-exists.js +18 -0
  29. package/lib/cache/resolve-archive-cache-dir.d.ts +26 -0
  30. package/lib/cache/resolve-archive-cache-dir.js +32 -0
  31. package/lib/cache/types.d.ts +33 -0
  32. package/lib/cache/types.js +1 -0
  33. package/lib/content-type/is-html-content-type.d.ts +17 -0
  34. package/lib/content-type/is-html-content-type.js +19 -0
  35. package/lib/content-type/normalize-content-type.d.ts +14 -0
  36. package/lib/content-type/normalize-content-type.js +20 -0
  37. package/lib/create-adjunct-tables.d.ts +66 -0
  38. package/lib/create-adjunct-tables.js +510 -0
  39. package/lib/create-entity-tables.d.ts +191 -0
  40. package/lib/create-entity-tables.js +387 -0
  41. package/lib/create-progress-reporter.d.ts +30 -0
  42. package/lib/create-progress-reporter.js +38 -0
  43. package/lib/create-ref-tables.d.ts +35 -0
  44. package/lib/create-ref-tables.js +185 -0
  45. package/lib/create-template-tables.d.ts +28 -0
  46. package/lib/create-template-tables.js +92 -0
  47. package/lib/database.d.ts +549 -0
  48. package/lib/database.js +803 -0
  49. package/lib/db-ops/_shared/clear-write-ref-caches.d.ts +27 -0
  50. package/lib/db-ops/_shared/clear-write-ref-caches.js +35 -0
  51. package/lib/db-ops/_shared/compress-payload.d.ts +26 -0
  52. package/lib/db-ops/_shared/compress-payload.js +30 -0
  53. package/lib/db-ops/_shared/create-write-ref-caches.d.ts +17 -0
  54. package/lib/db-ops/_shared/create-write-ref-caches.js +27 -0
  55. package/lib/db-ops/_shared/decode-json-ref.d.ts +17 -0
  56. package/lib/db-ops/_shared/decode-json-ref.js +31 -0
  57. package/lib/db-ops/_shared/keyset-paginate-by-id.d.ts +49 -0
  58. package/lib/db-ops/_shared/keyset-paginate-by-id.js +69 -0
  59. package/lib/db-ops/_shared/load-response-headers-by-set-ids.d.ts +20 -0
  60. package/lib/db-ops/_shared/load-response-headers-by-set-ids.js +53 -0
  61. package/lib/db-ops/_shared/resolve-content-item-id.d.ts +62 -0
  62. package/lib/db-ops/_shared/resolve-content-item-id.js +143 -0
  63. package/lib/db-ops/_shared/resolve-url-or-blob.d.ts +23 -0
  64. package/lib/db-ops/_shared/resolve-url-or-blob.js +29 -0
  65. package/lib/db-ops/_shared/retry-setting.d.ts +16 -0
  66. package/lib/db-ops/_shared/retry-setting.js +18 -0
  67. package/lib/db-ops/_shared/safe-parse-json.d.ts +11 -0
  68. package/lib/db-ops/_shared/safe-parse-json.js +18 -0
  69. package/lib/db-ops/_shared/types.d.ts +94 -0
  70. package/lib/db-ops/_shared/types.js +1 -0
  71. package/lib/db-ops/_shared/upsert-blob-ref.d.ts +25 -0
  72. package/lib/db-ops/_shared/upsert-blob-ref.js +48 -0
  73. package/lib/db-ops/_shared/upsert-content-type-ref.d.ts +30 -0
  74. package/lib/db-ops/_shared/upsert-content-type-ref.js +45 -0
  75. package/lib/db-ops/_shared/upsert-json-ref.d.ts +22 -0
  76. package/lib/db-ops/_shared/upsert-json-ref.js +41 -0
  77. package/lib/db-ops/_shared/upsert-response-headers.d.ts +35 -0
  78. package/lib/db-ops/_shared/upsert-response-headers.js +49 -0
  79. package/lib/db-ops/_shared/upsert-url-ref.d.ts +39 -0
  80. package/lib/db-ops/_shared/upsert-url-ref.js +62 -0
  81. package/lib/db-ops/anchors/get-anchors-on-page.d.ts +10 -0
  82. package/lib/db-ops/anchors/get-anchors-on-page.js +21 -0
  83. package/lib/db-ops/config/get-base-url.d.ts +8 -0
  84. package/lib/db-ops/config/get-base-url.js +14 -0
  85. package/lib/db-ops/config/get-config.d.ts +10 -0
  86. package/lib/db-ops/config/get-config.js +29 -0
  87. package/lib/db-ops/config/get-created-cwd.d.ts +11 -0
  88. package/lib/db-ops/config/get-created-cwd.js +13 -0
  89. package/lib/db-ops/config/get-name.d.ts +8 -0
  90. package/lib/db-ops/config/get-name.js +14 -0
  91. package/lib/db-ops/config/info-column-allowlist.d.ts +7 -0
  92. package/lib/db-ops/config/info-column-allowlist.js +29 -0
  93. package/lib/db-ops/config/info-json-columns.d.ts +5 -0
  94. package/lib/db-ops/config/info-json-columns.js +11 -0
  95. package/lib/db-ops/config/set-config.d.ts +12 -0
  96. package/lib/db-ops/config/set-config.js +21 -0
  97. package/lib/db-ops/config/update-config.d.ts +17 -0
  98. package/lib/db-ops/config/update-config.js +36 -0
  99. package/lib/db-ops/console-logs/compute-console-log-hash.d.ts +19 -0
  100. package/lib/db-ops/console-logs/compute-console-log-hash.js +30 -0
  101. package/lib/db-ops/console-logs/replace-console-logs.d.ts +34 -0
  102. package/lib/db-ops/console-logs/replace-console-logs.js +123 -0
  103. package/lib/db-ops/console-logs/stringify-console-log-args.d.ts +16 -0
  104. package/lib/db-ops/console-logs/stringify-console-log-args.js +26 -0
  105. package/lib/db-ops/console-logs/types.d.ts +38 -0
  106. package/lib/db-ops/console-logs/types.js +1 -0
  107. package/lib/db-ops/console-logs/upsert-console-log-item.d.ts +29 -0
  108. package/lib/db-ops/console-logs/upsert-console-log-item.js +52 -0
  109. package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts +18 -0
  110. package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js +23 -0
  111. package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts +12 -0
  112. package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.js +15 -0
  113. package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts +14 -0
  114. package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.js +30 -0
  115. package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.d.ts +57 -0
  116. package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.js +87 -0
  117. package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts +21 -0
  118. package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js +27 -0
  119. package/lib/db-ops/errors/insert-crawl-error.d.ts +15 -0
  120. package/lib/db-ops/errors/insert-crawl-error.js +21 -0
  121. package/lib/db-ops/errors/insert-page-error.d.ts +21 -0
  122. package/lib/db-ops/errors/insert-page-error.js +30 -0
  123. package/lib/db-ops/errors/list-dns-burned-host-candidates.d.ts +33 -0
  124. package/lib/db-ops/errors/list-dns-burned-host-candidates.js +158 -0
  125. package/lib/db-ops/html/read-page-html-blob.d.ts +17 -0
  126. package/lib/db-ops/html/read-page-html-blob.js +20 -0
  127. package/lib/db-ops/lifecycle/checkpoint.d.ts +8 -0
  128. package/lib/db-ops/lifecycle/checkpoint.js +9 -0
  129. package/lib/db-ops/lifecycle/destroy.d.ts +6 -0
  130. package/lib/db-ops/lifecycle/destroy.js +7 -0
  131. package/lib/db-ops/lifecycle/init.d.ts +58 -0
  132. package/lib/db-ops/lifecycle/init.js +122 -0
  133. package/lib/db-ops/list-reconcile/record-list-reconcile-run.d.ts +22 -0
  134. package/lib/db-ops/list-reconcile/record-list-reconcile-run.js +41 -0
  135. package/lib/db-ops/meta/get-audios-of-page.d.ts +11 -0
  136. package/lib/db-ops/meta/get-audios-of-page.js +15 -0
  137. package/lib/db-ops/meta/get-buttons-of-page.d.ts +11 -0
  138. package/lib/db-ops/meta/get-buttons-of-page.js +15 -0
  139. package/lib/db-ops/meta/get-canvases-of-page.d.ts +11 -0
  140. package/lib/db-ops/meta/get-canvases-of-page.js +15 -0
  141. package/lib/db-ops/meta/get-custom-elements-of-page.d.ts +11 -0
  142. package/lib/db-ops/meta/get-custom-elements-of-page.js +15 -0
  143. package/lib/db-ops/meta/get-headings-of-page.d.ts +11 -0
  144. package/lib/db-ops/meta/get-headings-of-page.js +15 -0
  145. package/lib/db-ops/meta/get-iframes-of-page.d.ts +11 -0
  146. package/lib/db-ops/meta/get-iframes-of-page.js +15 -0
  147. package/lib/db-ops/meta/get-jsonld-of-page.d.ts +13 -0
  148. package/lib/db-ops/meta/get-jsonld-of-page.js +27 -0
  149. package/lib/db-ops/meta/get-main-content-images-of-page.d.ts +11 -0
  150. package/lib/db-ops/meta/get-main-content-images-of-page.js +15 -0
  151. package/lib/db-ops/meta/get-main-content-tables-of-page.d.ts +11 -0
  152. package/lib/db-ops/meta/get-main-content-tables-of-page.js +15 -0
  153. package/lib/db-ops/meta/get-page-technologies-of-page.d.ts +11 -0
  154. package/lib/db-ops/meta/get-page-technologies-of-page.js +15 -0
  155. package/lib/db-ops/meta/get-technology-signals-of-page.d.ts +11 -0
  156. package/lib/db-ops/meta/get-technology-signals-of-page.js +15 -0
  157. package/lib/db-ops/meta/get-videos-of-page.d.ts +11 -0
  158. package/lib/db-ops/meta/get-videos-of-page.js +15 -0
  159. package/lib/db-ops/outages/close-network-outage.d.ts +12 -0
  160. package/lib/db-ops/outages/close-network-outage.js +15 -0
  161. package/lib/db-ops/outages/close-stale-open-network-outages.d.ts +26 -0
  162. package/lib/db-ops/outages/close-stale-open-network-outages.js +42 -0
  163. package/lib/db-ops/outages/compute-outage-clamp-timestamp.d.ts +26 -0
  164. package/lib/db-ops/outages/compute-outage-clamp-timestamp.js +31 -0
  165. package/lib/db-ops/outages/insert-network-outage.d.ts +14 -0
  166. package/lib/db-ops/outages/insert-network-outage.js +29 -0
  167. package/lib/db-ops/outages/list-network-outages.d.ts +28 -0
  168. package/lib/db-ops/outages/list-network-outages.js +42 -0
  169. package/lib/db-ops/pages/order/set-url-order.d.ts +13 -0
  170. package/lib/db-ops/pages/order/set-url-order.js +38 -0
  171. package/lib/db-ops/pages/read/build-page-query.d.ts +18 -0
  172. package/lib/db-ops/pages/read/build-page-query.js +40 -0
  173. package/lib/db-ops/pages/read/get-crawling-state.d.ts +117 -0
  174. package/lib/db-ops/pages/read/get-crawling-state.js +161 -0
  175. package/lib/db-ops/pages/read/get-existing-page-urls.d.ts +15 -0
  176. package/lib/db-ops/pages/read/get-existing-page-urls.js +30 -0
  177. package/lib/db-ops/pages/read/get-page-count.d.ts +12 -0
  178. package/lib/db-ops/pages/read/get-page-count.js +21 -0
  179. package/lib/db-ops/pages/read/get-page-source-by-url.d.ts +24 -0
  180. package/lib/db-ops/pages/read/get-page-source-by-url.js +28 -0
  181. package/lib/db-ops/pages/read/get-pages-with-rels.d.ts +38 -0
  182. package/lib/db-ops/pages/read/get-pages-with-rels.js +107 -0
  183. package/lib/db-ops/pages/read/get-pages.d.ts +11 -0
  184. package/lib/db-ops/pages/read/get-pages.js +51 -0
  185. package/lib/db-ops/pages/read/get-scraped-html-page-count.d.ts +18 -0
  186. package/lib/db-ops/pages/read/get-scraped-html-page-count.js +25 -0
  187. package/lib/db-ops/pages/read/reconstruct-page-rows.d.ts +31 -0
  188. package/lib/db-ops/pages/read/reconstruct-page-rows.js +32 -0
  189. package/lib/db-ops/pages/reset/clear-page-derived-rows.d.ts +20 -0
  190. package/lib/db-ops/pages/reset/clear-page-derived-rows.js +37 -0
  191. package/lib/db-ops/pages/reset/repromote-external-pages.d.ts +30 -0
  192. package/lib/db-ops/pages/reset/repromote-external-pages.js +91 -0
  193. package/lib/db-ops/pages/reset/reset-failed-pages.d.ts +88 -0
  194. package/lib/db-ops/pages/reset/reset-failed-pages.js +216 -0
  195. package/lib/db-ops/pages/reset/reset-pages-by-urls.d.ts +80 -0
  196. package/lib/db-ops/pages/reset/reset-pages-by-urls.js +151 -0
  197. package/lib/db-ops/pages/reset/retryable-image-scan-codes.d.ts +24 -0
  198. package/lib/db-ops/pages/reset/retryable-image-scan-codes.js +24 -0
  199. package/lib/db-ops/pages/write/insert-audios.d.ts +14 -0
  200. package/lib/db-ops/pages/write/insert-audios.js +25 -0
  201. package/lib/db-ops/pages/write/insert-buttons.d.ts +14 -0
  202. package/lib/db-ops/pages/write/insert-buttons.js +29 -0
  203. package/lib/db-ops/pages/write/insert-canvases.d.ts +14 -0
  204. package/lib/db-ops/pages/write/insert-canvases.js +26 -0
  205. package/lib/db-ops/pages/write/insert-custom-elements.d.ts +27 -0
  206. package/lib/db-ops/pages/write/insert-custom-elements.js +40 -0
  207. package/lib/db-ops/pages/write/insert-headings.d.ts +15 -0
  208. package/lib/db-ops/pages/write/insert-headings.js +27 -0
  209. package/lib/db-ops/pages/write/insert-iframes.d.ts +14 -0
  210. package/lib/db-ops/pages/write/insert-iframes.js +28 -0
  211. package/lib/db-ops/pages/write/insert-inventory-content-items.d.ts +38 -0
  212. package/lib/db-ops/pages/write/insert-inventory-content-items.js +60 -0
  213. package/lib/db-ops/pages/write/insert-inventory-seeds.d.ts +36 -0
  214. package/lib/db-ops/pages/write/insert-inventory-seeds.js +48 -0
  215. package/lib/db-ops/pages/write/insert-inventory-skipped-pages.d.ts +42 -0
  216. package/lib/db-ops/pages/write/insert-inventory-skipped-pages.js +56 -0
  217. package/lib/db-ops/pages/write/insert-jsonld.d.ts +17 -0
  218. package/lib/db-ops/pages/write/insert-jsonld.js +49 -0
  219. package/lib/db-ops/pages/write/insert-main-content-images.d.ts +20 -0
  220. package/lib/db-ops/pages/write/insert-main-content-images.js +32 -0
  221. package/lib/db-ops/pages/write/insert-main-content-tables.d.ts +14 -0
  222. package/lib/db-ops/pages/write/insert-main-content-tables.js +29 -0
  223. package/lib/db-ops/pages/write/insert-page.d.ts +56 -0
  224. package/lib/db-ops/pages/write/insert-page.js +259 -0
  225. package/lib/db-ops/pages/write/insert-technologies.d.ts +25 -0
  226. package/lib/db-ops/pages/write/insert-technologies.js +45 -0
  227. package/lib/db-ops/pages/write/insert-videos.d.ts +14 -0
  228. package/lib/db-ops/pages/write/insert-videos.js +28 -0
  229. package/lib/db-ops/pages/write/link-redirect-sources.d.ts +42 -0
  230. package/lib/db-ops/pages/write/link-redirect-sources.js +101 -0
  231. package/lib/db-ops/pages/write/record-redirect.d.ts +35 -0
  232. package/lib/db-ops/pages/write/record-redirect.js +100 -0
  233. package/lib/db-ops/pages/write/set-skipped-page.d.ts +13 -0
  234. package/lib/db-ops/pages/write/set-skipped-page.js +24 -0
  235. package/lib/db-ops/pages/write/update-page.d.ts +40 -0
  236. package/lib/db-ops/pages/write/update-page.js +419 -0
  237. package/lib/db-ops/pages/write/write-page-html-blob.d.ts +19 -0
  238. package/lib/db-ops/pages/write/write-page-html-blob.js +41 -0
  239. package/lib/db-ops/referrers/get-redirects-for-pages.d.ts +9 -0
  240. package/lib/db-ops/referrers/get-redirects-for-pages.js +15 -0
  241. package/lib/db-ops/referrers/get-referrers-of-page.d.ts +17 -0
  242. package/lib/db-ops/referrers/get-referrers-of-page.js +32 -0
  243. package/lib/db-ops/referrers/get-referrers-of-resource.d.ts +8 -0
  244. package/lib/db-ops/referrers/get-referrers-of-resource.js +15 -0
  245. package/lib/db-ops/resources/build-resource-query.d.ts +25 -0
  246. package/lib/db-ops/resources/build-resource-query.js +29 -0
  247. package/lib/db-ops/resources/get-existing-resource-urls.d.ts +9 -0
  248. package/lib/db-ops/resources/get-existing-resource-urls.js +24 -0
  249. package/lib/db-ops/resources/get-resource-by-url.d.ts +13 -0
  250. package/lib/db-ops/resources/get-resource-by-url.js +22 -0
  251. package/lib/db-ops/resources/get-resource-url-list.d.ts +17 -0
  252. package/lib/db-ops/resources/get-resource-url-list.js +26 -0
  253. package/lib/db-ops/resources/get-resources.d.ts +8 -0
  254. package/lib/db-ops/resources/get-resources.js +11 -0
  255. package/lib/db-ops/resources/insert-inventory-resources.d.ts +24 -0
  256. package/lib/db-ops/resources/insert-inventory-resources.js +64 -0
  257. package/lib/db-ops/resources/insert-resource-referrers.d.ts +15 -0
  258. package/lib/db-ops/resources/insert-resource-referrers.js +54 -0
  259. package/lib/db-ops/resources/insert-resource.d.ts +34 -0
  260. package/lib/db-ops/resources/insert-resource.js +73 -0
  261. package/lib/db-ops/resources/reconstruct-resource-rows.d.ts +26 -0
  262. package/lib/db-ops/resources/reconstruct-resource-rows.js +30 -0
  263. package/lib/db-ops/templates/assign-template-labels.d.ts +60 -0
  264. package/lib/db-ops/templates/assign-template-labels.js +187 -0
  265. package/lib/db-ops/templates/replace-page-templates.d.ts +44 -0
  266. package/lib/db-ops/templates/replace-page-templates.js +174 -0
  267. package/lib/db-ops/templates/serialize-template-cluster-reason.d.ts +16 -0
  268. package/lib/db-ops/templates/serialize-template-cluster-reason.js +17 -0
  269. package/lib/db-ops/templates/types.d.ts +103 -0
  270. package/lib/db-ops/templates/types.js +1 -0
  271. package/lib/debug.d.ts +8 -0
  272. package/lib/debug.js +9 -0
  273. package/lib/decode-html-blob.d.ts +20 -0
  274. package/lib/decode-html-blob.js +23 -0
  275. package/lib/decode-page-html-blob-bytes.d.ts +25 -0
  276. package/lib/decode-page-html-blob-bytes.js +29 -0
  277. package/lib/decode-stored-blob-bytes.d.ts +25 -0
  278. package/lib/decode-stored-blob-bytes.js +47 -0
  279. package/lib/derive-lineage-from-parent.d.ts +37 -0
  280. package/lib/derive-lineage-from-parent.js +42 -0
  281. package/lib/drop-legacy-tables.d.ts +45 -0
  282. package/lib/drop-legacy-tables.js +56 -0
  283. package/lib/error-kind/classify-error-kind.d.ts +20 -0
  284. package/lib/error-kind/classify-error-kind.js +136 -0
  285. package/lib/error-kind/network-related-error-kinds.d.ts +23 -0
  286. package/lib/error-kind/network-related-error-kinds.js +28 -0
  287. package/lib/error-kind/permanent-error-kinds.d.ts +48 -0
  288. package/lib/error-kind/permanent-error-kinds.js +54 -0
  289. package/lib/error-kind/types.d.ts +54 -0
  290. package/lib/error-kind/types.js +1 -0
  291. package/lib/filesystem/append-text.d.ts +9 -0
  292. package/lib/filesystem/append-text.js +14 -0
  293. package/lib/filesystem/copy-dir-sync.d.ts +6 -0
  294. package/lib/filesystem/copy-dir-sync.js +9 -0
  295. package/lib/filesystem/copy-dir.d.ts +7 -0
  296. package/lib/filesystem/copy-dir.js +13 -0
  297. package/lib/filesystem/copy-file-with-progress.d.ts +20 -0
  298. package/lib/filesystem/copy-file-with-progress.js +34 -0
  299. package/lib/filesystem/exists.d.ts +6 -0
  300. package/lib/filesystem/exists.js +9 -0
  301. package/lib/filesystem/get-file-list.d.ts +8 -0
  302. package/lib/filesystem/get-file-list.js +12 -0
  303. package/lib/filesystem/is-dir.d.ts +6 -0
  304. package/lib/filesystem/is-dir.js +10 -0
  305. package/lib/filesystem/mkdir.d.ts +8 -0
  306. package/lib/filesystem/mkdir.js +15 -0
  307. package/lib/filesystem/output-binary.d.ts +12 -0
  308. package/lib/filesystem/output-binary.js +17 -0
  309. package/lib/filesystem/output-json.d.ts +9 -0
  310. package/lib/filesystem/output-json.js +14 -0
  311. package/lib/filesystem/output-text.d.ts +11 -0
  312. package/lib/filesystem/output-text.js +32 -0
  313. package/lib/filesystem/parse-pax-path.d.ts +18 -0
  314. package/lib/filesystem/parse-pax-path.js +45 -0
  315. package/lib/filesystem/parse-tar-size-field.d.ts +13 -0
  316. package/lib/filesystem/parse-tar-size-field.js +37 -0
  317. package/lib/filesystem/peek-tar-top-dir.d.ts +35 -0
  318. package/lib/filesystem/peek-tar-top-dir.js +215 -0
  319. package/lib/filesystem/read-json.d.ts +7 -0
  320. package/lib/filesystem/read-json.js +11 -0
  321. package/lib/filesystem/read-text.d.ts +6 -0
  322. package/lib/filesystem/read-text.js +10 -0
  323. package/lib/filesystem/readline.d.ts +11 -0
  324. package/lib/filesystem/readline.js +26 -0
  325. package/lib/filesystem/remove.d.ts +5 -0
  326. package/lib/filesystem/remove.js +10 -0
  327. package/lib/filesystem/rename.d.ts +16 -0
  328. package/lib/filesystem/rename.js +48 -0
  329. package/lib/filesystem/tar.d.ts +23 -0
  330. package/lib/filesystem/tar.js +78 -0
  331. package/lib/filesystem/untar.d.ts +33 -0
  332. package/lib/filesystem/untar.js +84 -0
  333. package/lib/get-failed-page-messages.d.ts +56 -0
  334. package/lib/get-failed-page-messages.js +146 -0
  335. package/lib/get-json.d.ts +10 -0
  336. package/lib/get-json.js +25 -0
  337. package/lib/html-snapshot/match-selector-html.d.ts +25 -0
  338. package/lib/html-snapshot/match-selector-html.js +28 -0
  339. package/lib/html-snapshot/match-selector-snapshots.d.ts +33 -0
  340. package/lib/html-snapshot/match-selector-snapshots.js +33 -0
  341. package/lib/html-snapshot/search-html-document.d.ts +29 -0
  342. package/lib/html-snapshot/search-html-document.js +27 -0
  343. package/lib/html-snapshot/search-html-snapshots.d.ts +36 -0
  344. package/lib/html-snapshot/search-html-snapshots.js +35 -0
  345. package/lib/html-snapshot/types.d.ts +111 -0
  346. package/lib/html-snapshot/types.js +7 -0
  347. package/lib/html-snapshot/validate-search-pattern.d.ts +15 -0
  348. package/lib/html-snapshot/validate-search-pattern.js +17 -0
  349. package/lib/init-schema.d.ts +48 -0
  350. package/lib/init-schema.js +124 -0
  351. package/lib/is-inventory-source.d.ts +21 -0
  352. package/lib/is-inventory-source.js +22 -0
  353. package/lib/is-within-outage-window.d.ts +49 -0
  354. package/lib/is-within-outage-window.js +33 -0
  355. package/lib/isolated-cluster/create-isolated-cluster-run.d.ts +24 -0
  356. package/lib/isolated-cluster/create-isolated-cluster-run.js +26 -0
  357. package/lib/libsql-dialect.d.ts +115 -0
  358. package/lib/libsql-dialect.js +116 -0
  359. package/lib/limited-page-ids.d.ts +9 -0
  360. package/lib/limited-page-ids.js +18 -0
  361. package/lib/meta/assert-compatible-version.d.ts +60 -0
  362. package/lib/meta/assert-compatible-version.js +93 -0
  363. package/lib/meta/classify-jsonld-type.d.ts +23 -0
  364. package/lib/meta/classify-jsonld-type.js +43 -0
  365. package/lib/meta/compare-semver.d.ts +23 -0
  366. package/lib/meta/compare-semver.js +51 -0
  367. package/lib/meta/compute-main-contents-denormalized.d.ts +41 -0
  368. package/lib/meta/compute-main-contents-denormalized.js +88 -0
  369. package/lib/meta/compute-page-denormalized.d.ts +20 -0
  370. package/lib/meta/compute-page-denormalized.js +35 -0
  371. package/lib/meta/derive-flat-from-meta.d.ts +34 -0
  372. package/lib/meta/derive-flat-from-meta.js +158 -0
  373. package/lib/meta/derive-meta-extras.d.ts +20 -0
  374. package/lib/meta/derive-meta-extras.js +23 -0
  375. package/lib/meta/extract-tags-for-archive.d.ts +17 -0
  376. package/lib/meta/extract-tags-for-archive.js +36 -0
  377. package/lib/meta/parse-main-content-class-list.d.ts +19 -0
  378. package/lib/meta/parse-main-content-class-list.js +34 -0
  379. package/lib/meta/summarize-jsonld.d.ts +17 -0
  380. package/lib/meta/summarize-jsonld.js +29 -0
  381. package/lib/meta/technologies/combine-technology-confidence.d.ts +46 -0
  382. package/lib/meta/technologies/combine-technology-confidence.js +93 -0
  383. package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.d.ts +45 -0
  384. package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.js +53 -0
  385. package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.d.ts +33 -0
  386. package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.js +46 -0
  387. package/lib/meta/technologies/extract-technologies-for-archive.d.ts +33 -0
  388. package/lib/meta/technologies/extract-technologies-for-archive.js +30 -0
  389. package/lib/meta/technologies/match-js-license-comment-signal.d.ts +21 -0
  390. package/lib/meta/technologies/match-js-license-comment-signal.js +23 -0
  391. package/lib/meta/technologies/match-structural-technology-signals.d.ts +26 -0
  392. package/lib/meta/technologies/match-structural-technology-signals.js +48 -0
  393. package/lib/meta/technologies/normalize-wappalyzer-entries.d.ts +25 -0
  394. package/lib/meta/technologies/normalize-wappalyzer-entries.js +35 -0
  395. package/lib/meta/technologies/technology-signal-definitions.d.ts +44 -0
  396. package/lib/meta/technologies/technology-signal-definitions.js +60 -0
  397. package/lib/meta/technologies/types.d.ts +70 -0
  398. package/lib/meta/technologies/types.js +13 -0
  399. package/lib/meta/types.d.ts +1490 -0
  400. package/lib/meta/types.js +65 -0
  401. package/lib/migrate-content-items-alias-of-id.d.ts +52 -0
  402. package/lib/migrate-content-items-alias-of-id.js +69 -0
  403. package/lib/migrate-content-items-dedupe-cap-event-id.d.ts +46 -0
  404. package/lib/migrate-content-items-dedupe-cap-event-id.js +62 -0
  405. package/lib/migrate-content-items-is-metadata-only.d.ts +44 -0
  406. package/lib/migrate-content-items-is-metadata-only.js +60 -0
  407. package/lib/migrate-entity-tables.d.ts +55 -0
  408. package/lib/migrate-entity-tables.js +66 -0
  409. package/lib/migrate-info-created-cwd.d.ts +17 -0
  410. package/lib/migrate-info-created-cwd.js +36 -0
  411. package/lib/migrate-info-main-content-selector.d.ts +16 -0
  412. package/lib/migrate-info-main-content-selector.js +35 -0
  413. package/lib/migrate-info-request-header-names.d.ts +16 -0
  414. package/lib/migrate-info-request-header-names.js +35 -0
  415. package/lib/migrate-info-roots.d.ts +20 -0
  416. package/lib/migrate-info-roots.js +52 -0
  417. package/lib/migrate-inventory-runs-to-list-reconcile-runs.d.ts +34 -0
  418. package/lib/migrate-inventory-runs-to-list-reconcile-runs.js +51 -0
  419. package/lib/migrate-list-reconcile-runs-exclude-skipped.d.ts +25 -0
  420. package/lib/migrate-list-reconcile-runs-exclude-skipped.js +44 -0
  421. package/lib/migrate-list-reconcile-runs-invalid-skipped.d.ts +16 -0
  422. package/lib/migrate-list-reconcile-runs-invalid-skipped.js +35 -0
  423. package/lib/migrate-main-contents-columns.d.ts +26 -0
  424. package/lib/migrate-main-contents-columns.js +61 -0
  425. package/lib/migrate-page-meta-body-hash.d.ts +44 -0
  426. package/lib/migrate-page-meta-body-hash.js +63 -0
  427. package/lib/migrate-page-meta-console-error-count.d.ts +27 -0
  428. package/lib/migrate-page-meta-console-error-count.js +45 -0
  429. package/lib/migrate-page-meta-custom-element-count.d.ts +33 -0
  430. package/lib/migrate-page-meta-custom-element-count.js +51 -0
  431. package/lib/migrate-page-meta-image-scan.d.ts +27 -0
  432. package/lib/migrate-page-meta-image-scan.js +52 -0
  433. package/lib/migrate-page-tags-to-page-technologies.d.ts +39 -0
  434. package/lib/migrate-page-tags-to-page-technologies.js +79 -0
  435. package/lib/migrate-ref-tables.d.ts +25 -0
  436. package/lib/migrate-ref-tables.js +38 -0
  437. package/lib/native-executor/check-native-executor-config.d.ts +17 -0
  438. package/lib/native-executor/check-native-executor-config.js +20 -0
  439. package/lib/page-meta-column-maps.d.ts +32 -0
  440. package/lib/page-meta-column-maps.js +43 -0
  441. package/lib/page.d.ts +546 -0
  442. package/lib/page.js +764 -0
  443. package/lib/peek-archive-lock.d.ts +40 -0
  444. package/lib/peek-archive-lock.js +62 -0
  445. package/lib/populate-entity-tables/collapse-anchor-rows.d.ts +41 -0
  446. package/lib/populate-entity-tables/collapse-anchor-rows.js +87 -0
  447. package/lib/populate-entity-tables/derive-dom-path.d.ts +35 -0
  448. package/lib/populate-entity-tables/derive-dom-path.js +72 -0
  449. package/lib/populate-entity-tables/is-blob-ref-value.d.ts +16 -0
  450. package/lib/populate-entity-tables/is-blob-ref-value.js +19 -0
  451. package/lib/populate-entity-tables/match-images-to-dom-paths.d.ts +66 -0
  452. package/lib/populate-entity-tables/match-images-to-dom-paths.js +96 -0
  453. package/lib/populate-entity-tables/populate-anchor-edges.d.ts +33 -0
  454. package/lib/populate-entity-tables/populate-anchor-edges.js +153 -0
  455. package/lib/populate-entity-tables/populate-content-items.d.ts +40 -0
  456. package/lib/populate-entity-tables/populate-content-items.js +141 -0
  457. package/lib/populate-entity-tables/populate-entities.d.ts +81 -0
  458. package/lib/populate-entity-tables/populate-entities.js +110 -0
  459. package/lib/populate-entity-tables/populate-image-items.d.ts +91 -0
  460. package/lib/populate-entity-tables/populate-image-items.js +226 -0
  461. package/lib/populate-entity-tables/populate-page-meta.d.ts +33 -0
  462. package/lib/populate-entity-tables/populate-page-meta.js +267 -0
  463. package/lib/populate-entity-tables/populate-resource-items.d.ts +22 -0
  464. package/lib/populate-entity-tables/populate-resource-items.js +114 -0
  465. package/lib/populate-entity-tables/populate-resource-ref-edges.d.ts +31 -0
  466. package/lib/populate-entity-tables/populate-resource-ref-edges.js +33 -0
  467. package/lib/populate-entity-tables/resolve-blob-refs.d.ts +31 -0
  468. package/lib/populate-entity-tables/resolve-blob-refs.js +100 -0
  469. package/lib/populate-entity-tables/resolve-content-type-refs.d.ts +22 -0
  470. package/lib/populate-entity-tables/resolve-content-type-refs.js +27 -0
  471. package/lib/populate-entity-tables/resolve-header-sets.d.ts +49 -0
  472. package/lib/populate-entity-tables/resolve-header-sets.js +122 -0
  473. package/lib/populate-entity-tables/resolve-json-refs.d.ts +25 -0
  474. package/lib/populate-entity-tables/resolve-json-refs.js +67 -0
  475. package/lib/populate-entity-tables/resolve-text-refs.d.ts +30 -0
  476. package/lib/populate-entity-tables/resolve-text-refs.js +61 -0
  477. package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.d.ts +21 -0
  478. package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.js +27 -0
  479. package/lib/populate-entity-tables/resolve-url-refs.d.ts +33 -0
  480. package/lib/populate-entity-tables/resolve-url-refs.js +60 -0
  481. package/lib/populate-entity-tables/test-utils/count-rows.d.ts +17 -0
  482. package/lib/populate-entity-tables/test-utils/count-rows.js +20 -0
  483. package/lib/populate-entity-tables/test-utils/seed-content-items.d.ts +25 -0
  484. package/lib/populate-entity-tables/test-utils/seed-content-items.js +42 -0
  485. package/lib/populate-entity-tables/test-utils/setup-entities-db.d.ts +30 -0
  486. package/lib/populate-entity-tables/test-utils/setup-entities-db.js +175 -0
  487. package/lib/populate-entity-tables/types.d.ts +157 -0
  488. package/lib/populate-entity-tables/types.js +12 -0
  489. package/lib/populate-entity-tables/upsert-text-refs.d.ts +38 -0
  490. package/lib/populate-entity-tables/upsert-text-refs.js +78 -0
  491. package/lib/populate-ref-tables/classify-content-type.d.ts +16 -0
  492. package/lib/populate-ref-tables/classify-content-type.js +52 -0
  493. package/lib/populate-ref-tables/compute-content-hash.d.ts +22 -0
  494. package/lib/populate-ref-tables/compute-content-hash.js +26 -0
  495. package/lib/populate-ref-tables/compute-header-flags.d.ts +16 -0
  496. package/lib/populate-ref-tables/compute-header-flags.js +70 -0
  497. package/lib/populate-ref-tables/content-type-rules.d.ts +38 -0
  498. package/lib/populate-ref-tables/content-type-rules.js +133 -0
  499. package/lib/populate-ref-tables/create-header-table-caches.d.ts +25 -0
  500. package/lib/populate-ref-tables/create-header-table-caches.js +49 -0
  501. package/lib/populate-ref-tables/data-uri-url-refs-limit.d.ts +15 -0
  502. package/lib/populate-ref-tables/data-uri-url-refs-limit.js +15 -0
  503. package/lib/populate-ref-tables/decode-data-uri.d.ts +21 -0
  504. package/lib/populate-ref-tables/decode-data-uri.js +126 -0
  505. package/lib/populate-ref-tables/decompose-header-set.d.ts +29 -0
  506. package/lib/populate-ref-tables/decompose-header-set.js +157 -0
  507. package/lib/populate-ref-tables/decompose-url.d.ts +25 -0
  508. package/lib/populate-ref-tables/decompose-url.js +70 -0
  509. package/lib/populate-ref-tables/header-stability.d.ts +19 -0
  510. package/lib/populate-ref-tables/header-stability.js +22 -0
  511. package/lib/populate-ref-tables/header-value-cache-key.d.ts +17 -0
  512. package/lib/populate-ref-tables/header-value-cache-key.js +19 -0
  513. package/lib/populate-ref-tables/normalize-mime.d.ts +24 -0
  514. package/lib/populate-ref-tables/normalize-mime.js +36 -0
  515. package/lib/populate-ref-tables/populate-blob-refs.d.ts +38 -0
  516. package/lib/populate-ref-tables/populate-blob-refs.js +134 -0
  517. package/lib/populate-ref-tables/populate-content-type-refs.d.ts +27 -0
  518. package/lib/populate-ref-tables/populate-content-type-refs.js +70 -0
  519. package/lib/populate-ref-tables/populate-header-tables.d.ts +35 -0
  520. package/lib/populate-ref-tables/populate-header-tables.js +80 -0
  521. package/lib/populate-ref-tables/populate-json-refs.d.ts +29 -0
  522. package/lib/populate-ref-tables/populate-json-refs.js +101 -0
  523. package/lib/populate-ref-tables/populate-refs.d.ts +51 -0
  524. package/lib/populate-ref-tables/populate-refs.js +62 -0
  525. package/lib/populate-ref-tables/populate-text-refs.d.ts +32 -0
  526. package/lib/populate-ref-tables/populate-text-refs.js +133 -0
  527. package/lib/populate-ref-tables/populate-url-refs.d.ts +28 -0
  528. package/lib/populate-ref-tables/populate-url-refs.js +148 -0
  529. package/lib/populate-ref-tables/test-utils/count-rows.d.ts +15 -0
  530. package/lib/populate-ref-tables/test-utils/count-rows.js +17 -0
  531. package/lib/populate-ref-tables/types.d.ts +197 -0
  532. package/lib/populate-ref-tables/types.js +7 -0
  533. package/lib/populate-ref-tables/upsert-one-header-set.d.ts +34 -0
  534. package/lib/populate-ref-tables/upsert-one-header-set.js +208 -0
  535. package/lib/populate-ref-tables/volatile-header-names.d.ts +20 -0
  536. package/lib/populate-ref-tables/volatile-header-names.js +33 -0
  537. package/lib/redirect-table.d.ts +9 -0
  538. package/lib/redirect-table.js +28 -0
  539. package/lib/release-finished-statements.d.ts +42 -0
  540. package/lib/release-finished-statements.js +45 -0
  541. package/lib/resolve-redirect-chain.d.ts +33 -0
  542. package/lib/resolve-redirect-chain.js +27 -0
  543. package/lib/resource.d.ts +46 -0
  544. package/lib/resource.js +62 -0
  545. package/lib/retarget-legacy-fk-tables.d.ts +61 -0
  546. package/lib/retarget-legacy-fk-tables.js +137 -0
  547. package/lib/safe-path.d.ts +9 -0
  548. package/lib/safe-path.js +17 -0
  549. package/lib/scope/build-scope-map.d.ts +21 -0
  550. package/lib/scope/build-scope-map.js +32 -0
  551. package/lib/scope/find-scope-entry.d.ts +25 -0
  552. package/lib/scope/find-scope-entry.js +45 -0
  553. package/lib/template-classification/classify-archive-page-templates.d.ts +42 -0
  554. package/lib/template-classification/classify-archive-page-templates.js +70 -0
  555. package/lib/template-classification/classify-page-templates.d.ts +33 -0
  556. package/lib/template-classification/classify-page-templates.js +60 -0
  557. package/lib/template-classification/collect-page-ids-with-html.d.ts +24 -0
  558. package/lib/template-classification/collect-page-ids-with-html.js +30 -0
  559. package/lib/template-classification/collect-page-stylesheet-urls.d.ts +32 -0
  560. package/lib/template-classification/collect-page-stylesheet-urls.js +132 -0
  561. package/lib/template-classification/create-content-root-hint.d.ts +41 -0
  562. package/lib/template-classification/create-content-root-hint.js +51 -0
  563. package/lib/template-classification/create-page-cluster-factory.d.ts +36 -0
  564. package/lib/template-classification/create-page-cluster-factory.js +52 -0
  565. package/lib/template-classification/load-page-cluster-candidates.d.ts +32 -0
  566. package/lib/template-classification/load-page-cluster-candidates.js +93 -0
  567. package/lib/template-classification/page-cluster/resolve-page-cluster-keys.d.ts +34 -0
  568. package/lib/template-classification/page-cluster/resolve-page-cluster-keys.js +236 -0
  569. package/lib/template-classification/page-cluster/types.d.ts +123 -0
  570. package/lib/template-classification/page-cluster/types.js +1 -0
  571. package/lib/template-classification/types.d.ts +234 -0
  572. package/lib/template-classification/types.js +1 -0
  573. package/lib/test-utils/build-full-schema-test-db.d.ts +34 -0
  574. package/lib/test-utils/build-full-schema-test-db.js +74 -0
  575. package/lib/test-utils/compare-strings.d.ts +12 -0
  576. package/lib/test-utils/compare-strings.js +14 -0
  577. package/lib/test-utils/create-legacy-analysis-tables.d.ts +13 -0
  578. package/lib/test-utils/create-legacy-analysis-tables.js +37 -0
  579. package/lib/test-utils/fk-parent-tables.d.ts +15 -0
  580. package/lib/test-utils/fk-parent-tables.js +19 -0
  581. package/lib/test-utils/seed-content-item.d.ts +35 -0
  582. package/lib/test-utils/seed-content-item.js +42 -0
  583. package/lib/test-utils/setup-legacy-fk-db.d.ts +34 -0
  584. package/lib/test-utils/setup-legacy-fk-db.js +271 -0
  585. package/lib/transfer/attach-source-database.d.ts +31 -0
  586. package/lib/transfer/attach-source-database.js +40 -0
  587. package/lib/transfer/build-split-config.d.ts +30 -0
  588. package/lib/transfer/build-split-config.js +42 -0
  589. package/lib/transfer/clear-replaced-page-rows.d.ts +23 -0
  590. package/lib/transfer/clear-replaced-page-rows.js +61 -0
  591. package/lib/transfer/concat-archives.d.ts +55 -0
  592. package/lib/transfer/concat-archives.js +95 -0
  593. package/lib/transfer/concat-source-transfer-phases.d.ts +10 -0
  594. package/lib/transfer/concat-source-transfer-phases.js +17 -0
  595. package/lib/transfer/content-item-rank-sql.d.ts +21 -0
  596. package/lib/transfer/content-item-rank-sql.js +23 -0
  597. package/lib/transfer/copy-anchor-edges.d.ts +28 -0
  598. package/lib/transfer/copy-anchor-edges.js +42 -0
  599. package/lib/transfer/copy-dictionaries-for-concat.d.ts +39 -0
  600. package/lib/transfer/copy-dictionaries-for-concat.js +81 -0
  601. package/lib/transfer/copy-dictionaries-for-split.d.ts +29 -0
  602. package/lib/transfer/copy-dictionaries-for-split.js +224 -0
  603. package/lib/transfer/copy-header-set-children.d.ts +27 -0
  604. package/lib/transfer/copy-header-set-children.js +51 -0
  605. package/lib/transfer/copy-image-items.d.ts +21 -0
  606. package/lib/transfer/copy-image-items.js +62 -0
  607. package/lib/transfer/copy-inventory-source-lists.d.ts +30 -0
  608. package/lib/transfer/copy-inventory-source-lists.js +61 -0
  609. package/lib/transfer/copy-journal-tables.d.ts +30 -0
  610. package/lib/transfer/copy-journal-tables.js +56 -0
  611. package/lib/transfer/copy-page-console-logs.d.ts +16 -0
  612. package/lib/transfer/copy-page-console-logs.js +27 -0
  613. package/lib/transfer/copy-page-html-blobs.d.ts +31 -0
  614. package/lib/transfer/copy-page-html-blobs.js +43 -0
  615. package/lib/transfer/copy-page-meta.d.ts +27 -0
  616. package/lib/transfer/copy-page-meta.js +68 -0
  617. package/lib/transfer/copy-page-template-clusters.d.ts +31 -0
  618. package/lib/transfer/copy-page-template-clusters.js +56 -0
  619. package/lib/transfer/copy-page-template-labels.d.ts +34 -0
  620. package/lib/transfer/copy-page-template-labels.js +88 -0
  621. package/lib/transfer/copy-resource-items.d.ts +29 -0
  622. package/lib/transfer/copy-resource-items.js +89 -0
  623. package/lib/transfer/copy-resource-ref-edges.d.ts +20 -0
  624. package/lib/transfer/copy-resource-ref-edges.js +33 -0
  625. package/lib/transfer/copy-simple-page-scoped-tables.d.ts +15 -0
  626. package/lib/transfer/copy-simple-page-scoped-tables.js +34 -0
  627. package/lib/transfer/count-external-pages-in-scope.d.ts +26 -0
  628. package/lib/transfer/count-external-pages-in-scope.js +40 -0
  629. package/lib/transfer/create-transfer-temp-tables.d.ts +30 -0
  630. package/lib/transfer/create-transfer-temp-tables.js +59 -0
  631. package/lib/transfer/dictionary-copy-specs.d.ts +32 -0
  632. package/lib/transfer/dictionary-copy-specs.js +28 -0
  633. package/lib/transfer/drop-transfer-temp-tables.d.ts +9 -0
  634. package/lib/transfer/drop-transfer-temp-tables.js +18 -0
  635. package/lib/transfer/flatten-redirect-chains.d.ts +44 -0
  636. package/lib/transfer/flatten-redirect-chains.js +74 -0
  637. package/lib/transfer/insert-content-items.d.ts +35 -0
  638. package/lib/transfer/insert-content-items.js +116 -0
  639. package/lib/transfer/list-transfer-columns.d.ts +35 -0
  640. package/lib/transfer/list-transfer-columns.js +45 -0
  641. package/lib/transfer/mark-in-scope-content-items.d.ts +31 -0
  642. package/lib/transfer/mark-in-scope-content-items.js +54 -0
  643. package/lib/transfer/merge-archive-configs.d.ts +51 -0
  644. package/lib/transfer/merge-archive-configs.js +85 -0
  645. package/lib/transfer/plan-content-items-for-concat.d.ts +33 -0
  646. package/lib/transfer/plan-content-items-for-concat.js +51 -0
  647. package/lib/transfer/plan-content-items-for-split.d.ts +42 -0
  648. package/lib/transfer/plan-content-items-for-split.js +83 -0
  649. package/lib/transfer/plan-resource-items.d.ts +40 -0
  650. package/lib/transfer/plan-resource-items.js +69 -0
  651. package/lib/transfer/quote-transfer-identifier.d.ts +18 -0
  652. package/lib/transfer/quote-transfer-identifier.js +20 -0
  653. package/lib/transfer/reclassify-resource-externality.d.ts +37 -0
  654. package/lib/transfer/reclassify-resource-externality.js +66 -0
  655. package/lib/transfer/remap-redirect-dest-ids.d.ts +34 -0
  656. package/lib/transfer/remap-redirect-dest-ids.js +51 -0
  657. package/lib/transfer/replace-content-items.d.ts +39 -0
  658. package/lib/transfer/replace-content-items.js +83 -0
  659. package/lib/transfer/simple-page-scoped-tables.d.ts +25 -0
  660. package/lib/transfer/simple-page-scoped-tables.js +38 -0
  661. package/lib/transfer/split-archive.d.ts +55 -0
  662. package/lib/transfer/split-archive.js +98 -0
  663. package/lib/transfer/split-source-transfer-phases.d.ts +11 -0
  664. package/lib/transfer/split-source-transfer-phases.js +18 -0
  665. package/lib/transfer/transfer-action.d.ts +34 -0
  666. package/lib/transfer/transfer-action.js +32 -0
  667. package/lib/transfer/transfer-archive-rows.d.ts +65 -0
  668. package/lib/transfer/transfer-archive-rows.js +156 -0
  669. package/lib/transfer/transfer-dictionary-tables.d.ts +12 -0
  670. package/lib/transfer/transfer-dictionary-tables.js +22 -0
  671. package/lib/transfer/transfer-post-phases.d.ts +10 -0
  672. package/lib/transfer/transfer-post-phases.js +13 -0
  673. package/lib/transfer/transfer-source-alias.d.ts +9 -0
  674. package/lib/transfer/transfer-source-alias.js +9 -0
  675. package/lib/transfer/types.d.ts +111 -0
  676. package/lib/transfer/types.js +35 -0
  677. package/lib/types.d.ts +772 -0
  678. package/lib/types.js +1 -0
  679. package/lib/url-alias/compute-tier-a-alias-key.d.ts +44 -0
  680. package/lib/url-alias/compute-tier-a-alias-key.js +47 -0
  681. package/lib/url-alias/compute-tier-b-alias-key.d.ts +23 -0
  682. package/lib/url-alias/compute-tier-b-alias-key.js +26 -0
  683. package/lib/url-alias/group-alias-candidates.d.ts +27 -0
  684. package/lib/url-alias/group-alias-candidates.js +29 -0
  685. package/lib/url-alias/types.d.ts +12 -0
  686. package/lib/url-alias/types.js +1 -0
  687. package/lib/url-pattern/compute-shape-key.d.ts +58 -0
  688. package/lib/url-pattern/compute-shape-key.js +129 -0
  689. package/lib/url-pattern/decompose-url.d.ts +33 -0
  690. package/lib/url-pattern/decompose-url.js +79 -0
  691. package/lib/url-pattern/reconstruct-url.d.ts +15 -0
  692. package/lib/url-pattern/reconstruct-url.js +33 -0
  693. package/lib/url-sort/compute-url-sort-ranks.d.ts +30 -0
  694. package/lib/url-sort/compute-url-sort-ranks.js +33 -0
  695. package/lib/utils/array/each-splitted.d.ts +10 -0
  696. package/lib/utils/array/each-splitted.js +14 -0
  697. package/lib/utils/debug.d.ts +8 -0
  698. package/lib/utils/debug.js +8 -0
  699. package/lib/utils/error/emit-error-with-retry.d.ts +40 -0
  700. package/lib/utils/error/emit-error-with-retry.js +44 -0
  701. package/lib/utils/error/emit-error.d.ts +39 -0
  702. package/lib/utils/error/emit-error.js +41 -0
  703. package/lib/utils/object/parse-response-headers.d.ts +12 -0
  704. package/lib/utils/object/parse-response-headers.js +26 -0
  705. package/lib/utils/object/redact-request-headers.d.ts +15 -0
  706. package/lib/utils/object/redact-request-headers.js +27 -0
  707. package/lib/utils/types/types.d.ts +383 -0
  708. package/lib/utils/types/types.js +1 -0
  709. package/lib/verify-migration/capture-rejection.d.ts +24 -0
  710. package/lib/verify-migration/capture-rejection.js +31 -0
  711. package/lib/verify-migration/check-anchor-edges-count.d.ts +34 -0
  712. package/lib/verify-migration/check-anchor-edges-count.js +72 -0
  713. package/lib/verify-migration/check-anchor-edges-sum.d.ts +13 -0
  714. package/lib/verify-migration/check-anchor-edges-sum.js +27 -0
  715. package/lib/verify-migration/check-content-items-count.d.ts +16 -0
  716. package/lib/verify-migration/check-content-items-count.js +30 -0
  717. package/lib/verify-migration/check-content-type-preservation.d.ts +22 -0
  718. package/lib/verify-migration/check-content-type-preservation.js +40 -0
  719. package/lib/verify-migration/check-foreign-key-integrity.d.ts +31 -0
  720. package/lib/verify-migration/check-foreign-key-integrity.js +47 -0
  721. package/lib/verify-migration/check-image-items-count.d.ts +12 -0
  722. package/lib/verify-migration/check-image-items-count.js +26 -0
  723. package/lib/verify-migration/check-page-meta-count.d.ts +15 -0
  724. package/lib/verify-migration/check-page-meta-count.js +31 -0
  725. package/lib/verify-migration/check-reader-parity.d.ts +23 -0
  726. package/lib/verify-migration/check-reader-parity.js +182 -0
  727. package/lib/verify-migration/check-resource-items-count.d.ts +17 -0
  728. package/lib/verify-migration/check-resource-items-count.js +33 -0
  729. package/lib/verify-migration/check-url-round-trip.d.ts +43 -0
  730. package/lib/verify-migration/check-url-round-trip.js +112 -0
  731. package/lib/verify-migration/types.d.ts +70 -0
  732. package/lib/verify-migration/types.js +63 -0
  733. package/lib/verify-migration/verify-migration.d.ts +41 -0
  734. package/lib/verify-migration/verify-migration.js +120 -0
  735. package/lib/with-bulk-write-pragmas.d.ts +51 -0
  736. package/lib/with-bulk-write-pragmas.js +132 -0
  737. package/lib/write-queue.d.ts +32 -0
  738. package/lib/write-queue.js +55 -0
  739. package/package.json +350 -3
@@ -0,0 +1,24 @@
1
+ import { eachSplitted } from '../../utils/array/each-splitted.js';
2
+ /**
3
+ * Return the subset of `urls` that already exist as `resource_items` rows.
4
+ * See `getExistingPageUrls` — same chunking strategy.
5
+ * @param knex - Knex query builder connected to the archive DB.
6
+ * @param urls - URL strings to probe.
7
+ * @returns URLs found among `resource_items`.
8
+ */
9
+ export async function getExistingResourceUrls(knex, urls) {
10
+ if (urls.length === 0) {
11
+ return [];
12
+ }
13
+ const found = [];
14
+ await eachSplitted([...urls], 500, async (chunk) => {
15
+ const rows = await knex('resource_items')
16
+ .join('url_refs', 'url_refs.id', 'resource_items.url_id')
17
+ .select('url_refs.url as url')
18
+ .whereIn('url_refs.url', chunk);
19
+ for (const row of rows) {
20
+ found.push(row.url);
21
+ }
22
+ });
23
+ return found;
24
+ }
@@ -0,0 +1,13 @@
1
+ import type { DB_Resource } from '../../types.js';
2
+ import type { Knex } from 'knex';
3
+ /**
4
+ * Retrieves a single sub-resource by its URL.
5
+ *
6
+ * Accepts multiple URL candidates because the stored key is the resource's
7
+ * `href` while callers may only know the hash-stripped form; the first match
8
+ * wins.
9
+ * @param knex - Knex query builder connected to the archive DB.
10
+ * @param urls - URL candidates to match against `url_refs.url`.
11
+ * @returns The reconstructed {@link DB_Resource} row, or `null` if none match.
12
+ */
13
+ export declare function getResourceByUrl(knex: Knex, urls: readonly string[]): Promise<DB_Resource | null>;
@@ -0,0 +1,22 @@
1
+ import { buildResourceQuery } from './build-resource-query.js';
2
+ import { reconstructResourceRows } from './reconstruct-resource-rows.js';
3
+ /**
4
+ * Retrieves a single sub-resource by its URL.
5
+ *
6
+ * Accepts multiple URL candidates because the stored key is the resource's
7
+ * `href` while callers may only know the hash-stripped form; the first match
8
+ * wins.
9
+ * @param knex - Knex query builder connected to the archive DB.
10
+ * @param urls - URL candidates to match against `url_refs.url`.
11
+ * @returns The reconstructed {@link DB_Resource} row, or `null` if none match.
12
+ */
13
+ export async function getResourceByUrl(knex, urls) {
14
+ const row = await buildResourceQuery(knex)
15
+ .whereIn('ur.url', [...urls])
16
+ .first();
17
+ if (!row) {
18
+ return null;
19
+ }
20
+ const [reconstructed] = await reconstructResourceRows(knex, [row]);
21
+ return reconstructed ?? null;
22
+ }
@@ -0,0 +1,17 @@
1
+ import type { Knex } from 'knex';
2
+ /**
3
+ * Retrieves a flat list of all resource URLs from the `resource_items`
4
+ * table. URL text is normalised into `url_refs`, so the read joins the
5
+ * two tables and returns the resolved strings.
6
+ *
7
+ * Read in `resource_items.id`-keyset chunks rather than a single SELECT
8
+ * (issue #294) via `keysetPaginateById` — see that function's JSDoc for
9
+ * why (progress observability on a resource-heavy archive, not memory
10
+ * bounding).
11
+ * @param knex - Knex query builder connected to the archive DB.
12
+ * @param onProgress - Called after each chunk with the `resource_items.id`
13
+ * scanned up to so far and the max id. Omit for no reporting (the
14
+ * default; e.g. tests).
15
+ * @returns An array of resource URL strings.
16
+ */
17
+ export declare function getResourceUrlList(knex: Knex, onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<string[]>;
@@ -0,0 +1,26 @@
1
+ import { keysetPaginateById } from '../_shared/keyset-paginate-by-id.js';
2
+ /** Rows read per `resource_items.id` keyset chunk. */
3
+ const READ_CHUNK_SIZE = 2000;
4
+ /**
5
+ * Retrieves a flat list of all resource URLs from the `resource_items`
6
+ * table. URL text is normalised into `url_refs`, so the read joins the
7
+ * two tables and returns the resolved strings.
8
+ *
9
+ * Read in `resource_items.id`-keyset chunks rather than a single SELECT
10
+ * (issue #294) via `keysetPaginateById` — see that function's JSDoc for
11
+ * why (progress observability on a resource-heavy archive, not memory
12
+ * bounding).
13
+ * @param knex - Knex query builder connected to the archive DB.
14
+ * @param onProgress - Called after each chunk with the `resource_items.id`
15
+ * scanned up to so far and the max id. Omit for no reporting (the
16
+ * default; e.g. tests).
17
+ * @returns An array of resource URL strings.
18
+ */
19
+ export async function getResourceUrlList(knex, onProgress) {
20
+ return keysetPaginateById(knex, 'resource_items', (lastId) => knex('resource_items')
21
+ .join('url_refs', 'url_refs.id', 'resource_items.url_id')
22
+ .where('resource_items.id', '>', lastId)
23
+ .orderBy('resource_items.id', 'asc')
24
+ .limit(READ_CHUNK_SIZE)
25
+ .select('resource_items.id as id', 'url_refs.url as url'), (row) => row.url, onProgress);
26
+ }
@@ -0,0 +1,8 @@
1
+ import type { DB_Resource } from '../../types.js';
2
+ import type { Knex } from 'knex';
3
+ /**
4
+ * Retrieves all sub-resources.
5
+ * @param knex - Knex query builder connected to the archive DB.
6
+ * @returns An array of reconstructed {@link DB_Resource} rows.
7
+ */
8
+ export declare function getResources(knex: Knex): Promise<DB_Resource[]>;
@@ -0,0 +1,11 @@
1
+ import { buildResourceQuery } from './build-resource-query.js';
2
+ import { reconstructResourceRows } from './reconstruct-resource-rows.js';
3
+ /**
4
+ * Retrieves all sub-resources.
5
+ * @param knex - Knex query builder connected to the archive DB.
6
+ * @returns An array of reconstructed {@link DB_Resource} rows.
7
+ */
8
+ export async function getResources(knex) {
9
+ const rows = await buildResourceQuery(knex);
10
+ return reconstructResourceRows(knex, rows);
11
+ }
@@ -0,0 +1,24 @@
1
+ import type { WriteRefCaches } from '../_shared/types.js';
2
+ import type { Knex } from 'knex';
3
+ /**
4
+ * Pre-insert inventory non-HTML URLs into `resource_items` as placeholder
5
+ * rows with `source = 'inventory-seed'` and all metadata columns NULL —
6
+ * the non-HTML counterpart of `insertInventorySeeds`. Used by
7
+ * `CrawlerOrchestrator.inventory` so the ingestion phase commits all of
8
+ * its non-HTML URLs in one chunked round-trip per 500 instead of N
9
+ * sequential `insertResource` awaits — a per-URL loop would spend
10
+ * minutes inside the `.bak`-protected window on a 50k-URL inventory
11
+ * list, where the bulk path finishes in seconds.
12
+ *
13
+ * Idempotent: both the `url_refs` and `resource_items` inserts are
14
+ * `ON CONFLICT ... IGNORE`, leaving existing rows untouched (the
15
+ * orchestrator's `getExistingResourceUrls` filter is what keeps a
16
+ * crawled-lineage `resource_items` row from being downgraded to the
17
+ * inventory label here).
18
+ *
19
+ * Chunked at 500 to stay well under SQLite's `SQLITE_MAX_VARIABLE_NUMBER`.
20
+ * @param knex - Knex query builder connected to the archive DB.
21
+ * @param caches
22
+ * @param urls - URL strings (already in `withoutHashAndAuth` form).
23
+ */
24
+ export declare function insertInventoryResources(knex: Knex, caches: WriteRefCaches, urls: readonly string[]): Promise<void>;
@@ -0,0 +1,64 @@
1
+ import { resolveUrlRefs } from '../../populate-entity-tables/resolve-url-refs.js';
2
+ import { decomposeUrl } from '../../populate-ref-tables/decompose-url.js';
3
+ import { eachSplitted } from '../../utils/array/each-splitted.js';
4
+ /**
5
+ * Pre-insert inventory non-HTML URLs into `resource_items` as placeholder
6
+ * rows with `source = 'inventory-seed'` and all metadata columns NULL —
7
+ * the non-HTML counterpart of `insertInventorySeeds`. Used by
8
+ * `CrawlerOrchestrator.inventory` so the ingestion phase commits all of
9
+ * its non-HTML URLs in one chunked round-trip per 500 instead of N
10
+ * sequential `insertResource` awaits — a per-URL loop would spend
11
+ * minutes inside the `.bak`-protected window on a 50k-URL inventory
12
+ * list, where the bulk path finishes in seconds.
13
+ *
14
+ * Idempotent: both the `url_refs` and `resource_items` inserts are
15
+ * `ON CONFLICT ... IGNORE`, leaving existing rows untouched (the
16
+ * orchestrator's `getExistingResourceUrls` filter is what keeps a
17
+ * crawled-lineage `resource_items` row from being downgraded to the
18
+ * inventory label here).
19
+ *
20
+ * Chunked at 500 to stay well under SQLite's `SQLITE_MAX_VARIABLE_NUMBER`.
21
+ * @param knex - Knex query builder connected to the archive DB.
22
+ * @param caches
23
+ * @param urls - URL strings (already in `withoutHashAndAuth` form).
24
+ */
25
+ export async function insertInventoryResources(knex, caches, urls) {
26
+ if (urls.length === 0) {
27
+ return;
28
+ }
29
+ await eachSplitted([...urls], 500, async (chunk) => {
30
+ await knex('url_refs')
31
+ .insert(chunk.map((url) => ({ url, ...decomposeUrl(url) })))
32
+ .onConflict('url')
33
+ .ignore();
34
+ const urlIds = await resolveUrlRefs(knex, chunk);
35
+ const rows = chunk.map((url) => {
36
+ const urlId = urlIds.get(url);
37
+ if (urlId === undefined) {
38
+ throw new Error(`insertInventoryResources: url_refs.id not resolved for ${url}`);
39
+ }
40
+ caches.urlIds.set(url, urlId);
41
+ return {
42
+ url_id: urlId,
43
+ is_external: 0,
44
+ status: null,
45
+ status_text: null,
46
+ content_type_id: null,
47
+ content_length: null,
48
+ header_set_id: null,
49
+ compress: 0,
50
+ cdn: 0,
51
+ source: 'inventory-seed',
52
+ };
53
+ });
54
+ await knex('resource_items').insert(rows).onConflict('url_id').ignore();
55
+ const inserted = (await knex
56
+ .select('ri.id', 'ur.url')
57
+ .from('resource_items as ri')
58
+ .join('url_refs as ur', 'ur.id', 'ri.url_id')
59
+ .whereIn('ur.url', chunk));
60
+ for (const row of inserted) {
61
+ caches.resourceIds.set(row.url, row.id);
62
+ }
63
+ });
64
+ }
@@ -0,0 +1,15 @@
1
+ import type { WriteRefCaches } from '../_shared/types.js';
2
+ import type { Knex } from 'knex';
3
+ /**
4
+ * Inserts a referrer relationship between a resource and a page into the
5
+ * `resource_ref_edges` table. Silently skips if the resource is not found.
6
+ *
7
+ * A repeat observation of the same `(resource, page)` pair is ignored
8
+ * (`ON CONFLICT ... IGNORE`), leaving `count` at its DEFAULT `1` — one
9
+ * row per pair, matching the legacy uniqueness contract.
10
+ * @param knex - Knex query builder connected to the archive DB.
11
+ * @param caches - The connection's write-side id caches.
12
+ * @param src - The URL of the resource.
13
+ * @param pageUrl - The URL of the page that references the resource.
14
+ */
15
+ export declare function insertResourceReferrers(knex: Knex, caches: WriteRefCaches, src: string, pageUrl: string): Promise<void>;
@@ -0,0 +1,54 @@
1
+ import { resolveContentItemId } from '../_shared/resolve-content-item-id.js';
2
+ /**
3
+ * Inserts a referrer relationship between a resource and a page into the
4
+ * `resource_ref_edges` table. Silently skips if the resource is not found.
5
+ *
6
+ * A repeat observation of the same `(resource, page)` pair is ignored
7
+ * (`ON CONFLICT ... IGNORE`), leaving `count` at its DEFAULT `1` — one
8
+ * row per pair, matching the legacy uniqueness contract.
9
+ * @param knex - Knex query builder connected to the archive DB.
10
+ * @param caches - The connection's write-side id caches.
11
+ * @param src - The URL of the resource.
12
+ * @param pageUrl - The URL of the page that references the resource.
13
+ */
14
+ export async function insertResourceReferrers(knex, caches, src, pageUrl) {
15
+ const resourceId = await findResourceIdByUrl(knex, caches, src);
16
+ if (resourceId === undefined) {
17
+ // Ignore when the resource is not found
18
+ return;
19
+ }
20
+ const pageId = await resolveContentItemId(knex, caches, pageUrl);
21
+ await knex('resource_ref_edges')
22
+ .insert({
23
+ resource_id: resourceId,
24
+ page_id: pageId,
25
+ })
26
+ .onConflict(['resource_id', 'page_id'])
27
+ .ignore();
28
+ }
29
+ /**
30
+ * Cache-first lookup of `resource_items.id` by the resource URL. The
31
+ * cache only ever gains entries for URLs that exist in the table
32
+ * (resource identities are never deleted during a crawl), so a hit is
33
+ * always valid; a miss falls through to a `url_refs` join.
34
+ * @param knex - Knex query builder.
35
+ * @param caches - The connection's write-side id caches.
36
+ * @param url - The resource URL, verbatim as stored.
37
+ * @returns The resource id, or `undefined` when the URL is unknown.
38
+ */
39
+ async function findResourceIdByUrl(knex, caches, url) {
40
+ const cached = caches.resourceIds.get(url);
41
+ if (cached !== undefined) {
42
+ return cached;
43
+ }
44
+ const [row] = (await knex
45
+ .select('ri.id')
46
+ .from('resource_items as ri')
47
+ .join('url_refs as ur', 'ur.id', 'ri.url_id')
48
+ .where('ur.url', url));
49
+ if (row === undefined) {
50
+ return undefined;
51
+ }
52
+ caches.resourceIds.set(url, row.id);
53
+ return row.id;
54
+ }
@@ -0,0 +1,34 @@
1
+ import type { PageSource } from '../../types.js';
2
+ import type { Resource } from '../../utils/types/types.js';
3
+ import type { WriteRefCaches } from '../_shared/types.js';
4
+ import type { Knex } from 'knex';
5
+ /**
6
+ * Inserts a sub-resource into the `resource_items` table, interning its
7
+ * URL / content type / response headers into the ref tables first. A
8
+ * resource's identity URL routes through {@link resolveUrlOrBlob} like an
9
+ * image `src` — a large inline `data:` URI (e.g. a CSS
10
+ * `background-image` sub-resource) lands in `blob_refs` instead of
11
+ * `url_refs`.
12
+ *
13
+ * A conflict on the identity column (`url_id` or `url_blob_id`) is an
14
+ * **upgrade-only merge**, not first-write-wins: the existing row's
15
+ * response fields (`status` / `status_text` / `content_type_id` /
16
+ * `content_length` / `header_set_id` / `compress` / `cdn`) are replaced
17
+ * only when the existing row currently has `status IS NULL` (beholder's
18
+ * signal for "the request never got a response" — see the `Resource`
19
+ * type) AND the new observation has a non-null `status`. A resource that
20
+ * failed on first sight and later succeeds (the exact case
21
+ * `--retry-failed` re-renders a page for) is therefore healed instead of
22
+ * staying frozen at its failure; a resource that already has a real
23
+ * response never gets overwritten by a later transient failure, and two
24
+ * failures in a row are a no-op — neither carries new information.
25
+ *
26
+ * The `source` provenance label is written ONLY on insert; a conflict
27
+ * (upgrade or no-op alike) leaves an existing row's source untouched
28
+ * (this is what makes a second `crawl --inventory` non-destructive).
29
+ * @param knex - Knex query builder connected to the archive DB.
30
+ * @param caches - The connection's write-side id caches.
31
+ * @param resource - The resource data to insert.
32
+ * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
33
+ */
34
+ export declare function insertResource(knex: Knex, caches: WriteRefCaches, resource: Resource, source?: PageSource): Promise<void>;
@@ -0,0 +1,73 @@
1
+ import { normalizeContentType } from '../../content-type/normalize-content-type.js';
2
+ import { resolveUrlOrBlob } from '../_shared/resolve-url-or-blob.js';
3
+ import { upsertContentTypeRef } from '../_shared/upsert-content-type-ref.js';
4
+ import { upsertResponseHeaders } from '../_shared/upsert-response-headers.js';
5
+ /**
6
+ * Inserts a sub-resource into the `resource_items` table, interning its
7
+ * URL / content type / response headers into the ref tables first. A
8
+ * resource's identity URL routes through {@link resolveUrlOrBlob} like an
9
+ * image `src` — a large inline `data:` URI (e.g. a CSS
10
+ * `background-image` sub-resource) lands in `blob_refs` instead of
11
+ * `url_refs`.
12
+ *
13
+ * A conflict on the identity column (`url_id` or `url_blob_id`) is an
14
+ * **upgrade-only merge**, not first-write-wins: the existing row's
15
+ * response fields (`status` / `status_text` / `content_type_id` /
16
+ * `content_length` / `header_set_id` / `compress` / `cdn`) are replaced
17
+ * only when the existing row currently has `status IS NULL` (beholder's
18
+ * signal for "the request never got a response" — see the `Resource`
19
+ * type) AND the new observation has a non-null `status`. A resource that
20
+ * failed on first sight and later succeeds (the exact case
21
+ * `--retry-failed` re-renders a page for) is therefore healed instead of
22
+ * staying frozen at its failure; a resource that already has a real
23
+ * response never gets overwritten by a later transient failure, and two
24
+ * failures in a row are a no-op — neither carries new information.
25
+ *
26
+ * The `source` provenance label is written ONLY on insert; a conflict
27
+ * (upgrade or no-op alike) leaves an existing row's source untouched
28
+ * (this is what makes a second `crawl --inventory` non-destructive).
29
+ * @param knex - Knex query builder connected to the archive DB.
30
+ * @param caches - The connection's write-side id caches.
31
+ * @param resource - The resource data to insert.
32
+ * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
33
+ */
34
+ export async function insertResource(knex, caches, resource, source) {
35
+ const slot = await resolveUrlOrBlob(knex, caches, resource.url.href);
36
+ // Canonicalize like the page path (see `insertPage`) so resource
37
+ // content-type filters / dedupe keys are case- and whitespace-stable.
38
+ const contentType = normalizeContentType(resource.contentType);
39
+ const contentTypeId = contentType == null || contentType === ''
40
+ ? null
41
+ : await upsertContentTypeRef(knex, caches, contentType);
42
+ const headerSetId = await upsertResponseHeaders(knex, caches, resource.headers);
43
+ const identityColumn = slot.blob === null ? 'url_id' : 'url_blob_id';
44
+ const identityValue = slot.blob === null ? slot.url : slot.blob;
45
+ const responseFields = {
46
+ is_external: resource.isExternal ? 1 : 0,
47
+ status: resource.status,
48
+ status_text: resource.statusText,
49
+ content_type_id: contentTypeId,
50
+ content_length: resource.contentLength,
51
+ header_set_id: headerSetId,
52
+ compress: resource.compress || 0,
53
+ cdn: resource.cdn || 0,
54
+ };
55
+ const existing = await knex('resource_items')
56
+ .select('id', 'status')
57
+ .where(identityColumn, identityValue)
58
+ .first();
59
+ if (existing === undefined) {
60
+ await knex('resource_items').insert({
61
+ url_id: slot.url,
62
+ url_blob_id: slot.blob,
63
+ ...responseFields,
64
+ ...(source === undefined ? {} : { source }),
65
+ });
66
+ return;
67
+ }
68
+ if (existing.status === null && resource.status !== null) {
69
+ await knex('resource_items')
70
+ .where('id', existing.id)
71
+ .update(responseFields);
72
+ }
73
+ }
@@ -0,0 +1,26 @@
1
+ import type { DB_Resource } from '../../types.js';
2
+ import type { Knex } from 'knex';
3
+ /**
4
+ * Raw row shape produced by {@link ../resources/build-resource-query.js}
5
+ * before `responseHeaders` reconstruction.
6
+ */
7
+ interface RawResourceRow extends Omit<DB_Resource, 'responseHeaders'> {
8
+ /** `resource_items.header_set_id`, or null when no headers were recorded. */
9
+ headerSetId: number | null;
10
+ }
11
+ /**
12
+ * Reconstructs `responseHeaders` (JSON string) for a batch of raw resource
13
+ * rows. Headers load through
14
+ * {@link ../_shared/load-response-headers-by-set-ids.js} (chunked batch
15
+ * lookup, never N+1) — the same reconstruction the page read path uses,
16
+ * so pages and resources always agree on how a given `header_set_id`
17
+ * reads back.
18
+ * @param knex - Knex query builder connected to the archive DB.
19
+ * @param rows - Raw rows from {@link ../resources/build-resource-query.js}.
20
+ * @returns Fully reconstructed `DB_Resource` rows, in the same order as `rows`.
21
+ * @example
22
+ * const raw = await buildResourceQuery(knex).whereIn('ur.url', urls);
23
+ * const resources = await reconstructResourceRows(knex, raw);
24
+ */
25
+ export declare function reconstructResourceRows(knex: Knex, rows: readonly RawResourceRow[]): Promise<DB_Resource[]>;
26
+ export {};
@@ -0,0 +1,30 @@
1
+ import { loadResponseHeadersBySetIds } from '../_shared/load-response-headers-by-set-ids.js';
2
+ /**
3
+ * Reconstructs `responseHeaders` (JSON string) for a batch of raw resource
4
+ * rows. Headers load through
5
+ * {@link ../_shared/load-response-headers-by-set-ids.js} (chunked batch
6
+ * lookup, never N+1) — the same reconstruction the page read path uses,
7
+ * so pages and resources always agree on how a given `header_set_id`
8
+ * reads back.
9
+ * @param knex - Knex query builder connected to the archive DB.
10
+ * @param rows - Raw rows from {@link ../resources/build-resource-query.js}.
11
+ * @returns Fully reconstructed `DB_Resource` rows, in the same order as `rows`.
12
+ * @example
13
+ * const raw = await buildResourceQuery(knex).whereIn('ur.url', urls);
14
+ * const resources = await reconstructResourceRows(knex, raw);
15
+ */
16
+ export async function reconstructResourceRows(knex, rows) {
17
+ const headerSetIds = [
18
+ ...new Set(rows.map((r) => r.headerSetId).filter((id) => id != null)),
19
+ ];
20
+ const headersBySetId = await loadResponseHeadersBySetIds(knex, headerSetIds);
21
+ return rows.map((row) => {
22
+ const { headerSetId, ...rest } = row;
23
+ return {
24
+ ...rest,
25
+ responseHeaders: headerSetId == null
26
+ ? null
27
+ : JSON.stringify(headersBySetId.get(headerSetId) ?? {}),
28
+ };
29
+ });
30
+ }
@@ -0,0 +1,60 @@
1
+ import type { AssignTemplateLabelsParams, TemplateLabel } from './types.js';
2
+ /**
3
+ * Assigns every cluster of a fresh template classification its
4
+ * {@link TemplateLabel}, carrying labels forward from the previous
5
+ * classification wherever the same template survived the re-run.
6
+ *
7
+ * ## Inheritance
8
+ *
9
+ * A new cluster inherits an old cluster's label when the two share a
10
+ * **mutual majority** of pages: the overlap is more than half of the new
11
+ * cluster *and* more than half of the old one. Mutual majority means at
12
+ * most one new cluster can claim a given old label, and a cluster that
13
+ * merely absorbed a few pages of a much larger old template does not steal
14
+ * its name. A split keeps the old label on the half that holds most of the
15
+ * old pages; the other half is named afresh.
16
+ *
17
+ * The old cluster's size counts only pages that are classified in the new
18
+ * run too. Pages the crawl has since dropped (`--recrawl` turned them into
19
+ * redirects, errors, …) are not evidence that the template changed, so
20
+ * they must not dilute the old half of the majority — otherwise an old
21
+ * 7-page cluster whose 4 missing pages left 3 unchanged ones would fail
22
+ * `3 > 7/2` and be renamed for no reason.
23
+ *
24
+ * Page ids are the identity used for overlap (not URLs) because they are
25
+ * what `page_templates` stores and they survive `--append` / `--recrawl`
26
+ * without renumbering.
27
+ *
28
+ * ## Fresh labels
29
+ *
30
+ * Clusters left without an inherited label are numbered in page-count
31
+ * order (largest first, template key as a tie-break) so a first run reads
32
+ * naturally: `A` is the section's biggest template. Each takes the next
33
+ * ordinal **after every ordinal in `previousLabels` for its section**, so a
34
+ * letter is never re-issued to a different template — which would silently
35
+ * change what "events template B" means between two people's archives.
36
+ * That guarantee is only as good as `previousLabels`: `replacePageTemplates`
37
+ * keeps the rows of clusters that disappeared (retired labels) precisely so
38
+ * they stay in this set across any number of later runs.
39
+ *
40
+ * ## Section
41
+ *
42
+ * A cluster's section is the first path segment every member URL shares
43
+ * (`events` for `/events/...`). Members spread over several segments, or
44
+ * all at the site root, yield `null`: such a cluster is numbered in the
45
+ * site-wide sequence. An inherited label keeps its old section even if the
46
+ * cluster's members have since drifted — stability of the name wins over
47
+ * keeping the section literally accurate.
48
+ * @param params - See {@link AssignTemplateLabelsParams}.
49
+ * @returns Template key → label, one entry per cluster in `params.clusters`.
50
+ * @example
51
+ * ```ts
52
+ * const labels = assignTemplateLabels({
53
+ * clusters: new Map([['["path:events","cluster:0"]', { pageIds: [1, 2], urls: ['https://example.com/events/a', 'https://example.com/events/b'] }]]),
54
+ * previousMembership: new Map(),
55
+ * previousLabels: new Map(),
56
+ * });
57
+ * labels.get('["path:events","cluster:0"]'); // { section: 'events', ordinal: 1 }
58
+ * ```
59
+ */
60
+ export declare function assignTemplateLabels(params: AssignTemplateLabelsParams): Map<string, TemplateLabel>;