@nitpicker/archive 0.0.0-stage → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (737) hide show
  1. package/LICENSE +191 -0
  2. package/README.md +105 -2
  3. package/lib/apply-connection-pragmas.d.ts +18 -0
  4. package/lib/apply-connection-pragmas.js +40 -0
  5. package/lib/archive-accessor.d.ts +265 -0
  6. package/lib/archive-accessor.js +477 -0
  7. package/lib/archive-lock.d.ts +45 -0
  8. package/lib/archive-lock.js +154 -0
  9. package/lib/archive.d.ts +736 -0
  10. package/lib/archive.js +1056 -0
  11. package/lib/body-hash/compute-body-hash.d.ts +32 -0
  12. package/lib/body-hash/compute-body-hash.js +35 -0
  13. package/lib/cache/clear-archive-cache-entry.d.ts +18 -0
  14. package/lib/cache/clear-archive-cache-entry.js +24 -0
  15. package/lib/cache/clear-archive-cache-root.d.ts +26 -0
  16. package/lib/cache/clear-archive-cache-root.js +32 -0
  17. package/lib/cache/compute-archive-cache-key.d.ts +39 -0
  18. package/lib/cache/compute-archive-cache-key.js +95 -0
  19. package/lib/cache/extract-archive-to-cache.d.ts +57 -0
  20. package/lib/cache/extract-archive-to-cache.js +326 -0
  21. package/lib/cache/get-archive-cache-root.d.ts +20 -0
  22. package/lib/cache/get-archive-cache-root.js +53 -0
  23. package/lib/cache/is-archive-cache-disabled.d.ts +24 -0
  24. package/lib/cache/is-archive-cache-disabled.js +34 -0
  25. package/lib/cache/list-archive-cache-entries.d.ts +23 -0
  26. package/lib/cache/list-archive-cache-entries.js +130 -0
  27. package/lib/cache/path-exists.d.ts +9 -0
  28. package/lib/cache/path-exists.js +18 -0
  29. package/lib/cache/resolve-archive-cache-dir.d.ts +26 -0
  30. package/lib/cache/resolve-archive-cache-dir.js +32 -0
  31. package/lib/cache/types.d.ts +33 -0
  32. package/lib/cache/types.js +1 -0
  33. package/lib/content-type/is-html-content-type.d.ts +17 -0
  34. package/lib/content-type/is-html-content-type.js +19 -0
  35. package/lib/content-type/normalize-content-type.d.ts +14 -0
  36. package/lib/content-type/normalize-content-type.js +20 -0
  37. package/lib/create-adjunct-tables.d.ts +66 -0
  38. package/lib/create-adjunct-tables.js +510 -0
  39. package/lib/create-entity-tables.d.ts +191 -0
  40. package/lib/create-entity-tables.js +387 -0
  41. package/lib/create-progress-reporter.d.ts +30 -0
  42. package/lib/create-progress-reporter.js +38 -0
  43. package/lib/create-ref-tables.d.ts +35 -0
  44. package/lib/create-ref-tables.js +185 -0
  45. package/lib/create-template-tables.d.ts +28 -0
  46. package/lib/create-template-tables.js +92 -0
  47. package/lib/database.d.ts +549 -0
  48. package/lib/database.js +804 -0
  49. package/lib/db-ops/_shared/clear-write-ref-caches.d.ts +27 -0
  50. package/lib/db-ops/_shared/clear-write-ref-caches.js +35 -0
  51. package/lib/db-ops/_shared/compress-payload.d.ts +26 -0
  52. package/lib/db-ops/_shared/compress-payload.js +30 -0
  53. package/lib/db-ops/_shared/create-write-ref-caches.d.ts +17 -0
  54. package/lib/db-ops/_shared/create-write-ref-caches.js +27 -0
  55. package/lib/db-ops/_shared/decode-json-ref.d.ts +17 -0
  56. package/lib/db-ops/_shared/decode-json-ref.js +31 -0
  57. package/lib/db-ops/_shared/keyset-paginate-by-id.d.ts +49 -0
  58. package/lib/db-ops/_shared/keyset-paginate-by-id.js +69 -0
  59. package/lib/db-ops/_shared/load-response-headers-by-set-ids.d.ts +20 -0
  60. package/lib/db-ops/_shared/load-response-headers-by-set-ids.js +53 -0
  61. package/lib/db-ops/_shared/resolve-content-item-id.d.ts +62 -0
  62. package/lib/db-ops/_shared/resolve-content-item-id.js +143 -0
  63. package/lib/db-ops/_shared/resolve-url-or-blob.d.ts +23 -0
  64. package/lib/db-ops/_shared/resolve-url-or-blob.js +29 -0
  65. package/lib/db-ops/_shared/retry-setting.d.ts +16 -0
  66. package/lib/db-ops/_shared/retry-setting.js +18 -0
  67. package/lib/db-ops/_shared/safe-parse-json.d.ts +11 -0
  68. package/lib/db-ops/_shared/safe-parse-json.js +18 -0
  69. package/lib/db-ops/_shared/types.d.ts +94 -0
  70. package/lib/db-ops/_shared/types.js +1 -0
  71. package/lib/db-ops/_shared/upsert-blob-ref.d.ts +25 -0
  72. package/lib/db-ops/_shared/upsert-blob-ref.js +48 -0
  73. package/lib/db-ops/_shared/upsert-content-type-ref.d.ts +30 -0
  74. package/lib/db-ops/_shared/upsert-content-type-ref.js +45 -0
  75. package/lib/db-ops/_shared/upsert-json-ref.d.ts +22 -0
  76. package/lib/db-ops/_shared/upsert-json-ref.js +41 -0
  77. package/lib/db-ops/_shared/upsert-response-headers.d.ts +35 -0
  78. package/lib/db-ops/_shared/upsert-response-headers.js +49 -0
  79. package/lib/db-ops/_shared/upsert-url-ref.d.ts +39 -0
  80. package/lib/db-ops/_shared/upsert-url-ref.js +62 -0
  81. package/lib/db-ops/anchors/get-anchors-on-page.d.ts +10 -0
  82. package/lib/db-ops/anchors/get-anchors-on-page.js +21 -0
  83. package/lib/db-ops/config/get-base-url.d.ts +8 -0
  84. package/lib/db-ops/config/get-base-url.js +14 -0
  85. package/lib/db-ops/config/get-config.d.ts +10 -0
  86. package/lib/db-ops/config/get-config.js +29 -0
  87. package/lib/db-ops/config/get-created-cwd.d.ts +11 -0
  88. package/lib/db-ops/config/get-created-cwd.js +13 -0
  89. package/lib/db-ops/config/get-name.d.ts +8 -0
  90. package/lib/db-ops/config/get-name.js +14 -0
  91. package/lib/db-ops/config/info-column-allowlist.d.ts +7 -0
  92. package/lib/db-ops/config/info-column-allowlist.js +29 -0
  93. package/lib/db-ops/config/info-json-columns.d.ts +5 -0
  94. package/lib/db-ops/config/info-json-columns.js +11 -0
  95. package/lib/db-ops/config/set-config.d.ts +12 -0
  96. package/lib/db-ops/config/set-config.js +21 -0
  97. package/lib/db-ops/config/update-config.d.ts +17 -0
  98. package/lib/db-ops/config/update-config.js +36 -0
  99. package/lib/db-ops/console-logs/compute-console-log-hash.d.ts +19 -0
  100. package/lib/db-ops/console-logs/compute-console-log-hash.js +30 -0
  101. package/lib/db-ops/console-logs/replace-console-logs.d.ts +34 -0
  102. package/lib/db-ops/console-logs/replace-console-logs.js +123 -0
  103. package/lib/db-ops/console-logs/stringify-console-log-args.d.ts +16 -0
  104. package/lib/db-ops/console-logs/stringify-console-log-args.js +26 -0
  105. package/lib/db-ops/console-logs/types.d.ts +38 -0
  106. package/lib/db-ops/console-logs/types.js +1 -0
  107. package/lib/db-ops/console-logs/upsert-console-log-item.d.ts +29 -0
  108. package/lib/db-ops/console-logs/upsert-console-log-item.js +52 -0
  109. package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts +18 -0
  110. package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js +23 -0
  111. package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts +12 -0
  112. package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.js +15 -0
  113. package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts +14 -0
  114. package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.js +30 -0
  115. package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.d.ts +57 -0
  116. package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.js +87 -0
  117. package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts +21 -0
  118. package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js +27 -0
  119. package/lib/db-ops/errors/insert-crawl-error.d.ts +15 -0
  120. package/lib/db-ops/errors/insert-crawl-error.js +21 -0
  121. package/lib/db-ops/errors/insert-page-error.d.ts +21 -0
  122. package/lib/db-ops/errors/insert-page-error.js +30 -0
  123. package/lib/db-ops/errors/list-dns-burned-host-candidates.d.ts +33 -0
  124. package/lib/db-ops/errors/list-dns-burned-host-candidates.js +158 -0
  125. package/lib/db-ops/html/read-page-html-blob.d.ts +17 -0
  126. package/lib/db-ops/html/read-page-html-blob.js +20 -0
  127. package/lib/db-ops/lifecycle/checkpoint.d.ts +8 -0
  128. package/lib/db-ops/lifecycle/checkpoint.js +9 -0
  129. package/lib/db-ops/lifecycle/destroy.d.ts +6 -0
  130. package/lib/db-ops/lifecycle/destroy.js +7 -0
  131. package/lib/db-ops/lifecycle/init.d.ts +58 -0
  132. package/lib/db-ops/lifecycle/init.js +122 -0
  133. package/lib/db-ops/list-reconcile/record-list-reconcile-run.d.ts +22 -0
  134. package/lib/db-ops/list-reconcile/record-list-reconcile-run.js +41 -0
  135. package/lib/db-ops/meta/get-audios-of-page.d.ts +11 -0
  136. package/lib/db-ops/meta/get-audios-of-page.js +15 -0
  137. package/lib/db-ops/meta/get-buttons-of-page.d.ts +11 -0
  138. package/lib/db-ops/meta/get-buttons-of-page.js +15 -0
  139. package/lib/db-ops/meta/get-canvases-of-page.d.ts +11 -0
  140. package/lib/db-ops/meta/get-canvases-of-page.js +15 -0
  141. package/lib/db-ops/meta/get-custom-elements-of-page.d.ts +11 -0
  142. package/lib/db-ops/meta/get-custom-elements-of-page.js +15 -0
  143. package/lib/db-ops/meta/get-headings-of-page.d.ts +11 -0
  144. package/lib/db-ops/meta/get-headings-of-page.js +15 -0
  145. package/lib/db-ops/meta/get-iframes-of-page.d.ts +11 -0
  146. package/lib/db-ops/meta/get-iframes-of-page.js +15 -0
  147. package/lib/db-ops/meta/get-jsonld-of-page.d.ts +13 -0
  148. package/lib/db-ops/meta/get-jsonld-of-page.js +27 -0
  149. package/lib/db-ops/meta/get-main-content-images-of-page.d.ts +11 -0
  150. package/lib/db-ops/meta/get-main-content-images-of-page.js +15 -0
  151. package/lib/db-ops/meta/get-main-content-tables-of-page.d.ts +11 -0
  152. package/lib/db-ops/meta/get-main-content-tables-of-page.js +15 -0
  153. package/lib/db-ops/meta/get-page-technologies-of-page.d.ts +11 -0
  154. package/lib/db-ops/meta/get-page-technologies-of-page.js +15 -0
  155. package/lib/db-ops/meta/get-technology-signals-of-page.d.ts +11 -0
  156. package/lib/db-ops/meta/get-technology-signals-of-page.js +15 -0
  157. package/lib/db-ops/meta/get-videos-of-page.d.ts +11 -0
  158. package/lib/db-ops/meta/get-videos-of-page.js +15 -0
  159. package/lib/db-ops/outages/close-network-outage.d.ts +12 -0
  160. package/lib/db-ops/outages/close-network-outage.js +15 -0
  161. package/lib/db-ops/outages/close-stale-open-network-outages.d.ts +26 -0
  162. package/lib/db-ops/outages/close-stale-open-network-outages.js +42 -0
  163. package/lib/db-ops/outages/compute-outage-clamp-timestamp.d.ts +26 -0
  164. package/lib/db-ops/outages/compute-outage-clamp-timestamp.js +31 -0
  165. package/lib/db-ops/outages/insert-network-outage.d.ts +14 -0
  166. package/lib/db-ops/outages/insert-network-outage.js +29 -0
  167. package/lib/db-ops/outages/list-network-outages.d.ts +28 -0
  168. package/lib/db-ops/outages/list-network-outages.js +42 -0
  169. package/lib/db-ops/pages/order/set-url-order.d.ts +13 -0
  170. package/lib/db-ops/pages/order/set-url-order.js +38 -0
  171. package/lib/db-ops/pages/read/build-page-query.d.ts +18 -0
  172. package/lib/db-ops/pages/read/build-page-query.js +40 -0
  173. package/lib/db-ops/pages/read/get-crawling-state.d.ts +117 -0
  174. package/lib/db-ops/pages/read/get-crawling-state.js +161 -0
  175. package/lib/db-ops/pages/read/get-existing-page-urls.d.ts +15 -0
  176. package/lib/db-ops/pages/read/get-existing-page-urls.js +30 -0
  177. package/lib/db-ops/pages/read/get-page-count.d.ts +12 -0
  178. package/lib/db-ops/pages/read/get-page-count.js +21 -0
  179. package/lib/db-ops/pages/read/get-page-source-by-url.d.ts +24 -0
  180. package/lib/db-ops/pages/read/get-page-source-by-url.js +28 -0
  181. package/lib/db-ops/pages/read/get-pages-with-rels.d.ts +38 -0
  182. package/lib/db-ops/pages/read/get-pages-with-rels.js +107 -0
  183. package/lib/db-ops/pages/read/get-pages.d.ts +11 -0
  184. package/lib/db-ops/pages/read/get-pages.js +51 -0
  185. package/lib/db-ops/pages/read/get-scraped-html-page-count.d.ts +18 -0
  186. package/lib/db-ops/pages/read/get-scraped-html-page-count.js +25 -0
  187. package/lib/db-ops/pages/read/reconstruct-page-rows.d.ts +31 -0
  188. package/lib/db-ops/pages/read/reconstruct-page-rows.js +32 -0
  189. package/lib/db-ops/pages/reset/clear-page-derived-rows.d.ts +20 -0
  190. package/lib/db-ops/pages/reset/clear-page-derived-rows.js +37 -0
  191. package/lib/db-ops/pages/reset/repromote-external-pages.d.ts +30 -0
  192. package/lib/db-ops/pages/reset/repromote-external-pages.js +91 -0
  193. package/lib/db-ops/pages/reset/reset-failed-pages.d.ts +88 -0
  194. package/lib/db-ops/pages/reset/reset-failed-pages.js +216 -0
  195. package/lib/db-ops/pages/reset/reset-pages-by-urls.d.ts +80 -0
  196. package/lib/db-ops/pages/reset/reset-pages-by-urls.js +151 -0
  197. package/lib/db-ops/pages/reset/retryable-image-scan-codes.d.ts +24 -0
  198. package/lib/db-ops/pages/reset/retryable-image-scan-codes.js +24 -0
  199. package/lib/db-ops/pages/write/insert-audios.d.ts +14 -0
  200. package/lib/db-ops/pages/write/insert-audios.js +25 -0
  201. package/lib/db-ops/pages/write/insert-buttons.d.ts +14 -0
  202. package/lib/db-ops/pages/write/insert-buttons.js +29 -0
  203. package/lib/db-ops/pages/write/insert-canvases.d.ts +14 -0
  204. package/lib/db-ops/pages/write/insert-canvases.js +26 -0
  205. package/lib/db-ops/pages/write/insert-custom-elements.d.ts +27 -0
  206. package/lib/db-ops/pages/write/insert-custom-elements.js +40 -0
  207. package/lib/db-ops/pages/write/insert-headings.d.ts +15 -0
  208. package/lib/db-ops/pages/write/insert-headings.js +27 -0
  209. package/lib/db-ops/pages/write/insert-iframes.d.ts +14 -0
  210. package/lib/db-ops/pages/write/insert-iframes.js +28 -0
  211. package/lib/db-ops/pages/write/insert-inventory-content-items.d.ts +38 -0
  212. package/lib/db-ops/pages/write/insert-inventory-content-items.js +60 -0
  213. package/lib/db-ops/pages/write/insert-inventory-seeds.d.ts +36 -0
  214. package/lib/db-ops/pages/write/insert-inventory-seeds.js +48 -0
  215. package/lib/db-ops/pages/write/insert-inventory-skipped-pages.d.ts +42 -0
  216. package/lib/db-ops/pages/write/insert-inventory-skipped-pages.js +56 -0
  217. package/lib/db-ops/pages/write/insert-jsonld.d.ts +17 -0
  218. package/lib/db-ops/pages/write/insert-jsonld.js +49 -0
  219. package/lib/db-ops/pages/write/insert-main-content-images.d.ts +20 -0
  220. package/lib/db-ops/pages/write/insert-main-content-images.js +32 -0
  221. package/lib/db-ops/pages/write/insert-main-content-tables.d.ts +14 -0
  222. package/lib/db-ops/pages/write/insert-main-content-tables.js +29 -0
  223. package/lib/db-ops/pages/write/insert-page.d.ts +56 -0
  224. package/lib/db-ops/pages/write/insert-page.js +259 -0
  225. package/lib/db-ops/pages/write/insert-technologies.d.ts +25 -0
  226. package/lib/db-ops/pages/write/insert-technologies.js +45 -0
  227. package/lib/db-ops/pages/write/insert-videos.d.ts +14 -0
  228. package/lib/db-ops/pages/write/insert-videos.js +28 -0
  229. package/lib/db-ops/pages/write/link-redirect-sources.d.ts +42 -0
  230. package/lib/db-ops/pages/write/link-redirect-sources.js +101 -0
  231. package/lib/db-ops/pages/write/record-redirect.d.ts +35 -0
  232. package/lib/db-ops/pages/write/record-redirect.js +100 -0
  233. package/lib/db-ops/pages/write/set-skipped-page.d.ts +13 -0
  234. package/lib/db-ops/pages/write/set-skipped-page.js +24 -0
  235. package/lib/db-ops/pages/write/update-page.d.ts +40 -0
  236. package/lib/db-ops/pages/write/update-page.js +419 -0
  237. package/lib/db-ops/pages/write/write-page-html-blob.d.ts +19 -0
  238. package/lib/db-ops/pages/write/write-page-html-blob.js +41 -0
  239. package/lib/db-ops/referrers/get-redirects-for-pages.d.ts +9 -0
  240. package/lib/db-ops/referrers/get-redirects-for-pages.js +15 -0
  241. package/lib/db-ops/referrers/get-referrers-of-page.d.ts +17 -0
  242. package/lib/db-ops/referrers/get-referrers-of-page.js +32 -0
  243. package/lib/db-ops/referrers/get-referrers-of-resource.d.ts +8 -0
  244. package/lib/db-ops/referrers/get-referrers-of-resource.js +15 -0
  245. package/lib/db-ops/resources/build-resource-query.d.ts +25 -0
  246. package/lib/db-ops/resources/build-resource-query.js +29 -0
  247. package/lib/db-ops/resources/get-existing-resource-urls.d.ts +9 -0
  248. package/lib/db-ops/resources/get-existing-resource-urls.js +24 -0
  249. package/lib/db-ops/resources/get-resource-by-url.d.ts +13 -0
  250. package/lib/db-ops/resources/get-resource-by-url.js +22 -0
  251. package/lib/db-ops/resources/get-resource-url-list.d.ts +17 -0
  252. package/lib/db-ops/resources/get-resource-url-list.js +26 -0
  253. package/lib/db-ops/resources/get-resources.d.ts +8 -0
  254. package/lib/db-ops/resources/get-resources.js +11 -0
  255. package/lib/db-ops/resources/insert-inventory-resources.d.ts +24 -0
  256. package/lib/db-ops/resources/insert-inventory-resources.js +64 -0
  257. package/lib/db-ops/resources/insert-resource-referrers.d.ts +15 -0
  258. package/lib/db-ops/resources/insert-resource-referrers.js +54 -0
  259. package/lib/db-ops/resources/insert-resource.d.ts +34 -0
  260. package/lib/db-ops/resources/insert-resource.js +73 -0
  261. package/lib/db-ops/resources/reconstruct-resource-rows.d.ts +26 -0
  262. package/lib/db-ops/resources/reconstruct-resource-rows.js +30 -0
  263. package/lib/db-ops/templates/assign-template-labels.d.ts +60 -0
  264. package/lib/db-ops/templates/assign-template-labels.js +187 -0
  265. package/lib/db-ops/templates/replace-page-templates.d.ts +44 -0
  266. package/lib/db-ops/templates/replace-page-templates.js +174 -0
  267. package/lib/db-ops/templates/serialize-template-cluster-reason.d.ts +16 -0
  268. package/lib/db-ops/templates/serialize-template-cluster-reason.js +17 -0
  269. package/lib/db-ops/templates/types.d.ts +103 -0
  270. package/lib/db-ops/templates/types.js +1 -0
  271. package/lib/debug.d.ts +8 -0
  272. package/lib/debug.js +9 -0
  273. package/lib/decode-html-blob.d.ts +20 -0
  274. package/lib/decode-html-blob.js +23 -0
  275. package/lib/decode-page-html-blob-bytes.d.ts +25 -0
  276. package/lib/decode-page-html-blob-bytes.js +29 -0
  277. package/lib/decode-stored-blob-bytes.d.ts +25 -0
  278. package/lib/decode-stored-blob-bytes.js +47 -0
  279. package/lib/derive-lineage-from-parent.d.ts +37 -0
  280. package/lib/derive-lineage-from-parent.js +42 -0
  281. package/lib/drop-legacy-tables.d.ts +45 -0
  282. package/lib/drop-legacy-tables.js +56 -0
  283. package/lib/error-kind/classify-error-kind.d.ts +20 -0
  284. package/lib/error-kind/classify-error-kind.js +136 -0
  285. package/lib/error-kind/network-related-error-kinds.d.ts +23 -0
  286. package/lib/error-kind/network-related-error-kinds.js +28 -0
  287. package/lib/error-kind/permanent-error-kinds.d.ts +48 -0
  288. package/lib/error-kind/permanent-error-kinds.js +54 -0
  289. package/lib/error-kind/types.d.ts +54 -0
  290. package/lib/error-kind/types.js +1 -0
  291. package/lib/filesystem/append-text.d.ts +9 -0
  292. package/lib/filesystem/append-text.js +14 -0
  293. package/lib/filesystem/copy-dir-sync.d.ts +6 -0
  294. package/lib/filesystem/copy-dir-sync.js +9 -0
  295. package/lib/filesystem/copy-dir.d.ts +7 -0
  296. package/lib/filesystem/copy-dir.js +13 -0
  297. package/lib/filesystem/copy-file-with-progress.d.ts +20 -0
  298. package/lib/filesystem/copy-file-with-progress.js +34 -0
  299. package/lib/filesystem/exists.d.ts +6 -0
  300. package/lib/filesystem/exists.js +9 -0
  301. package/lib/filesystem/get-file-list.d.ts +8 -0
  302. package/lib/filesystem/get-file-list.js +12 -0
  303. package/lib/filesystem/is-dir.d.ts +6 -0
  304. package/lib/filesystem/is-dir.js +10 -0
  305. package/lib/filesystem/mkdir.d.ts +8 -0
  306. package/lib/filesystem/mkdir.js +15 -0
  307. package/lib/filesystem/output-binary.d.ts +12 -0
  308. package/lib/filesystem/output-binary.js +17 -0
  309. package/lib/filesystem/output-json.d.ts +9 -0
  310. package/lib/filesystem/output-json.js +14 -0
  311. package/lib/filesystem/output-text.d.ts +11 -0
  312. package/lib/filesystem/output-text.js +32 -0
  313. package/lib/filesystem/parse-pax-path.d.ts +18 -0
  314. package/lib/filesystem/parse-pax-path.js +47 -0
  315. package/lib/filesystem/parse-tar-size-field.d.ts +13 -0
  316. package/lib/filesystem/parse-tar-size-field.js +40 -0
  317. package/lib/filesystem/peek-tar-top-dir.d.ts +35 -0
  318. package/lib/filesystem/peek-tar-top-dir.js +215 -0
  319. package/lib/filesystem/read-json.d.ts +7 -0
  320. package/lib/filesystem/read-json.js +11 -0
  321. package/lib/filesystem/read-text.d.ts +6 -0
  322. package/lib/filesystem/read-text.js +10 -0
  323. package/lib/filesystem/readline.d.ts +11 -0
  324. package/lib/filesystem/readline.js +26 -0
  325. package/lib/filesystem/remove.d.ts +5 -0
  326. package/lib/filesystem/remove.js +10 -0
  327. package/lib/filesystem/rename.d.ts +16 -0
  328. package/lib/filesystem/rename.js +48 -0
  329. package/lib/filesystem/tar.d.ts +23 -0
  330. package/lib/filesystem/tar.js +78 -0
  331. package/lib/filesystem/untar.d.ts +33 -0
  332. package/lib/filesystem/untar.js +84 -0
  333. package/lib/get-failed-page-messages.d.ts +56 -0
  334. package/lib/get-failed-page-messages.js +146 -0
  335. package/lib/get-json.d.ts +10 -0
  336. package/lib/get-json.js +25 -0
  337. package/lib/html-snapshot/match-selector-html.d.ts +25 -0
  338. package/lib/html-snapshot/match-selector-html.js +28 -0
  339. package/lib/html-snapshot/match-selector-snapshots.d.ts +33 -0
  340. package/lib/html-snapshot/match-selector-snapshots.js +33 -0
  341. package/lib/html-snapshot/search-html-document.d.ts +29 -0
  342. package/lib/html-snapshot/search-html-document.js +27 -0
  343. package/lib/html-snapshot/search-html-snapshots.d.ts +36 -0
  344. package/lib/html-snapshot/search-html-snapshots.js +35 -0
  345. package/lib/html-snapshot/types.d.ts +111 -0
  346. package/lib/html-snapshot/types.js +7 -0
  347. package/lib/html-snapshot/validate-search-pattern.d.ts +15 -0
  348. package/lib/html-snapshot/validate-search-pattern.js +17 -0
  349. package/lib/init-schema.d.ts +48 -0
  350. package/lib/init-schema.js +124 -0
  351. package/lib/is-inventory-source.d.ts +21 -0
  352. package/lib/is-inventory-source.js +22 -0
  353. package/lib/is-within-outage-window.d.ts +49 -0
  354. package/lib/is-within-outage-window.js +33 -0
  355. package/lib/isolated-cluster/create-isolated-cluster-run.d.ts +24 -0
  356. package/lib/isolated-cluster/create-isolated-cluster-run.js +26 -0
  357. package/lib/libsql-dialect.d.ts +115 -0
  358. package/lib/libsql-dialect.js +116 -0
  359. package/lib/limited-page-ids.d.ts +9 -0
  360. package/lib/limited-page-ids.js +18 -0
  361. package/lib/meta/assert-compatible-version.d.ts +60 -0
  362. package/lib/meta/assert-compatible-version.js +93 -0
  363. package/lib/meta/classify-jsonld-type.d.ts +23 -0
  364. package/lib/meta/classify-jsonld-type.js +43 -0
  365. package/lib/meta/compare-semver.d.ts +23 -0
  366. package/lib/meta/compare-semver.js +51 -0
  367. package/lib/meta/compute-main-contents-denormalized.d.ts +41 -0
  368. package/lib/meta/compute-main-contents-denormalized.js +88 -0
  369. package/lib/meta/compute-page-denormalized.d.ts +20 -0
  370. package/lib/meta/compute-page-denormalized.js +35 -0
  371. package/lib/meta/derive-flat-from-meta.d.ts +34 -0
  372. package/lib/meta/derive-flat-from-meta.js +158 -0
  373. package/lib/meta/derive-meta-extras.d.ts +20 -0
  374. package/lib/meta/derive-meta-extras.js +23 -0
  375. package/lib/meta/extract-tags-for-archive.d.ts +17 -0
  376. package/lib/meta/extract-tags-for-archive.js +36 -0
  377. package/lib/meta/parse-main-content-class-list.d.ts +19 -0
  378. package/lib/meta/parse-main-content-class-list.js +34 -0
  379. package/lib/meta/summarize-jsonld.d.ts +17 -0
  380. package/lib/meta/summarize-jsonld.js +29 -0
  381. package/lib/meta/technologies/combine-technology-confidence.d.ts +46 -0
  382. package/lib/meta/technologies/combine-technology-confidence.js +93 -0
  383. package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.d.ts +45 -0
  384. package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.js +53 -0
  385. package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.d.ts +33 -0
  386. package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.js +46 -0
  387. package/lib/meta/technologies/extract-technologies-for-archive.d.ts +33 -0
  388. package/lib/meta/technologies/extract-technologies-for-archive.js +30 -0
  389. package/lib/meta/technologies/match-js-license-comment-signal.d.ts +21 -0
  390. package/lib/meta/technologies/match-js-license-comment-signal.js +23 -0
  391. package/lib/meta/technologies/match-structural-technology-signals.d.ts +26 -0
  392. package/lib/meta/technologies/match-structural-technology-signals.js +48 -0
  393. package/lib/meta/technologies/normalize-wappalyzer-entries.d.ts +25 -0
  394. package/lib/meta/technologies/normalize-wappalyzer-entries.js +35 -0
  395. package/lib/meta/technologies/technology-signal-definitions.d.ts +44 -0
  396. package/lib/meta/technologies/technology-signal-definitions.js +60 -0
  397. package/lib/meta/technologies/types.d.ts +70 -0
  398. package/lib/meta/technologies/types.js +13 -0
  399. package/lib/meta/types.d.ts +1490 -0
  400. package/lib/meta/types.js +65 -0
  401. package/lib/migrate-content-items-alias-of-id.d.ts +52 -0
  402. package/lib/migrate-content-items-alias-of-id.js +69 -0
  403. package/lib/migrate-content-items-dedupe-cap-event-id.d.ts +46 -0
  404. package/lib/migrate-content-items-dedupe-cap-event-id.js +62 -0
  405. package/lib/migrate-content-items-is-metadata-only.d.ts +44 -0
  406. package/lib/migrate-content-items-is-metadata-only.js +60 -0
  407. package/lib/migrate-entity-tables.d.ts +55 -0
  408. package/lib/migrate-entity-tables.js +66 -0
  409. package/lib/migrate-info-created-cwd.d.ts +17 -0
  410. package/lib/migrate-info-created-cwd.js +36 -0
  411. package/lib/migrate-info-main-content-selector.d.ts +16 -0
  412. package/lib/migrate-info-main-content-selector.js +35 -0
  413. package/lib/migrate-info-request-header-names.d.ts +16 -0
  414. package/lib/migrate-info-request-header-names.js +35 -0
  415. package/lib/migrate-info-roots.d.ts +20 -0
  416. package/lib/migrate-info-roots.js +52 -0
  417. package/lib/migrate-inventory-runs-to-list-reconcile-runs.d.ts +34 -0
  418. package/lib/migrate-inventory-runs-to-list-reconcile-runs.js +51 -0
  419. package/lib/migrate-list-reconcile-runs-exclude-skipped.d.ts +25 -0
  420. package/lib/migrate-list-reconcile-runs-exclude-skipped.js +44 -0
  421. package/lib/migrate-list-reconcile-runs-invalid-skipped.d.ts +16 -0
  422. package/lib/migrate-list-reconcile-runs-invalid-skipped.js +35 -0
  423. package/lib/migrate-main-contents-columns.d.ts +26 -0
  424. package/lib/migrate-main-contents-columns.js +61 -0
  425. package/lib/migrate-page-meta-body-hash.d.ts +44 -0
  426. package/lib/migrate-page-meta-body-hash.js +63 -0
  427. package/lib/migrate-page-meta-console-error-count.d.ts +27 -0
  428. package/lib/migrate-page-meta-console-error-count.js +45 -0
  429. package/lib/migrate-page-meta-custom-element-count.d.ts +33 -0
  430. package/lib/migrate-page-meta-custom-element-count.js +51 -0
  431. package/lib/migrate-page-meta-image-scan.d.ts +27 -0
  432. package/lib/migrate-page-meta-image-scan.js +52 -0
  433. package/lib/migrate-page-tags-to-page-technologies.d.ts +39 -0
  434. package/lib/migrate-page-tags-to-page-technologies.js +79 -0
  435. package/lib/migrate-ref-tables.d.ts +25 -0
  436. package/lib/migrate-ref-tables.js +38 -0
  437. package/lib/native-executor/check-native-executor-config.d.ts +17 -0
  438. package/lib/native-executor/check-native-executor-config.js +20 -0
  439. package/lib/page-meta-column-maps.d.ts +32 -0
  440. package/lib/page-meta-column-maps.js +43 -0
  441. package/lib/page.d.ts +546 -0
  442. package/lib/page.js +764 -0
  443. package/lib/peek-archive-lock.d.ts +40 -0
  444. package/lib/peek-archive-lock.js +62 -0
  445. package/lib/populate-entity-tables/collapse-anchor-rows.d.ts +41 -0
  446. package/lib/populate-entity-tables/collapse-anchor-rows.js +87 -0
  447. package/lib/populate-entity-tables/derive-dom-path.d.ts +35 -0
  448. package/lib/populate-entity-tables/derive-dom-path.js +72 -0
  449. package/lib/populate-entity-tables/is-blob-ref-value.d.ts +16 -0
  450. package/lib/populate-entity-tables/is-blob-ref-value.js +19 -0
  451. package/lib/populate-entity-tables/match-images-to-dom-paths.d.ts +66 -0
  452. package/lib/populate-entity-tables/match-images-to-dom-paths.js +96 -0
  453. package/lib/populate-entity-tables/populate-anchor-edges.d.ts +33 -0
  454. package/lib/populate-entity-tables/populate-anchor-edges.js +153 -0
  455. package/lib/populate-entity-tables/populate-content-items.d.ts +40 -0
  456. package/lib/populate-entity-tables/populate-content-items.js +141 -0
  457. package/lib/populate-entity-tables/populate-entities.d.ts +81 -0
  458. package/lib/populate-entity-tables/populate-entities.js +110 -0
  459. package/lib/populate-entity-tables/populate-image-items.d.ts +91 -0
  460. package/lib/populate-entity-tables/populate-image-items.js +226 -0
  461. package/lib/populate-entity-tables/populate-page-meta.d.ts +33 -0
  462. package/lib/populate-entity-tables/populate-page-meta.js +267 -0
  463. package/lib/populate-entity-tables/populate-resource-items.d.ts +22 -0
  464. package/lib/populate-entity-tables/populate-resource-items.js +114 -0
  465. package/lib/populate-entity-tables/populate-resource-ref-edges.d.ts +31 -0
  466. package/lib/populate-entity-tables/populate-resource-ref-edges.js +33 -0
  467. package/lib/populate-entity-tables/resolve-blob-refs.d.ts +31 -0
  468. package/lib/populate-entity-tables/resolve-blob-refs.js +100 -0
  469. package/lib/populate-entity-tables/resolve-content-type-refs.d.ts +22 -0
  470. package/lib/populate-entity-tables/resolve-content-type-refs.js +27 -0
  471. package/lib/populate-entity-tables/resolve-header-sets.d.ts +49 -0
  472. package/lib/populate-entity-tables/resolve-header-sets.js +122 -0
  473. package/lib/populate-entity-tables/resolve-json-refs.d.ts +25 -0
  474. package/lib/populate-entity-tables/resolve-json-refs.js +67 -0
  475. package/lib/populate-entity-tables/resolve-text-refs.d.ts +30 -0
  476. package/lib/populate-entity-tables/resolve-text-refs.js +61 -0
  477. package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.d.ts +21 -0
  478. package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.js +27 -0
  479. package/lib/populate-entity-tables/resolve-url-refs.d.ts +33 -0
  480. package/lib/populate-entity-tables/resolve-url-refs.js +60 -0
  481. package/lib/populate-entity-tables/test-utils/count-rows.d.ts +17 -0
  482. package/lib/populate-entity-tables/test-utils/count-rows.js +20 -0
  483. package/lib/populate-entity-tables/test-utils/seed-content-items.d.ts +25 -0
  484. package/lib/populate-entity-tables/test-utils/seed-content-items.js +42 -0
  485. package/lib/populate-entity-tables/test-utils/setup-entities-db.d.ts +30 -0
  486. package/lib/populate-entity-tables/test-utils/setup-entities-db.js +175 -0
  487. package/lib/populate-entity-tables/types.d.ts +157 -0
  488. package/lib/populate-entity-tables/types.js +12 -0
  489. package/lib/populate-entity-tables/upsert-text-refs.d.ts +38 -0
  490. package/lib/populate-entity-tables/upsert-text-refs.js +78 -0
  491. package/lib/populate-ref-tables/classify-content-type.d.ts +16 -0
  492. package/lib/populate-ref-tables/classify-content-type.js +52 -0
  493. package/lib/populate-ref-tables/compute-content-hash.d.ts +22 -0
  494. package/lib/populate-ref-tables/compute-content-hash.js +26 -0
  495. package/lib/populate-ref-tables/compute-header-flags.d.ts +16 -0
  496. package/lib/populate-ref-tables/compute-header-flags.js +70 -0
  497. package/lib/populate-ref-tables/content-type-rules.d.ts +38 -0
  498. package/lib/populate-ref-tables/content-type-rules.js +133 -0
  499. package/lib/populate-ref-tables/create-header-table-caches.d.ts +25 -0
  500. package/lib/populate-ref-tables/create-header-table-caches.js +49 -0
  501. package/lib/populate-ref-tables/data-uri-url-refs-limit.d.ts +15 -0
  502. package/lib/populate-ref-tables/data-uri-url-refs-limit.js +15 -0
  503. package/lib/populate-ref-tables/decode-data-uri.d.ts +21 -0
  504. package/lib/populate-ref-tables/decode-data-uri.js +126 -0
  505. package/lib/populate-ref-tables/decompose-header-set.d.ts +29 -0
  506. package/lib/populate-ref-tables/decompose-header-set.js +157 -0
  507. package/lib/populate-ref-tables/decompose-url.d.ts +25 -0
  508. package/lib/populate-ref-tables/decompose-url.js +70 -0
  509. package/lib/populate-ref-tables/header-stability.d.ts +19 -0
  510. package/lib/populate-ref-tables/header-stability.js +22 -0
  511. package/lib/populate-ref-tables/header-value-cache-key.d.ts +17 -0
  512. package/lib/populate-ref-tables/header-value-cache-key.js +19 -0
  513. package/lib/populate-ref-tables/normalize-mime.d.ts +24 -0
  514. package/lib/populate-ref-tables/normalize-mime.js +36 -0
  515. package/lib/populate-ref-tables/populate-blob-refs.d.ts +38 -0
  516. package/lib/populate-ref-tables/populate-blob-refs.js +134 -0
  517. package/lib/populate-ref-tables/populate-content-type-refs.d.ts +27 -0
  518. package/lib/populate-ref-tables/populate-content-type-refs.js +70 -0
  519. package/lib/populate-ref-tables/populate-header-tables.d.ts +35 -0
  520. package/lib/populate-ref-tables/populate-header-tables.js +80 -0
  521. package/lib/populate-ref-tables/populate-json-refs.d.ts +29 -0
  522. package/lib/populate-ref-tables/populate-json-refs.js +101 -0
  523. package/lib/populate-ref-tables/populate-refs.d.ts +51 -0
  524. package/lib/populate-ref-tables/populate-refs.js +62 -0
  525. package/lib/populate-ref-tables/populate-text-refs.d.ts +32 -0
  526. package/lib/populate-ref-tables/populate-text-refs.js +133 -0
  527. package/lib/populate-ref-tables/populate-url-refs.d.ts +28 -0
  528. package/lib/populate-ref-tables/populate-url-refs.js +148 -0
  529. package/lib/populate-ref-tables/test-utils/count-rows.d.ts +15 -0
  530. package/lib/populate-ref-tables/test-utils/count-rows.js +17 -0
  531. package/lib/populate-ref-tables/types.d.ts +197 -0
  532. package/lib/populate-ref-tables/types.js +7 -0
  533. package/lib/populate-ref-tables/upsert-one-header-set.d.ts +34 -0
  534. package/lib/populate-ref-tables/upsert-one-header-set.js +208 -0
  535. package/lib/populate-ref-tables/volatile-header-names.d.ts +20 -0
  536. package/lib/populate-ref-tables/volatile-header-names.js +33 -0
  537. package/lib/redirect-table.d.ts +9 -0
  538. package/lib/redirect-table.js +28 -0
  539. package/lib/release-finished-statements.d.ts +42 -0
  540. package/lib/release-finished-statements.js +45 -0
  541. package/lib/resolve-redirect-chain.d.ts +33 -0
  542. package/lib/resolve-redirect-chain.js +27 -0
  543. package/lib/resource.d.ts +46 -0
  544. package/lib/resource.js +62 -0
  545. package/lib/retarget-legacy-fk-tables.d.ts +61 -0
  546. package/lib/retarget-legacy-fk-tables.js +137 -0
  547. package/lib/safe-path.d.ts +9 -0
  548. package/lib/safe-path.js +17 -0
  549. package/lib/scope/build-scope-map.d.ts +21 -0
  550. package/lib/scope/build-scope-map.js +32 -0
  551. package/lib/scope/find-scope-entry.d.ts +25 -0
  552. package/lib/scope/find-scope-entry.js +45 -0
  553. package/lib/template-classification/classify-archive-page-templates.d.ts +42 -0
  554. package/lib/template-classification/classify-archive-page-templates.js +70 -0
  555. package/lib/template-classification/classify-page-templates.d.ts +33 -0
  556. package/lib/template-classification/classify-page-templates.js +60 -0
  557. package/lib/template-classification/collect-page-ids-with-html.d.ts +24 -0
  558. package/lib/template-classification/collect-page-ids-with-html.js +30 -0
  559. package/lib/template-classification/collect-page-stylesheet-urls.d.ts +32 -0
  560. package/lib/template-classification/collect-page-stylesheet-urls.js +130 -0
  561. package/lib/template-classification/create-content-root-hint.d.ts +41 -0
  562. package/lib/template-classification/create-content-root-hint.js +51 -0
  563. package/lib/template-classification/create-page-cluster-factory.d.ts +36 -0
  564. package/lib/template-classification/create-page-cluster-factory.js +52 -0
  565. package/lib/template-classification/load-page-cluster-candidates.d.ts +32 -0
  566. package/lib/template-classification/load-page-cluster-candidates.js +93 -0
  567. package/lib/template-classification/page-cluster/resolve-page-cluster-keys.d.ts +34 -0
  568. package/lib/template-classification/page-cluster/resolve-page-cluster-keys.js +236 -0
  569. package/lib/template-classification/page-cluster/types.d.ts +123 -0
  570. package/lib/template-classification/page-cluster/types.js +1 -0
  571. package/lib/template-classification/types.d.ts +234 -0
  572. package/lib/template-classification/types.js +1 -0
  573. package/lib/test-utils/build-full-schema-test-db.d.ts +34 -0
  574. package/lib/test-utils/build-full-schema-test-db.js +74 -0
  575. package/lib/test-utils/create-legacy-analysis-tables.d.ts +13 -0
  576. package/lib/test-utils/create-legacy-analysis-tables.js +37 -0
  577. package/lib/test-utils/fk-parent-tables.d.ts +15 -0
  578. package/lib/test-utils/fk-parent-tables.js +19 -0
  579. package/lib/test-utils/seed-content-item.d.ts +35 -0
  580. package/lib/test-utils/seed-content-item.js +42 -0
  581. package/lib/test-utils/setup-legacy-fk-db.d.ts +34 -0
  582. package/lib/test-utils/setup-legacy-fk-db.js +271 -0
  583. package/lib/transfer/attach-source-database.d.ts +31 -0
  584. package/lib/transfer/attach-source-database.js +40 -0
  585. package/lib/transfer/build-split-config.d.ts +30 -0
  586. package/lib/transfer/build-split-config.js +42 -0
  587. package/lib/transfer/clear-replaced-page-rows.d.ts +23 -0
  588. package/lib/transfer/clear-replaced-page-rows.js +61 -0
  589. package/lib/transfer/concat-archives.d.ts +55 -0
  590. package/lib/transfer/concat-archives.js +95 -0
  591. package/lib/transfer/concat-source-transfer-phases.d.ts +10 -0
  592. package/lib/transfer/concat-source-transfer-phases.js +17 -0
  593. package/lib/transfer/content-item-rank-sql.d.ts +21 -0
  594. package/lib/transfer/content-item-rank-sql.js +23 -0
  595. package/lib/transfer/copy-anchor-edges.d.ts +28 -0
  596. package/lib/transfer/copy-anchor-edges.js +42 -0
  597. package/lib/transfer/copy-dictionaries-for-concat.d.ts +39 -0
  598. package/lib/transfer/copy-dictionaries-for-concat.js +81 -0
  599. package/lib/transfer/copy-dictionaries-for-split.d.ts +29 -0
  600. package/lib/transfer/copy-dictionaries-for-split.js +224 -0
  601. package/lib/transfer/copy-header-set-children.d.ts +27 -0
  602. package/lib/transfer/copy-header-set-children.js +51 -0
  603. package/lib/transfer/copy-image-items.d.ts +21 -0
  604. package/lib/transfer/copy-image-items.js +62 -0
  605. package/lib/transfer/copy-inventory-source-lists.d.ts +30 -0
  606. package/lib/transfer/copy-inventory-source-lists.js +61 -0
  607. package/lib/transfer/copy-journal-tables.d.ts +30 -0
  608. package/lib/transfer/copy-journal-tables.js +56 -0
  609. package/lib/transfer/copy-page-console-logs.d.ts +16 -0
  610. package/lib/transfer/copy-page-console-logs.js +27 -0
  611. package/lib/transfer/copy-page-html-blobs.d.ts +31 -0
  612. package/lib/transfer/copy-page-html-blobs.js +43 -0
  613. package/lib/transfer/copy-page-meta.d.ts +27 -0
  614. package/lib/transfer/copy-page-meta.js +68 -0
  615. package/lib/transfer/copy-page-template-clusters.d.ts +31 -0
  616. package/lib/transfer/copy-page-template-clusters.js +56 -0
  617. package/lib/transfer/copy-page-template-labels.d.ts +34 -0
  618. package/lib/transfer/copy-page-template-labels.js +88 -0
  619. package/lib/transfer/copy-resource-items.d.ts +29 -0
  620. package/lib/transfer/copy-resource-items.js +89 -0
  621. package/lib/transfer/copy-resource-ref-edges.d.ts +20 -0
  622. package/lib/transfer/copy-resource-ref-edges.js +33 -0
  623. package/lib/transfer/copy-simple-page-scoped-tables.d.ts +15 -0
  624. package/lib/transfer/copy-simple-page-scoped-tables.js +34 -0
  625. package/lib/transfer/count-external-pages-in-scope.d.ts +26 -0
  626. package/lib/transfer/count-external-pages-in-scope.js +40 -0
  627. package/lib/transfer/create-transfer-temp-tables.d.ts +30 -0
  628. package/lib/transfer/create-transfer-temp-tables.js +59 -0
  629. package/lib/transfer/dictionary-copy-specs.d.ts +32 -0
  630. package/lib/transfer/dictionary-copy-specs.js +28 -0
  631. package/lib/transfer/drop-transfer-temp-tables.d.ts +9 -0
  632. package/lib/transfer/drop-transfer-temp-tables.js +18 -0
  633. package/lib/transfer/flatten-redirect-chains.d.ts +44 -0
  634. package/lib/transfer/flatten-redirect-chains.js +74 -0
  635. package/lib/transfer/insert-content-items.d.ts +35 -0
  636. package/lib/transfer/insert-content-items.js +116 -0
  637. package/lib/transfer/list-transfer-columns.d.ts +35 -0
  638. package/lib/transfer/list-transfer-columns.js +45 -0
  639. package/lib/transfer/mark-in-scope-content-items.d.ts +31 -0
  640. package/lib/transfer/mark-in-scope-content-items.js +54 -0
  641. package/lib/transfer/merge-archive-configs.d.ts +51 -0
  642. package/lib/transfer/merge-archive-configs.js +85 -0
  643. package/lib/transfer/plan-content-items-for-concat.d.ts +33 -0
  644. package/lib/transfer/plan-content-items-for-concat.js +51 -0
  645. package/lib/transfer/plan-content-items-for-split.d.ts +42 -0
  646. package/lib/transfer/plan-content-items-for-split.js +83 -0
  647. package/lib/transfer/plan-resource-items.d.ts +40 -0
  648. package/lib/transfer/plan-resource-items.js +69 -0
  649. package/lib/transfer/quote-transfer-identifier.d.ts +18 -0
  650. package/lib/transfer/quote-transfer-identifier.js +20 -0
  651. package/lib/transfer/reclassify-resource-externality.d.ts +37 -0
  652. package/lib/transfer/reclassify-resource-externality.js +66 -0
  653. package/lib/transfer/remap-redirect-dest-ids.d.ts +34 -0
  654. package/lib/transfer/remap-redirect-dest-ids.js +51 -0
  655. package/lib/transfer/replace-content-items.d.ts +39 -0
  656. package/lib/transfer/replace-content-items.js +83 -0
  657. package/lib/transfer/simple-page-scoped-tables.d.ts +25 -0
  658. package/lib/transfer/simple-page-scoped-tables.js +38 -0
  659. package/lib/transfer/split-archive.d.ts +55 -0
  660. package/lib/transfer/split-archive.js +98 -0
  661. package/lib/transfer/split-source-transfer-phases.d.ts +11 -0
  662. package/lib/transfer/split-source-transfer-phases.js +18 -0
  663. package/lib/transfer/transfer-action.d.ts +34 -0
  664. package/lib/transfer/transfer-action.js +32 -0
  665. package/lib/transfer/transfer-archive-rows.d.ts +65 -0
  666. package/lib/transfer/transfer-archive-rows.js +156 -0
  667. package/lib/transfer/transfer-dictionary-tables.d.ts +12 -0
  668. package/lib/transfer/transfer-dictionary-tables.js +22 -0
  669. package/lib/transfer/transfer-post-phases.d.ts +10 -0
  670. package/lib/transfer/transfer-post-phases.js +13 -0
  671. package/lib/transfer/transfer-source-alias.d.ts +9 -0
  672. package/lib/transfer/transfer-source-alias.js +9 -0
  673. package/lib/transfer/types.d.ts +111 -0
  674. package/lib/transfer/types.js +35 -0
  675. package/lib/types.d.ts +772 -0
  676. package/lib/types.js +1 -0
  677. package/lib/url-alias/compute-tier-a-alias-key.d.ts +44 -0
  678. package/lib/url-alias/compute-tier-a-alias-key.js +47 -0
  679. package/lib/url-alias/compute-tier-b-alias-key.d.ts +23 -0
  680. package/lib/url-alias/compute-tier-b-alias-key.js +26 -0
  681. package/lib/url-alias/group-alias-candidates.d.ts +27 -0
  682. package/lib/url-alias/group-alias-candidates.js +29 -0
  683. package/lib/url-alias/types.d.ts +12 -0
  684. package/lib/url-alias/types.js +1 -0
  685. package/lib/url-pattern/compute-shape-key.d.ts +58 -0
  686. package/lib/url-pattern/compute-shape-key.js +129 -0
  687. package/lib/url-pattern/decompose-url.d.ts +33 -0
  688. package/lib/url-pattern/decompose-url.js +79 -0
  689. package/lib/url-pattern/reconstruct-url.d.ts +15 -0
  690. package/lib/url-pattern/reconstruct-url.js +33 -0
  691. package/lib/url-sort/compute-url-sort-ranks.d.ts +30 -0
  692. package/lib/url-sort/compute-url-sort-ranks.js +33 -0
  693. package/lib/utils/array/each-splitted.d.ts +10 -0
  694. package/lib/utils/array/each-splitted.js +14 -0
  695. package/lib/utils/debug.d.ts +8 -0
  696. package/lib/utils/debug.js +8 -0
  697. package/lib/utils/error/emit-error-with-retry.d.ts +40 -0
  698. package/lib/utils/error/emit-error-with-retry.js +44 -0
  699. package/lib/utils/error/emit-error.d.ts +39 -0
  700. package/lib/utils/error/emit-error.js +41 -0
  701. package/lib/utils/object/parse-response-headers.d.ts +12 -0
  702. package/lib/utils/object/parse-response-headers.js +26 -0
  703. package/lib/utils/object/redact-request-headers.d.ts +15 -0
  704. package/lib/utils/object/redact-request-headers.js +27 -0
  705. package/lib/utils/types/types.d.ts +383 -0
  706. package/lib/utils/types/types.js +1 -0
  707. package/lib/verify-migration/capture-rejection.d.ts +24 -0
  708. package/lib/verify-migration/capture-rejection.js +31 -0
  709. package/lib/verify-migration/check-anchor-edges-count.d.ts +34 -0
  710. package/lib/verify-migration/check-anchor-edges-count.js +72 -0
  711. package/lib/verify-migration/check-anchor-edges-sum.d.ts +13 -0
  712. package/lib/verify-migration/check-anchor-edges-sum.js +27 -0
  713. package/lib/verify-migration/check-content-items-count.d.ts +16 -0
  714. package/lib/verify-migration/check-content-items-count.js +30 -0
  715. package/lib/verify-migration/check-content-type-preservation.d.ts +22 -0
  716. package/lib/verify-migration/check-content-type-preservation.js +40 -0
  717. package/lib/verify-migration/check-foreign-key-integrity.d.ts +31 -0
  718. package/lib/verify-migration/check-foreign-key-integrity.js +47 -0
  719. package/lib/verify-migration/check-image-items-count.d.ts +12 -0
  720. package/lib/verify-migration/check-image-items-count.js +26 -0
  721. package/lib/verify-migration/check-page-meta-count.d.ts +15 -0
  722. package/lib/verify-migration/check-page-meta-count.js +31 -0
  723. package/lib/verify-migration/check-reader-parity.d.ts +23 -0
  724. package/lib/verify-migration/check-reader-parity.js +182 -0
  725. package/lib/verify-migration/check-resource-items-count.d.ts +17 -0
  726. package/lib/verify-migration/check-resource-items-count.js +33 -0
  727. package/lib/verify-migration/check-url-round-trip.d.ts +43 -0
  728. package/lib/verify-migration/check-url-round-trip.js +112 -0
  729. package/lib/verify-migration/types.d.ts +70 -0
  730. package/lib/verify-migration/types.js +63 -0
  731. package/lib/verify-migration/verify-migration.d.ts +41 -0
  732. package/lib/verify-migration/verify-migration.js +120 -0
  733. package/lib/with-bulk-write-pragmas.d.ts +51 -0
  734. package/lib/with-bulk-write-pragmas.js +132 -0
  735. package/lib/write-queue.d.ts +32 -0
  736. package/lib/write-queue.js +55 -0
  737. package/package.json +360 -5
@@ -0,0 +1,182 @@
1
+ import { MigrationVerificationError } from './types.js';
2
+ /**
3
+ * Compares totals reported by 0.13 readers (which query the 0.13
4
+ * entity tables exclusively) against equivalent inline SQL against the
5
+ * still-present legacy write-model tables (`pages` / `anchors` / `images` /
6
+ * `resources` / `resources-referrers`). A mismatch here means either the
7
+ * 0.13 populates lost rows or one of the new reader implementations
8
+ * has a filter/predicate off-by-one relative to its legacy counterpart.
9
+ *
10
+ * This is deliberately lighter than the "byte-identical items array"
11
+ * verification issue #195 acceptance references — running every reader on
12
+ * every archive and diffing full result sets would balloon the migrator's
13
+ * runtime and require importing `@nitpicker/query` back into the crawler
14
+ * package (a dependency inversion). The seven totals below capture every
15
+ * reader whose scope was flagged in issue #195 and catch the classes of
16
+ * regression the 0.13 row-count invariants cannot: predicate drift
17
+ * inside a reader function (e.g. a `WHERE contentType='text/html'` filter
18
+ * silently swapping meaning between `pages.contentType` and
19
+ * `content_type_refs.raw`).
20
+ * @param trx - The open migration transaction.
21
+ * @throws {MigrationVerificationError} If any pair of totals disagrees.
22
+ */
23
+ export async function checkReaderParity(trx) {
24
+ const checks = [
25
+ {
26
+ label: 'listPages default (scraped=1, redirect null, html-or-null)',
27
+ legacy: async () => scalar(trx('pages')
28
+ .where('scraped', 1)
29
+ .whereNull('redirectDestId')
30
+ .where((qb) => {
31
+ qb.where('isSkipped', 0).orWhereNull('isSkipped');
32
+ })
33
+ .where((qb) => {
34
+ qb.whereNull('contentType').orWhere('contentType', 'text/html');
35
+ })
36
+ .count({ count: '*' })),
37
+ current: async () => scalar(trx('content_items as ci')
38
+ .leftJoin('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
39
+ .where('ci.scraped', 1)
40
+ .whereNull('ci.redirect_dest_id')
41
+ .where((qb) => {
42
+ qb.where('ci.is_skipped', 0).orWhereNull('ci.is_skipped');
43
+ })
44
+ .where((qb) => {
45
+ qb.whereNull('ctr.raw').orWhere('ctr.raw', 'text/html');
46
+ })
47
+ .count({ count: '*' })),
48
+ },
49
+ {
50
+ label: 'listImages total',
51
+ legacy: async () => scalar(trx('images').count({ count: '*' })),
52
+ current: async () => scalar(trx('image_items').count({ count: '*' })),
53
+ },
54
+ {
55
+ label: 'listResources total',
56
+ legacy: async () => scalar(trx('resources').count({ count: '*' })),
57
+ current: async () => scalar(trx('resource_items').count({ count: '*' })),
58
+ },
59
+ {
60
+ label: 'listLinks broken (status=404 through redirect)',
61
+ legacy: async () => scalar(trx('anchors')
62
+ .join('pages as dest', 'anchors.hrefId', 'dest.id')
63
+ .leftJoin('pages as canonical', 'dest.redirectDestId', 'canonical.id')
64
+ .whereRaw('COALESCE("canonical"."status", "dest"."status") = 404')
65
+ .count({ count: '*' })),
66
+ current: async () => scalar(trx('anchor_edges as ae')
67
+ .join('content_items as dest', 'ae.href_page_id', 'dest.id')
68
+ .leftJoin('content_items as canonical', 'dest.redirect_dest_id', 'canonical.id')
69
+ .whereRaw('COALESCE("canonical"."status", "dest"."status") = 404')
70
+ .sum({ count: 'ae.count' })),
71
+ },
72
+ {
73
+ label: 'checkHeaders scope (internal html scraped, no redirect)',
74
+ legacy: async () => scalar(trx('pages')
75
+ .where({ scraped: 1, isExternal: 0, contentType: 'text/html' })
76
+ .whereNull('redirectDestId')
77
+ .count({ count: '*' })),
78
+ current: async () => scalar(trx('content_items as ci')
79
+ .join('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
80
+ .where({
81
+ 'ci.scraped': 1,
82
+ 'ci.is_external': 0,
83
+ 'ctr.raw': 'text/html',
84
+ })
85
+ .whereNull('ci.redirect_dest_id')
86
+ .count({ count: '*' })),
87
+ },
88
+ {
89
+ label: 'findDuplicates(title) group count',
90
+ legacy: async () => scalar(trx
91
+ .count({ count: '*' })
92
+ .from(trx('pages')
93
+ .select('title')
94
+ .where({ scraped: 1, isExternal: 0, contentType: 'text/html' })
95
+ .whereNull('redirectDestId')
96
+ .whereNotNull('title')
97
+ .whereNot('title', '')
98
+ .groupBy('title')
99
+ .having(trx.raw('count(*) > 1'))
100
+ .as('g'))),
101
+ current: async () => scalar(trx.count({ count: '*' }).from(trx('content_items as ci')
102
+ .join('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
103
+ .join('page_meta as pm', 'pm.page_id', 'ci.id')
104
+ .join('text_refs as tr', 'tr.id', 'pm.title_text_id')
105
+ .select('pm.title_text_id')
106
+ .where({
107
+ 'ci.scraped': 1,
108
+ 'ci.is_external': 0,
109
+ 'ctr.raw': 'text/html',
110
+ })
111
+ .whereNull('ci.redirect_dest_id')
112
+ .whereNotNull('pm.title_text_id')
113
+ .whereNot('tr.text', '')
114
+ .groupBy('pm.title_text_id')
115
+ .having(trx.raw('count(*) > 1'))
116
+ .as('g'))),
117
+ },
118
+ {
119
+ label: 'findMismatches(canonical) count',
120
+ legacy: async () => scalar(trx('pages')
121
+ .where({ scraped: 1, isExternal: 0, contentType: 'text/html' })
122
+ .whereNull('redirectDestId')
123
+ .whereNotNull('canonical')
124
+ .whereNot('canonical', '')
125
+ .whereRaw('canonical != url')
126
+ .count({ count: '*' })),
127
+ current: async () => scalar(trx('content_items as ci')
128
+ .join('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
129
+ .join('page_meta as pm', 'pm.page_id', 'ci.id')
130
+ .join('url_refs as canonical_ur', 'canonical_ur.id', 'pm.canonical_url_id')
131
+ .where({
132
+ 'ci.scraped': 1,
133
+ 'ci.is_external': 0,
134
+ 'ctr.raw': 'text/html',
135
+ })
136
+ .whereNull('ci.redirect_dest_id')
137
+ .whereNotNull('pm.canonical_url_id')
138
+ .whereNot('canonical_ur.url', '')
139
+ .whereRaw('"pm"."canonical_url_id" != "ci"."url_id"')
140
+ .count({ count: '*' })),
141
+ },
142
+ ];
143
+ const failures = [];
144
+ // Zero-vs-zero cannot detect a "populate silently produced no rows"
145
+ // bug, so a check whose LEGACY side returns 0 is skipped rather than
146
+ // counted as pass — the parity check exists to catch predicate drift
147
+ // once both sides have rows to compare. `checkContentItemsCount` etc.
148
+ // already guard the "current is empty when legacy is not" direction
149
+ // upstream in {@link verifyMigration}.
150
+ let comparedChecks = 0;
151
+ for (const { label, legacy, current } of checks) {
152
+ const [legacyValue, currentValue] = await Promise.all([legacy(), current()]);
153
+ if (legacyValue === 0 && currentValue === 0) {
154
+ continue;
155
+ }
156
+ comparedChecks++;
157
+ if (legacyValue !== currentValue) {
158
+ failures.push(`${label}: legacy=${legacyValue}, current=${currentValue}`);
159
+ }
160
+ }
161
+ if (failures.length > 0) {
162
+ throw new MigrationVerificationError({
163
+ check: '#9 reader parity',
164
+ context: {
165
+ failures: failures.join('; '),
166
+ compared_checks: comparedChecks,
167
+ },
168
+ });
169
+ }
170
+ }
171
+ /**
172
+ * Extracts the single numeric value from a Knex `count()` / `sum()` result.
173
+ * SQLite returns strings for aggregate expressions; we coerce and treat
174
+ * `null` as `0` (an empty aggregate).
175
+ * @param queryBuilder - The pending Knex query that resolves to a single
176
+ * aggregate row.
177
+ * @returns The aggregate value as a JS number.
178
+ */
179
+ async function scalar(queryBuilder) {
180
+ const rows = (await queryBuilder);
181
+ return Number(rows[0]?.count ?? 0);
182
+ }
@@ -0,0 +1,17 @@
1
+ import type { Knex } from 'knex';
2
+ /**
3
+ * Verifies 0.13 invariant #6: every legacy `resources` row is mirrored
4
+ * by one row in `resource_items`.
5
+ *
6
+ * `populateResourceItems` fills `resource_items` with the same PK as
7
+ * `resources.id`
8
+ * (see `populate-resource-items.ts`); the invariant is broken only if the
9
+ * populate loop skipped rows during URL / header-set / content-type
10
+ * resolution. The paired invariant `resource_ref_edges = "resources-referrers"`
11
+ * is left unchecked here because issue #194's spec list stops at the
12
+ * six enumerated checks, and `resource_ref_edges` populate is a straight
13
+ * `INSERT … SELECT` (no dedup, no resolution) with negligible failure surface.
14
+ * @param trx - Knex instance or transaction connected to the populated archive.
15
+ * @throws {MigrationVerificationError} when the row counts diverge.
16
+ */
17
+ export declare function checkResourceItemsCount(trx: Knex): Promise<void>;
@@ -0,0 +1,33 @@
1
+ import { MigrationVerificationError } from './types.js';
2
+ /**
3
+ * Verifies 0.13 invariant #6: every legacy `resources` row is mirrored
4
+ * by one row in `resource_items`.
5
+ *
6
+ * `populateResourceItems` fills `resource_items` with the same PK as
7
+ * `resources.id`
8
+ * (see `populate-resource-items.ts`); the invariant is broken only if the
9
+ * populate loop skipped rows during URL / header-set / content-type
10
+ * resolution. The paired invariant `resource_ref_edges = "resources-referrers"`
11
+ * is left unchecked here because issue #194's spec list stops at the
12
+ * six enumerated checks, and `resource_ref_edges` populate is a straight
13
+ * `INSERT … SELECT` (no dedup, no resolution) with negligible failure surface.
14
+ * @param trx - Knex instance or transaction connected to the populated archive.
15
+ * @throws {MigrationVerificationError} when the row counts diverge.
16
+ */
17
+ export async function checkResourceItemsCount(trx) {
18
+ const resourceItemsRows = await trx('resource_items').count({
19
+ n: '*',
20
+ });
21
+ const resourcesRows = await trx('resources').count({ n: '*' });
22
+ const resourceItemsCount = Number(resourceItemsRows[0].n);
23
+ const resourcesCount = Number(resourcesRows[0].n);
24
+ if (resourceItemsCount !== resourcesCount) {
25
+ throw new MigrationVerificationError({
26
+ check: '#6 resource_items row count',
27
+ context: {
28
+ resource_items: resourceItemsCount,
29
+ resources: resourcesCount,
30
+ },
31
+ });
32
+ }
33
+ }
@@ -0,0 +1,43 @@
1
+ import type { Knex } from 'knex';
2
+ /**
3
+ * Verifies 0.13 invariant #8: `content_items.url_id → url_refs.url`
4
+ * round-trips back to the original `pages.url` on a deterministic
5
+ * ≤ {@link SAMPLE_SIZE} sample.
6
+ *
7
+ * A mismatch means one of two things:
8
+ *
9
+ * - `populate-url-refs.ts` collapsed two distinct URLs into the same
10
+ * `url_refs.id` (BINARY collation bug, whitespace normalisation, …).
11
+ * - `populate-content-items.ts` associated a page id with the wrong
12
+ * `url_ref.id` (row-order dependency in the batch resolver).
13
+ *
14
+ * Additionally an FK gap (`content_items.url_id` pointing at a
15
+ * non-existent `url_refs.id`) or a missing `pages` row surfaces as a
16
+ * `roundTripUrl` / `sourceUrl` of `null`; the check treats either as a
17
+ * round-trip failure. LEFT JOINs are used deliberately so orphan rows
18
+ * are observable — an INNER JOIN would silently drop them and let the
19
+ * bug the invariant is meant to catch slip through.
20
+ *
21
+ * Sampling is **deterministic**: stride = ⌈count(content_items) / N⌉
22
+ * gives approximately {@link SAMPLE_SIZE} rows spread across the id
23
+ * range for large archives, and the full table for archives with
24
+ * ≤ {@link SAMPLE_SIZE} rows. Deterministic sampling means the same
25
+ * archive always produces the same verdict; a random sample would let a
26
+ * failing archive pass on retry if the offending rows happen to fall
27
+ * outside the second draw.
28
+ *
29
+ * The check throws when the sampled row count is smaller than expected
30
+ * (LEFT JOINs may return `null` sides but never fewer rows than the
31
+ * driving `content_items` set — a shortfall means SQLite silently
32
+ * dropped rows we intended to inspect). "Expected" is a separate
33
+ * `COUNT(*)` against the same `(id % stride) = 0` predicate, not a
34
+ * `floor(total / stride)` estimate: `content_items.id` is copied
35
+ * verbatim from the legacy `pages.id`, which accumulates gaps over an
36
+ * archive's crawl history (redirect consolidation, deleted pages, …).
37
+ * A density estimate over-counts whenever the id range is sparser than
38
+ * `[1, total]`, which produced false-positive failures on real
39
+ * multi-thousand-page archives.
40
+ * @param trx - Knex instance or transaction connected to the populated archive.
41
+ * @throws {MigrationVerificationError} when at least one sampled URL does not round-trip.
42
+ */
43
+ export declare function checkUrlRoundTrip(trx: Knex): Promise<void>;
@@ -0,0 +1,112 @@
1
+ import { MigrationVerificationError } from './types.js';
2
+ /**
3
+ * Sample size for the URL round-trip smoke test — a 1,000-row sample per
4
+ * issue #194.
5
+ * Not statistical coverage — URL normalisation bugs tend to be systematic
6
+ * (all rows with a particular scheme or host), so a modest evenly-spread
7
+ * sample catches them reliably.
8
+ */
9
+ const SAMPLE_SIZE = 1000;
10
+ /**
11
+ * Verifies 0.13 invariant #8: `content_items.url_id → url_refs.url`
12
+ * round-trips back to the original `pages.url` on a deterministic
13
+ * ≤ {@link SAMPLE_SIZE} sample.
14
+ *
15
+ * A mismatch means one of two things:
16
+ *
17
+ * - `populate-url-refs.ts` collapsed two distinct URLs into the same
18
+ * `url_refs.id` (BINARY collation bug, whitespace normalisation, …).
19
+ * - `populate-content-items.ts` associated a page id with the wrong
20
+ * `url_ref.id` (row-order dependency in the batch resolver).
21
+ *
22
+ * Additionally an FK gap (`content_items.url_id` pointing at a
23
+ * non-existent `url_refs.id`) or a missing `pages` row surfaces as a
24
+ * `roundTripUrl` / `sourceUrl` of `null`; the check treats either as a
25
+ * round-trip failure. LEFT JOINs are used deliberately so orphan rows
26
+ * are observable — an INNER JOIN would silently drop them and let the
27
+ * bug the invariant is meant to catch slip through.
28
+ *
29
+ * Sampling is **deterministic**: stride = ⌈count(content_items) / N⌉
30
+ * gives approximately {@link SAMPLE_SIZE} rows spread across the id
31
+ * range for large archives, and the full table for archives with
32
+ * ≤ {@link SAMPLE_SIZE} rows. Deterministic sampling means the same
33
+ * archive always produces the same verdict; a random sample would let a
34
+ * failing archive pass on retry if the offending rows happen to fall
35
+ * outside the second draw.
36
+ *
37
+ * The check throws when the sampled row count is smaller than expected
38
+ * (LEFT JOINs may return `null` sides but never fewer rows than the
39
+ * driving `content_items` set — a shortfall means SQLite silently
40
+ * dropped rows we intended to inspect). "Expected" is a separate
41
+ * `COUNT(*)` against the same `(id % stride) = 0` predicate, not a
42
+ * `floor(total / stride)` estimate: `content_items.id` is copied
43
+ * verbatim from the legacy `pages.id`, which accumulates gaps over an
44
+ * archive's crawl history (redirect consolidation, deleted pages, …).
45
+ * A density estimate over-counts whenever the id range is sparser than
46
+ * `[1, total]`, which produced false-positive failures on real
47
+ * multi-thousand-page archives.
48
+ * @param trx - Knex instance or transaction connected to the populated archive.
49
+ * @throws {MigrationVerificationError} when at least one sampled URL does not round-trip.
50
+ */
51
+ export async function checkUrlRoundTrip(trx) {
52
+ const totalRows = await trx('content_items').count({ n: '*' });
53
+ const total = Number(totalRows[0].n);
54
+ if (total === 0) {
55
+ return;
56
+ }
57
+ // stride = ⌈total / SAMPLE_SIZE⌉ so `id % stride = 0` picks approximately
58
+ // SAMPLE_SIZE rows evenly across the id range for large archives, and the
59
+ // full table for archives ≤ SAMPLE_SIZE.
60
+ const stride = Math.max(1, Math.ceil(total / SAMPLE_SIZE));
61
+ const matchingRows = await trx('content_items')
62
+ .whereRaw('(id % ?) = 0', [stride])
63
+ .count({ n: '*' });
64
+ const expectedSize = Math.min(SAMPLE_SIZE, Number(matchingRows[0].n));
65
+ const sample = await trx('content_items as ci')
66
+ .leftJoin('pages as p', 'p.id', 'ci.id')
67
+ .leftJoin('url_refs as ur', 'ur.id', 'ci.url_id')
68
+ .select('ci.id as id', 'p.url as sourceUrl', 'ur.url as roundTripUrl')
69
+ .whereRaw('(ci.id % ?) = 0', [stride])
70
+ .orderBy('ci.id')
71
+ .limit(SAMPLE_SIZE);
72
+ if (sample.length < expectedSize) {
73
+ // LEFT JOINs preserve driving-table row counts (`p.id` / `url_refs.id`
74
+ // are PKs → 1:1 or 0:1 match). A shortfall means SQLite silently
75
+ // dropped rows the WHERE clause matched — a real integrity signal.
76
+ throw new MigrationVerificationError({
77
+ check: '#8 URL round-trip',
78
+ context: {
79
+ content_items_total: total,
80
+ stride,
81
+ expected_sample_size: expectedSize,
82
+ actual_sample_size: sample.length,
83
+ reason: 'sample query returned fewer rows than expected',
84
+ },
85
+ });
86
+ }
87
+ for (const row of sample) {
88
+ if (row.sourceUrl === null || row.roundTripUrl === null) {
89
+ throw new MigrationVerificationError({
90
+ check: '#8 URL round-trip',
91
+ context: {
92
+ page_id: row.id,
93
+ source_url: row.sourceUrl,
94
+ round_trip_url: row.roundTripUrl,
95
+ sample_size: sample.length,
96
+ reason: 'orphan row — pages or url_refs join returned null (FK gap)',
97
+ },
98
+ });
99
+ }
100
+ if (row.sourceUrl !== row.roundTripUrl) {
101
+ throw new MigrationVerificationError({
102
+ check: '#8 URL round-trip',
103
+ context: {
104
+ page_id: row.id,
105
+ source_url: row.sourceUrl,
106
+ round_trip_url: row.roundTripUrl,
107
+ sample_size: sample.length,
108
+ },
109
+ });
110
+ }
111
+ }
112
+ }
@@ -0,0 +1,70 @@
1
+ /**
2
+ * Shared type definitions for 0.13 migration verification.
3
+ *
4
+ * These are the acceptance-invariant checks that `scripts/migrate-to-0.13.mjs`
5
+ * runs at the end of the migration to confirm that the 0.13 entity /
6
+ * edge populates produced a consistent archive. Each check is one file
7
+ * under `verify-migration/` and throws {@link MigrationVerificationError} on mismatch;
8
+ * the orchestrator {@link ./verify-migration.ts} chains them in the
9
+ * order defined by issue #194.
10
+ * @module
11
+ */
12
+ /**
13
+ * Row-count snapshot collected by every successful verify run and returned
14
+ * by {@link import('./verify-migration.js').verifyMigration}.
15
+ * The migration script logs these on the successful path so operators can
16
+ * eyeball migration outcomes from stdout without re-opening the archive.
17
+ */
18
+ export interface MigrationVerificationSummary {
19
+ /** `count(*) FROM content_items` — matches `count(pages)`. */
20
+ readonly contentItems: number;
21
+ /** `count(*) FROM page_meta` — matches `count(pages WHERE scraped=1)`. */
22
+ readonly pageMeta: number;
23
+ /** `count(*) FROM anchor_edges` — always ≤ `count(anchors)`. */
24
+ readonly anchorEdges: number;
25
+ /** `SUM(count) FROM anchor_edges` — matches `count(anchors)`. */
26
+ readonly anchorEdgesSum: number;
27
+ /** `count(*) FROM image_items` — matches `count(images)`. */
28
+ readonly imageItems: number;
29
+ /** `count(*) FROM resource_items` — matches `count(resources)`. */
30
+ readonly resourceItems: number;
31
+ }
32
+ /**
33
+ * Structured details of a single migration invariant that failed. Attached to
34
+ * {@link MigrationVerificationError} so operators can see which check tripped
35
+ * without parsing the error message.
36
+ */
37
+ export interface MigrationVerificationErrorDetails {
38
+ /**
39
+ * Human-readable check identifier matching issue #194's numbered list
40
+ * (e.g. `'#3'` for the anchor-edge dedup range check). Included in the
41
+ * error message so migration logs point at the exact clause.
42
+ */
43
+ readonly check: string;
44
+ /**
45
+ * Optional structured payload — typically `expected` / `actual` counts or
46
+ * the offending page id from the URL round-trip sample. Rendered into the
47
+ * error message so operators can diagnose without re-running the check.
48
+ */
49
+ readonly context?: Readonly<Record<string, string | number | null>>;
50
+ }
51
+ /**
52
+ * Error thrown by any 0.13 check when an invariant does not hold. The
53
+ * migration script catches this at the top level and aborts with a non-zero
54
+ * exit code; the enclosing transaction rolls back so the archive returns to
55
+ * its pre-populate state (ref tables from ref populate stay populated but are
56
+ * additive).
57
+ *
58
+ * Prefer a single error class with a structured {@link details} payload over
59
+ * one subclass per check: eight subclasses would create eight nearly
60
+ * identical constructors, and callers (script + integration test) only need
61
+ * to distinguish "verification failed" from other error kinds.
62
+ */
63
+ export declare class MigrationVerificationError extends Error {
64
+ readonly details: MigrationVerificationErrorDetails;
65
+ /**
66
+ * @param details - Structured description of which check failed and why.
67
+ * Rendered into the error message on construction.
68
+ */
69
+ constructor(details: MigrationVerificationErrorDetails);
70
+ }
@@ -0,0 +1,63 @@
1
+ /**
2
+ * Shared type definitions for 0.13 migration verification.
3
+ *
4
+ * These are the acceptance-invariant checks that `scripts/migrate-to-0.13.mjs`
5
+ * runs at the end of the migration to confirm that the 0.13 entity /
6
+ * edge populates produced a consistent archive. Each check is one file
7
+ * under `verify-migration/` and throws {@link MigrationVerificationError} on mismatch;
8
+ * the orchestrator {@link ./verify-migration.ts} chains them in the
9
+ * order defined by issue #194.
10
+ * @module
11
+ */
12
+ /**
13
+ * Error thrown by any 0.13 check when an invariant does not hold. The
14
+ * migration script catches this at the top level and aborts with a non-zero
15
+ * exit code; the enclosing transaction rolls back so the archive returns to
16
+ * its pre-populate state (ref tables from ref populate stay populated but are
17
+ * additive).
18
+ *
19
+ * Prefer a single error class with a structured {@link details} payload over
20
+ * one subclass per check: eight subclasses would create eight nearly
21
+ * identical constructors, and callers (script + integration test) only need
22
+ * to distinguish "verification failed" from other error kinds.
23
+ */
24
+ export class MigrationVerificationError extends Error {
25
+ details;
26
+ /**
27
+ * @param details - Structured description of which check failed and why.
28
+ * Rendered into the error message on construction.
29
+ */
30
+ constructor(details) {
31
+ const contextText = details.context === undefined
32
+ ? ''
33
+ : ' — ' +
34
+ Object.entries(details.context)
35
+ .map(([key, value]) =>
36
+ // Render null as the parenthesised literal `(null)` so
37
+ // operators reading the log can distinguish "the sample
38
+ // query returned no rows" from a caller that passed the
39
+ // four-character string `"null"`. Strings get quoted for
40
+ // the same reason.
41
+ `${key}=${formatContextValue(value)}`)
42
+ .join(', ');
43
+ super(`migration verification failed ${details.check}${contextText}`);
44
+ this.details = details;
45
+ this.name = 'MigrationVerificationError';
46
+ }
47
+ }
48
+ /**
49
+ * Renders one context value for {@link MigrationVerificationError}'s message.
50
+ * `null` becomes the literal `(null)`; strings are wrapped in double quotes;
51
+ * numbers pass through as their decimal representation. The intent is to
52
+ * make missing-vs-present distinguishable in operator logs.
53
+ * @param value - Value from `MigrationVerificationErrorDetails.context`.
54
+ */
55
+ function formatContextValue(value) {
56
+ if (value === null) {
57
+ return '(null)';
58
+ }
59
+ if (typeof value === 'string') {
60
+ return JSON.stringify(value);
61
+ }
62
+ return String(value);
63
+ }
@@ -0,0 +1,41 @@
1
+ import type { MigrationVerificationSummary } from './types.js';
2
+ import type { Knex } from 'knex';
3
+ /**
4
+ * Runs every 0.13 migration acceptance invariant against a populated archive and
5
+ * returns the verified row-count summary on success. The migration script
6
+ * (`scripts/migrate-to-0.13.mjs`) calls this **inside** the same
7
+ * `knex.transaction()` block that ran `populateEntityTables` so a thrown
8
+ * {@link MigrationVerificationError} rolls back the entire entity populate — ref
9
+ * tables from ref populate stay committed because they live in a separate, additive
10
+ * transaction and never lose facts on re-run.
11
+ *
12
+ * The returned {@link MigrationVerificationSummary} is echoed to stdout by the
13
+ * migration script so operators can eyeball migration outcomes (`did we lose
14
+ * rows silently within the invariant window?`) from batch pipeline logs
15
+ * without re-opening the archive. Callers must run this inside a `db.transaction()`
16
+ * for the counts and the JOIN-based checks (#7 / #8) to see a consistent
17
+ * snapshot of the archive.
18
+ *
19
+ * Checks run sequentially in issue #194's numbered order so error messages
20
+ * pinpoint the earliest broken invariant. Sequential (rather than parallel)
21
+ * execution keeps the failure diagnostics focused: fanning eight COUNTs out
22
+ * at once would return the first-completed rejection rather than the
23
+ * lowest-numbered one, which surprises operators reading migration logs.
24
+ * The individual COUNT queries are fast enough at any real archive scale
25
+ * for sequential wall-clock cost to be irrelevant.
26
+ *
27
+ * Non-{@link MigrationVerificationError} exceptions from a check (e.g. a
28
+ * transient libsql error or a schema-drift SqliteError) are wrapped in a
29
+ * {@link MigrationVerificationError} so the migration script's `catch` sees a
30
+ * uniform failure surface — otherwise operators grepping stderr for
31
+ * `migration verification failed` would miss driver-side errors.
32
+ * @param trx - Knex instance or transaction connected to the archive
33
+ * **after** every 0.13 migration populate has run. Must be a transaction
34
+ * in production so ref counts, JOIN samples, and error rollback see the
35
+ * same snapshot; non-transactional callers (unit tests) work but do not
36
+ * get snapshot isolation.
37
+ * @returns Row-count summary of the verified archive.
38
+ * @throws {MigrationVerificationError} on the first invariant that does not
39
+ * hold; subsequent checks are skipped.
40
+ */
41
+ export declare function verifyMigration(trx: Knex): Promise<MigrationVerificationSummary>;