@nitpicker/archive 0.0.0 → 0.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (739) hide show
  1. package/LICENSE +191 -0
  2. package/README.md +106 -0
  3. package/lib/apply-connection-pragmas.d.ts +18 -0
  4. package/lib/apply-connection-pragmas.js +40 -0
  5. package/lib/archive-accessor.d.ts +265 -0
  6. package/lib/archive-accessor.js +477 -0
  7. package/lib/archive-lock.d.ts +45 -0
  8. package/lib/archive-lock.js +154 -0
  9. package/lib/archive.d.ts +736 -0
  10. package/lib/archive.js +1055 -0
  11. package/lib/body-hash/compute-body-hash.d.ts +32 -0
  12. package/lib/body-hash/compute-body-hash.js +35 -0
  13. package/lib/cache/clear-archive-cache-entry.d.ts +18 -0
  14. package/lib/cache/clear-archive-cache-entry.js +24 -0
  15. package/lib/cache/clear-archive-cache-root.d.ts +26 -0
  16. package/lib/cache/clear-archive-cache-root.js +32 -0
  17. package/lib/cache/compute-archive-cache-key.d.ts +39 -0
  18. package/lib/cache/compute-archive-cache-key.js +95 -0
  19. package/lib/cache/extract-archive-to-cache.d.ts +57 -0
  20. package/lib/cache/extract-archive-to-cache.js +326 -0
  21. package/lib/cache/get-archive-cache-root.d.ts +20 -0
  22. package/lib/cache/get-archive-cache-root.js +53 -0
  23. package/lib/cache/is-archive-cache-disabled.d.ts +24 -0
  24. package/lib/cache/is-archive-cache-disabled.js +34 -0
  25. package/lib/cache/list-archive-cache-entries.d.ts +23 -0
  26. package/lib/cache/list-archive-cache-entries.js +130 -0
  27. package/lib/cache/path-exists.d.ts +9 -0
  28. package/lib/cache/path-exists.js +18 -0
  29. package/lib/cache/resolve-archive-cache-dir.d.ts +26 -0
  30. package/lib/cache/resolve-archive-cache-dir.js +32 -0
  31. package/lib/cache/types.d.ts +33 -0
  32. package/lib/cache/types.js +1 -0
  33. package/lib/content-type/is-html-content-type.d.ts +17 -0
  34. package/lib/content-type/is-html-content-type.js +19 -0
  35. package/lib/content-type/normalize-content-type.d.ts +14 -0
  36. package/lib/content-type/normalize-content-type.js +20 -0
  37. package/lib/create-adjunct-tables.d.ts +66 -0
  38. package/lib/create-adjunct-tables.js +510 -0
  39. package/lib/create-entity-tables.d.ts +191 -0
  40. package/lib/create-entity-tables.js +387 -0
  41. package/lib/create-progress-reporter.d.ts +30 -0
  42. package/lib/create-progress-reporter.js +38 -0
  43. package/lib/create-ref-tables.d.ts +35 -0
  44. package/lib/create-ref-tables.js +185 -0
  45. package/lib/create-template-tables.d.ts +28 -0
  46. package/lib/create-template-tables.js +92 -0
  47. package/lib/database.d.ts +549 -0
  48. package/lib/database.js +803 -0
  49. package/lib/db-ops/_shared/clear-write-ref-caches.d.ts +27 -0
  50. package/lib/db-ops/_shared/clear-write-ref-caches.js +35 -0
  51. package/lib/db-ops/_shared/compress-payload.d.ts +26 -0
  52. package/lib/db-ops/_shared/compress-payload.js +30 -0
  53. package/lib/db-ops/_shared/create-write-ref-caches.d.ts +17 -0
  54. package/lib/db-ops/_shared/create-write-ref-caches.js +27 -0
  55. package/lib/db-ops/_shared/decode-json-ref.d.ts +17 -0
  56. package/lib/db-ops/_shared/decode-json-ref.js +31 -0
  57. package/lib/db-ops/_shared/keyset-paginate-by-id.d.ts +49 -0
  58. package/lib/db-ops/_shared/keyset-paginate-by-id.js +69 -0
  59. package/lib/db-ops/_shared/load-response-headers-by-set-ids.d.ts +20 -0
  60. package/lib/db-ops/_shared/load-response-headers-by-set-ids.js +53 -0
  61. package/lib/db-ops/_shared/resolve-content-item-id.d.ts +62 -0
  62. package/lib/db-ops/_shared/resolve-content-item-id.js +143 -0
  63. package/lib/db-ops/_shared/resolve-url-or-blob.d.ts +23 -0
  64. package/lib/db-ops/_shared/resolve-url-or-blob.js +29 -0
  65. package/lib/db-ops/_shared/retry-setting.d.ts +16 -0
  66. package/lib/db-ops/_shared/retry-setting.js +18 -0
  67. package/lib/db-ops/_shared/safe-parse-json.d.ts +11 -0
  68. package/lib/db-ops/_shared/safe-parse-json.js +18 -0
  69. package/lib/db-ops/_shared/types.d.ts +94 -0
  70. package/lib/db-ops/_shared/types.js +1 -0
  71. package/lib/db-ops/_shared/upsert-blob-ref.d.ts +25 -0
  72. package/lib/db-ops/_shared/upsert-blob-ref.js +48 -0
  73. package/lib/db-ops/_shared/upsert-content-type-ref.d.ts +30 -0
  74. package/lib/db-ops/_shared/upsert-content-type-ref.js +45 -0
  75. package/lib/db-ops/_shared/upsert-json-ref.d.ts +22 -0
  76. package/lib/db-ops/_shared/upsert-json-ref.js +41 -0
  77. package/lib/db-ops/_shared/upsert-response-headers.d.ts +35 -0
  78. package/lib/db-ops/_shared/upsert-response-headers.js +49 -0
  79. package/lib/db-ops/_shared/upsert-url-ref.d.ts +39 -0
  80. package/lib/db-ops/_shared/upsert-url-ref.js +62 -0
  81. package/lib/db-ops/anchors/get-anchors-on-page.d.ts +10 -0
  82. package/lib/db-ops/anchors/get-anchors-on-page.js +21 -0
  83. package/lib/db-ops/config/get-base-url.d.ts +8 -0
  84. package/lib/db-ops/config/get-base-url.js +14 -0
  85. package/lib/db-ops/config/get-config.d.ts +10 -0
  86. package/lib/db-ops/config/get-config.js +29 -0
  87. package/lib/db-ops/config/get-created-cwd.d.ts +11 -0
  88. package/lib/db-ops/config/get-created-cwd.js +13 -0
  89. package/lib/db-ops/config/get-name.d.ts +8 -0
  90. package/lib/db-ops/config/get-name.js +14 -0
  91. package/lib/db-ops/config/info-column-allowlist.d.ts +7 -0
  92. package/lib/db-ops/config/info-column-allowlist.js +29 -0
  93. package/lib/db-ops/config/info-json-columns.d.ts +5 -0
  94. package/lib/db-ops/config/info-json-columns.js +11 -0
  95. package/lib/db-ops/config/set-config.d.ts +12 -0
  96. package/lib/db-ops/config/set-config.js +21 -0
  97. package/lib/db-ops/config/update-config.d.ts +17 -0
  98. package/lib/db-ops/config/update-config.js +36 -0
  99. package/lib/db-ops/console-logs/compute-console-log-hash.d.ts +19 -0
  100. package/lib/db-ops/console-logs/compute-console-log-hash.js +30 -0
  101. package/lib/db-ops/console-logs/replace-console-logs.d.ts +34 -0
  102. package/lib/db-ops/console-logs/replace-console-logs.js +123 -0
  103. package/lib/db-ops/console-logs/stringify-console-log-args.d.ts +16 -0
  104. package/lib/db-ops/console-logs/stringify-console-log-args.js +26 -0
  105. package/lib/db-ops/console-logs/types.d.ts +38 -0
  106. package/lib/db-ops/console-logs/types.js +1 -0
  107. package/lib/db-ops/console-logs/upsert-console-log-item.d.ts +29 -0
  108. package/lib/db-ops/console-logs/upsert-console-log-item.js +52 -0
  109. package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts +18 -0
  110. package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js +23 -0
  111. package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts +12 -0
  112. package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.js +15 -0
  113. package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts +14 -0
  114. package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.js +30 -0
  115. package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.d.ts +57 -0
  116. package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.js +87 -0
  117. package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts +21 -0
  118. package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js +27 -0
  119. package/lib/db-ops/errors/insert-crawl-error.d.ts +15 -0
  120. package/lib/db-ops/errors/insert-crawl-error.js +21 -0
  121. package/lib/db-ops/errors/insert-page-error.d.ts +21 -0
  122. package/lib/db-ops/errors/insert-page-error.js +30 -0
  123. package/lib/db-ops/errors/list-dns-burned-host-candidates.d.ts +33 -0
  124. package/lib/db-ops/errors/list-dns-burned-host-candidates.js +158 -0
  125. package/lib/db-ops/html/read-page-html-blob.d.ts +17 -0
  126. package/lib/db-ops/html/read-page-html-blob.js +20 -0
  127. package/lib/db-ops/lifecycle/checkpoint.d.ts +8 -0
  128. package/lib/db-ops/lifecycle/checkpoint.js +9 -0
  129. package/lib/db-ops/lifecycle/destroy.d.ts +6 -0
  130. package/lib/db-ops/lifecycle/destroy.js +7 -0
  131. package/lib/db-ops/lifecycle/init.d.ts +58 -0
  132. package/lib/db-ops/lifecycle/init.js +122 -0
  133. package/lib/db-ops/list-reconcile/record-list-reconcile-run.d.ts +22 -0
  134. package/lib/db-ops/list-reconcile/record-list-reconcile-run.js +41 -0
  135. package/lib/db-ops/meta/get-audios-of-page.d.ts +11 -0
  136. package/lib/db-ops/meta/get-audios-of-page.js +15 -0
  137. package/lib/db-ops/meta/get-buttons-of-page.d.ts +11 -0
  138. package/lib/db-ops/meta/get-buttons-of-page.js +15 -0
  139. package/lib/db-ops/meta/get-canvases-of-page.d.ts +11 -0
  140. package/lib/db-ops/meta/get-canvases-of-page.js +15 -0
  141. package/lib/db-ops/meta/get-custom-elements-of-page.d.ts +11 -0
  142. package/lib/db-ops/meta/get-custom-elements-of-page.js +15 -0
  143. package/lib/db-ops/meta/get-headings-of-page.d.ts +11 -0
  144. package/lib/db-ops/meta/get-headings-of-page.js +15 -0
  145. package/lib/db-ops/meta/get-iframes-of-page.d.ts +11 -0
  146. package/lib/db-ops/meta/get-iframes-of-page.js +15 -0
  147. package/lib/db-ops/meta/get-jsonld-of-page.d.ts +13 -0
  148. package/lib/db-ops/meta/get-jsonld-of-page.js +27 -0
  149. package/lib/db-ops/meta/get-main-content-images-of-page.d.ts +11 -0
  150. package/lib/db-ops/meta/get-main-content-images-of-page.js +15 -0
  151. package/lib/db-ops/meta/get-main-content-tables-of-page.d.ts +11 -0
  152. package/lib/db-ops/meta/get-main-content-tables-of-page.js +15 -0
  153. package/lib/db-ops/meta/get-page-technologies-of-page.d.ts +11 -0
  154. package/lib/db-ops/meta/get-page-technologies-of-page.js +15 -0
  155. package/lib/db-ops/meta/get-technology-signals-of-page.d.ts +11 -0
  156. package/lib/db-ops/meta/get-technology-signals-of-page.js +15 -0
  157. package/lib/db-ops/meta/get-videos-of-page.d.ts +11 -0
  158. package/lib/db-ops/meta/get-videos-of-page.js +15 -0
  159. package/lib/db-ops/outages/close-network-outage.d.ts +12 -0
  160. package/lib/db-ops/outages/close-network-outage.js +15 -0
  161. package/lib/db-ops/outages/close-stale-open-network-outages.d.ts +26 -0
  162. package/lib/db-ops/outages/close-stale-open-network-outages.js +42 -0
  163. package/lib/db-ops/outages/compute-outage-clamp-timestamp.d.ts +26 -0
  164. package/lib/db-ops/outages/compute-outage-clamp-timestamp.js +31 -0
  165. package/lib/db-ops/outages/insert-network-outage.d.ts +14 -0
  166. package/lib/db-ops/outages/insert-network-outage.js +29 -0
  167. package/lib/db-ops/outages/list-network-outages.d.ts +28 -0
  168. package/lib/db-ops/outages/list-network-outages.js +42 -0
  169. package/lib/db-ops/pages/order/set-url-order.d.ts +13 -0
  170. package/lib/db-ops/pages/order/set-url-order.js +38 -0
  171. package/lib/db-ops/pages/read/build-page-query.d.ts +18 -0
  172. package/lib/db-ops/pages/read/build-page-query.js +40 -0
  173. package/lib/db-ops/pages/read/get-crawling-state.d.ts +117 -0
  174. package/lib/db-ops/pages/read/get-crawling-state.js +161 -0
  175. package/lib/db-ops/pages/read/get-existing-page-urls.d.ts +15 -0
  176. package/lib/db-ops/pages/read/get-existing-page-urls.js +30 -0
  177. package/lib/db-ops/pages/read/get-page-count.d.ts +12 -0
  178. package/lib/db-ops/pages/read/get-page-count.js +21 -0
  179. package/lib/db-ops/pages/read/get-page-source-by-url.d.ts +24 -0
  180. package/lib/db-ops/pages/read/get-page-source-by-url.js +28 -0
  181. package/lib/db-ops/pages/read/get-pages-with-rels.d.ts +38 -0
  182. package/lib/db-ops/pages/read/get-pages-with-rels.js +107 -0
  183. package/lib/db-ops/pages/read/get-pages.d.ts +11 -0
  184. package/lib/db-ops/pages/read/get-pages.js +51 -0
  185. package/lib/db-ops/pages/read/get-scraped-html-page-count.d.ts +18 -0
  186. package/lib/db-ops/pages/read/get-scraped-html-page-count.js +25 -0
  187. package/lib/db-ops/pages/read/reconstruct-page-rows.d.ts +31 -0
  188. package/lib/db-ops/pages/read/reconstruct-page-rows.js +32 -0
  189. package/lib/db-ops/pages/reset/clear-page-derived-rows.d.ts +20 -0
  190. package/lib/db-ops/pages/reset/clear-page-derived-rows.js +37 -0
  191. package/lib/db-ops/pages/reset/repromote-external-pages.d.ts +30 -0
  192. package/lib/db-ops/pages/reset/repromote-external-pages.js +91 -0
  193. package/lib/db-ops/pages/reset/reset-failed-pages.d.ts +88 -0
  194. package/lib/db-ops/pages/reset/reset-failed-pages.js +216 -0
  195. package/lib/db-ops/pages/reset/reset-pages-by-urls.d.ts +80 -0
  196. package/lib/db-ops/pages/reset/reset-pages-by-urls.js +151 -0
  197. package/lib/db-ops/pages/reset/retryable-image-scan-codes.d.ts +24 -0
  198. package/lib/db-ops/pages/reset/retryable-image-scan-codes.js +24 -0
  199. package/lib/db-ops/pages/write/insert-audios.d.ts +14 -0
  200. package/lib/db-ops/pages/write/insert-audios.js +25 -0
  201. package/lib/db-ops/pages/write/insert-buttons.d.ts +14 -0
  202. package/lib/db-ops/pages/write/insert-buttons.js +29 -0
  203. package/lib/db-ops/pages/write/insert-canvases.d.ts +14 -0
  204. package/lib/db-ops/pages/write/insert-canvases.js +26 -0
  205. package/lib/db-ops/pages/write/insert-custom-elements.d.ts +27 -0
  206. package/lib/db-ops/pages/write/insert-custom-elements.js +40 -0
  207. package/lib/db-ops/pages/write/insert-headings.d.ts +15 -0
  208. package/lib/db-ops/pages/write/insert-headings.js +27 -0
  209. package/lib/db-ops/pages/write/insert-iframes.d.ts +14 -0
  210. package/lib/db-ops/pages/write/insert-iframes.js +28 -0
  211. package/lib/db-ops/pages/write/insert-inventory-content-items.d.ts +38 -0
  212. package/lib/db-ops/pages/write/insert-inventory-content-items.js +60 -0
  213. package/lib/db-ops/pages/write/insert-inventory-seeds.d.ts +36 -0
  214. package/lib/db-ops/pages/write/insert-inventory-seeds.js +48 -0
  215. package/lib/db-ops/pages/write/insert-inventory-skipped-pages.d.ts +42 -0
  216. package/lib/db-ops/pages/write/insert-inventory-skipped-pages.js +56 -0
  217. package/lib/db-ops/pages/write/insert-jsonld.d.ts +17 -0
  218. package/lib/db-ops/pages/write/insert-jsonld.js +49 -0
  219. package/lib/db-ops/pages/write/insert-main-content-images.d.ts +20 -0
  220. package/lib/db-ops/pages/write/insert-main-content-images.js +32 -0
  221. package/lib/db-ops/pages/write/insert-main-content-tables.d.ts +14 -0
  222. package/lib/db-ops/pages/write/insert-main-content-tables.js +29 -0
  223. package/lib/db-ops/pages/write/insert-page.d.ts +56 -0
  224. package/lib/db-ops/pages/write/insert-page.js +259 -0
  225. package/lib/db-ops/pages/write/insert-technologies.d.ts +25 -0
  226. package/lib/db-ops/pages/write/insert-technologies.js +45 -0
  227. package/lib/db-ops/pages/write/insert-videos.d.ts +14 -0
  228. package/lib/db-ops/pages/write/insert-videos.js +28 -0
  229. package/lib/db-ops/pages/write/link-redirect-sources.d.ts +42 -0
  230. package/lib/db-ops/pages/write/link-redirect-sources.js +101 -0
  231. package/lib/db-ops/pages/write/record-redirect.d.ts +35 -0
  232. package/lib/db-ops/pages/write/record-redirect.js +100 -0
  233. package/lib/db-ops/pages/write/set-skipped-page.d.ts +13 -0
  234. package/lib/db-ops/pages/write/set-skipped-page.js +24 -0
  235. package/lib/db-ops/pages/write/update-page.d.ts +40 -0
  236. package/lib/db-ops/pages/write/update-page.js +419 -0
  237. package/lib/db-ops/pages/write/write-page-html-blob.d.ts +19 -0
  238. package/lib/db-ops/pages/write/write-page-html-blob.js +41 -0
  239. package/lib/db-ops/referrers/get-redirects-for-pages.d.ts +9 -0
  240. package/lib/db-ops/referrers/get-redirects-for-pages.js +15 -0
  241. package/lib/db-ops/referrers/get-referrers-of-page.d.ts +17 -0
  242. package/lib/db-ops/referrers/get-referrers-of-page.js +32 -0
  243. package/lib/db-ops/referrers/get-referrers-of-resource.d.ts +8 -0
  244. package/lib/db-ops/referrers/get-referrers-of-resource.js +15 -0
  245. package/lib/db-ops/resources/build-resource-query.d.ts +25 -0
  246. package/lib/db-ops/resources/build-resource-query.js +29 -0
  247. package/lib/db-ops/resources/get-existing-resource-urls.d.ts +9 -0
  248. package/lib/db-ops/resources/get-existing-resource-urls.js +24 -0
  249. package/lib/db-ops/resources/get-resource-by-url.d.ts +13 -0
  250. package/lib/db-ops/resources/get-resource-by-url.js +22 -0
  251. package/lib/db-ops/resources/get-resource-url-list.d.ts +17 -0
  252. package/lib/db-ops/resources/get-resource-url-list.js +26 -0
  253. package/lib/db-ops/resources/get-resources.d.ts +8 -0
  254. package/lib/db-ops/resources/get-resources.js +11 -0
  255. package/lib/db-ops/resources/insert-inventory-resources.d.ts +24 -0
  256. package/lib/db-ops/resources/insert-inventory-resources.js +64 -0
  257. package/lib/db-ops/resources/insert-resource-referrers.d.ts +15 -0
  258. package/lib/db-ops/resources/insert-resource-referrers.js +54 -0
  259. package/lib/db-ops/resources/insert-resource.d.ts +34 -0
  260. package/lib/db-ops/resources/insert-resource.js +73 -0
  261. package/lib/db-ops/resources/reconstruct-resource-rows.d.ts +26 -0
  262. package/lib/db-ops/resources/reconstruct-resource-rows.js +30 -0
  263. package/lib/db-ops/templates/assign-template-labels.d.ts +60 -0
  264. package/lib/db-ops/templates/assign-template-labels.js +187 -0
  265. package/lib/db-ops/templates/replace-page-templates.d.ts +44 -0
  266. package/lib/db-ops/templates/replace-page-templates.js +174 -0
  267. package/lib/db-ops/templates/serialize-template-cluster-reason.d.ts +16 -0
  268. package/lib/db-ops/templates/serialize-template-cluster-reason.js +17 -0
  269. package/lib/db-ops/templates/types.d.ts +103 -0
  270. package/lib/db-ops/templates/types.js +1 -0
  271. package/lib/debug.d.ts +8 -0
  272. package/lib/debug.js +9 -0
  273. package/lib/decode-html-blob.d.ts +20 -0
  274. package/lib/decode-html-blob.js +23 -0
  275. package/lib/decode-page-html-blob-bytes.d.ts +25 -0
  276. package/lib/decode-page-html-blob-bytes.js +29 -0
  277. package/lib/decode-stored-blob-bytes.d.ts +25 -0
  278. package/lib/decode-stored-blob-bytes.js +47 -0
  279. package/lib/derive-lineage-from-parent.d.ts +37 -0
  280. package/lib/derive-lineage-from-parent.js +42 -0
  281. package/lib/drop-legacy-tables.d.ts +45 -0
  282. package/lib/drop-legacy-tables.js +56 -0
  283. package/lib/error-kind/classify-error-kind.d.ts +20 -0
  284. package/lib/error-kind/classify-error-kind.js +136 -0
  285. package/lib/error-kind/network-related-error-kinds.d.ts +23 -0
  286. package/lib/error-kind/network-related-error-kinds.js +28 -0
  287. package/lib/error-kind/permanent-error-kinds.d.ts +48 -0
  288. package/lib/error-kind/permanent-error-kinds.js +54 -0
  289. package/lib/error-kind/types.d.ts +54 -0
  290. package/lib/error-kind/types.js +1 -0
  291. package/lib/filesystem/append-text.d.ts +9 -0
  292. package/lib/filesystem/append-text.js +14 -0
  293. package/lib/filesystem/copy-dir-sync.d.ts +6 -0
  294. package/lib/filesystem/copy-dir-sync.js +9 -0
  295. package/lib/filesystem/copy-dir.d.ts +7 -0
  296. package/lib/filesystem/copy-dir.js +13 -0
  297. package/lib/filesystem/copy-file-with-progress.d.ts +20 -0
  298. package/lib/filesystem/copy-file-with-progress.js +34 -0
  299. package/lib/filesystem/exists.d.ts +6 -0
  300. package/lib/filesystem/exists.js +9 -0
  301. package/lib/filesystem/get-file-list.d.ts +8 -0
  302. package/lib/filesystem/get-file-list.js +12 -0
  303. package/lib/filesystem/is-dir.d.ts +6 -0
  304. package/lib/filesystem/is-dir.js +10 -0
  305. package/lib/filesystem/mkdir.d.ts +8 -0
  306. package/lib/filesystem/mkdir.js +15 -0
  307. package/lib/filesystem/output-binary.d.ts +12 -0
  308. package/lib/filesystem/output-binary.js +17 -0
  309. package/lib/filesystem/output-json.d.ts +9 -0
  310. package/lib/filesystem/output-json.js +14 -0
  311. package/lib/filesystem/output-text.d.ts +11 -0
  312. package/lib/filesystem/output-text.js +32 -0
  313. package/lib/filesystem/parse-pax-path.d.ts +18 -0
  314. package/lib/filesystem/parse-pax-path.js +45 -0
  315. package/lib/filesystem/parse-tar-size-field.d.ts +13 -0
  316. package/lib/filesystem/parse-tar-size-field.js +37 -0
  317. package/lib/filesystem/peek-tar-top-dir.d.ts +35 -0
  318. package/lib/filesystem/peek-tar-top-dir.js +215 -0
  319. package/lib/filesystem/read-json.d.ts +7 -0
  320. package/lib/filesystem/read-json.js +11 -0
  321. package/lib/filesystem/read-text.d.ts +6 -0
  322. package/lib/filesystem/read-text.js +10 -0
  323. package/lib/filesystem/readline.d.ts +11 -0
  324. package/lib/filesystem/readline.js +26 -0
  325. package/lib/filesystem/remove.d.ts +5 -0
  326. package/lib/filesystem/remove.js +10 -0
  327. package/lib/filesystem/rename.d.ts +16 -0
  328. package/lib/filesystem/rename.js +48 -0
  329. package/lib/filesystem/tar.d.ts +23 -0
  330. package/lib/filesystem/tar.js +78 -0
  331. package/lib/filesystem/untar.d.ts +33 -0
  332. package/lib/filesystem/untar.js +84 -0
  333. package/lib/get-failed-page-messages.d.ts +56 -0
  334. package/lib/get-failed-page-messages.js +146 -0
  335. package/lib/get-json.d.ts +10 -0
  336. package/lib/get-json.js +25 -0
  337. package/lib/html-snapshot/match-selector-html.d.ts +25 -0
  338. package/lib/html-snapshot/match-selector-html.js +28 -0
  339. package/lib/html-snapshot/match-selector-snapshots.d.ts +33 -0
  340. package/lib/html-snapshot/match-selector-snapshots.js +33 -0
  341. package/lib/html-snapshot/search-html-document.d.ts +29 -0
  342. package/lib/html-snapshot/search-html-document.js +27 -0
  343. package/lib/html-snapshot/search-html-snapshots.d.ts +36 -0
  344. package/lib/html-snapshot/search-html-snapshots.js +35 -0
  345. package/lib/html-snapshot/types.d.ts +111 -0
  346. package/lib/html-snapshot/types.js +7 -0
  347. package/lib/html-snapshot/validate-search-pattern.d.ts +15 -0
  348. package/lib/html-snapshot/validate-search-pattern.js +17 -0
  349. package/lib/init-schema.d.ts +48 -0
  350. package/lib/init-schema.js +124 -0
  351. package/lib/is-inventory-source.d.ts +21 -0
  352. package/lib/is-inventory-source.js +22 -0
  353. package/lib/is-within-outage-window.d.ts +49 -0
  354. package/lib/is-within-outage-window.js +33 -0
  355. package/lib/isolated-cluster/create-isolated-cluster-run.d.ts +24 -0
  356. package/lib/isolated-cluster/create-isolated-cluster-run.js +26 -0
  357. package/lib/libsql-dialect.d.ts +115 -0
  358. package/lib/libsql-dialect.js +116 -0
  359. package/lib/limited-page-ids.d.ts +9 -0
  360. package/lib/limited-page-ids.js +18 -0
  361. package/lib/meta/assert-compatible-version.d.ts +60 -0
  362. package/lib/meta/assert-compatible-version.js +93 -0
  363. package/lib/meta/classify-jsonld-type.d.ts +23 -0
  364. package/lib/meta/classify-jsonld-type.js +43 -0
  365. package/lib/meta/compare-semver.d.ts +23 -0
  366. package/lib/meta/compare-semver.js +51 -0
  367. package/lib/meta/compute-main-contents-denormalized.d.ts +41 -0
  368. package/lib/meta/compute-main-contents-denormalized.js +88 -0
  369. package/lib/meta/compute-page-denormalized.d.ts +20 -0
  370. package/lib/meta/compute-page-denormalized.js +35 -0
  371. package/lib/meta/derive-flat-from-meta.d.ts +34 -0
  372. package/lib/meta/derive-flat-from-meta.js +158 -0
  373. package/lib/meta/derive-meta-extras.d.ts +20 -0
  374. package/lib/meta/derive-meta-extras.js +23 -0
  375. package/lib/meta/extract-tags-for-archive.d.ts +17 -0
  376. package/lib/meta/extract-tags-for-archive.js +36 -0
  377. package/lib/meta/parse-main-content-class-list.d.ts +19 -0
  378. package/lib/meta/parse-main-content-class-list.js +34 -0
  379. package/lib/meta/summarize-jsonld.d.ts +17 -0
  380. package/lib/meta/summarize-jsonld.js +29 -0
  381. package/lib/meta/technologies/combine-technology-confidence.d.ts +46 -0
  382. package/lib/meta/technologies/combine-technology-confidence.js +93 -0
  383. package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.d.ts +45 -0
  384. package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.js +53 -0
  385. package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.d.ts +33 -0
  386. package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.js +46 -0
  387. package/lib/meta/technologies/extract-technologies-for-archive.d.ts +33 -0
  388. package/lib/meta/technologies/extract-technologies-for-archive.js +30 -0
  389. package/lib/meta/technologies/match-js-license-comment-signal.d.ts +21 -0
  390. package/lib/meta/technologies/match-js-license-comment-signal.js +23 -0
  391. package/lib/meta/technologies/match-structural-technology-signals.d.ts +26 -0
  392. package/lib/meta/technologies/match-structural-technology-signals.js +48 -0
  393. package/lib/meta/technologies/normalize-wappalyzer-entries.d.ts +25 -0
  394. package/lib/meta/technologies/normalize-wappalyzer-entries.js +35 -0
  395. package/lib/meta/technologies/technology-signal-definitions.d.ts +44 -0
  396. package/lib/meta/technologies/technology-signal-definitions.js +60 -0
  397. package/lib/meta/technologies/types.d.ts +70 -0
  398. package/lib/meta/technologies/types.js +13 -0
  399. package/lib/meta/types.d.ts +1490 -0
  400. package/lib/meta/types.js +65 -0
  401. package/lib/migrate-content-items-alias-of-id.d.ts +52 -0
  402. package/lib/migrate-content-items-alias-of-id.js +69 -0
  403. package/lib/migrate-content-items-dedupe-cap-event-id.d.ts +46 -0
  404. package/lib/migrate-content-items-dedupe-cap-event-id.js +62 -0
  405. package/lib/migrate-content-items-is-metadata-only.d.ts +44 -0
  406. package/lib/migrate-content-items-is-metadata-only.js +60 -0
  407. package/lib/migrate-entity-tables.d.ts +55 -0
  408. package/lib/migrate-entity-tables.js +66 -0
  409. package/lib/migrate-info-created-cwd.d.ts +17 -0
  410. package/lib/migrate-info-created-cwd.js +36 -0
  411. package/lib/migrate-info-main-content-selector.d.ts +16 -0
  412. package/lib/migrate-info-main-content-selector.js +35 -0
  413. package/lib/migrate-info-request-header-names.d.ts +16 -0
  414. package/lib/migrate-info-request-header-names.js +35 -0
  415. package/lib/migrate-info-roots.d.ts +20 -0
  416. package/lib/migrate-info-roots.js +52 -0
  417. package/lib/migrate-inventory-runs-to-list-reconcile-runs.d.ts +34 -0
  418. package/lib/migrate-inventory-runs-to-list-reconcile-runs.js +51 -0
  419. package/lib/migrate-list-reconcile-runs-exclude-skipped.d.ts +25 -0
  420. package/lib/migrate-list-reconcile-runs-exclude-skipped.js +44 -0
  421. package/lib/migrate-list-reconcile-runs-invalid-skipped.d.ts +16 -0
  422. package/lib/migrate-list-reconcile-runs-invalid-skipped.js +35 -0
  423. package/lib/migrate-main-contents-columns.d.ts +26 -0
  424. package/lib/migrate-main-contents-columns.js +61 -0
  425. package/lib/migrate-page-meta-body-hash.d.ts +44 -0
  426. package/lib/migrate-page-meta-body-hash.js +63 -0
  427. package/lib/migrate-page-meta-console-error-count.d.ts +27 -0
  428. package/lib/migrate-page-meta-console-error-count.js +45 -0
  429. package/lib/migrate-page-meta-custom-element-count.d.ts +33 -0
  430. package/lib/migrate-page-meta-custom-element-count.js +51 -0
  431. package/lib/migrate-page-meta-image-scan.d.ts +27 -0
  432. package/lib/migrate-page-meta-image-scan.js +52 -0
  433. package/lib/migrate-page-tags-to-page-technologies.d.ts +39 -0
  434. package/lib/migrate-page-tags-to-page-technologies.js +79 -0
  435. package/lib/migrate-ref-tables.d.ts +25 -0
  436. package/lib/migrate-ref-tables.js +38 -0
  437. package/lib/native-executor/check-native-executor-config.d.ts +17 -0
  438. package/lib/native-executor/check-native-executor-config.js +20 -0
  439. package/lib/page-meta-column-maps.d.ts +32 -0
  440. package/lib/page-meta-column-maps.js +43 -0
  441. package/lib/page.d.ts +546 -0
  442. package/lib/page.js +764 -0
  443. package/lib/peek-archive-lock.d.ts +40 -0
  444. package/lib/peek-archive-lock.js +62 -0
  445. package/lib/populate-entity-tables/collapse-anchor-rows.d.ts +41 -0
  446. package/lib/populate-entity-tables/collapse-anchor-rows.js +87 -0
  447. package/lib/populate-entity-tables/derive-dom-path.d.ts +35 -0
  448. package/lib/populate-entity-tables/derive-dom-path.js +72 -0
  449. package/lib/populate-entity-tables/is-blob-ref-value.d.ts +16 -0
  450. package/lib/populate-entity-tables/is-blob-ref-value.js +19 -0
  451. package/lib/populate-entity-tables/match-images-to-dom-paths.d.ts +66 -0
  452. package/lib/populate-entity-tables/match-images-to-dom-paths.js +96 -0
  453. package/lib/populate-entity-tables/populate-anchor-edges.d.ts +33 -0
  454. package/lib/populate-entity-tables/populate-anchor-edges.js +153 -0
  455. package/lib/populate-entity-tables/populate-content-items.d.ts +40 -0
  456. package/lib/populate-entity-tables/populate-content-items.js +141 -0
  457. package/lib/populate-entity-tables/populate-entities.d.ts +81 -0
  458. package/lib/populate-entity-tables/populate-entities.js +110 -0
  459. package/lib/populate-entity-tables/populate-image-items.d.ts +91 -0
  460. package/lib/populate-entity-tables/populate-image-items.js +226 -0
  461. package/lib/populate-entity-tables/populate-page-meta.d.ts +33 -0
  462. package/lib/populate-entity-tables/populate-page-meta.js +267 -0
  463. package/lib/populate-entity-tables/populate-resource-items.d.ts +22 -0
  464. package/lib/populate-entity-tables/populate-resource-items.js +114 -0
  465. package/lib/populate-entity-tables/populate-resource-ref-edges.d.ts +31 -0
  466. package/lib/populate-entity-tables/populate-resource-ref-edges.js +33 -0
  467. package/lib/populate-entity-tables/resolve-blob-refs.d.ts +31 -0
  468. package/lib/populate-entity-tables/resolve-blob-refs.js +100 -0
  469. package/lib/populate-entity-tables/resolve-content-type-refs.d.ts +22 -0
  470. package/lib/populate-entity-tables/resolve-content-type-refs.js +27 -0
  471. package/lib/populate-entity-tables/resolve-header-sets.d.ts +49 -0
  472. package/lib/populate-entity-tables/resolve-header-sets.js +122 -0
  473. package/lib/populate-entity-tables/resolve-json-refs.d.ts +25 -0
  474. package/lib/populate-entity-tables/resolve-json-refs.js +67 -0
  475. package/lib/populate-entity-tables/resolve-text-refs.d.ts +30 -0
  476. package/lib/populate-entity-tables/resolve-text-refs.js +61 -0
  477. package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.d.ts +21 -0
  478. package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.js +27 -0
  479. package/lib/populate-entity-tables/resolve-url-refs.d.ts +33 -0
  480. package/lib/populate-entity-tables/resolve-url-refs.js +60 -0
  481. package/lib/populate-entity-tables/test-utils/count-rows.d.ts +17 -0
  482. package/lib/populate-entity-tables/test-utils/count-rows.js +20 -0
  483. package/lib/populate-entity-tables/test-utils/seed-content-items.d.ts +25 -0
  484. package/lib/populate-entity-tables/test-utils/seed-content-items.js +42 -0
  485. package/lib/populate-entity-tables/test-utils/setup-entities-db.d.ts +30 -0
  486. package/lib/populate-entity-tables/test-utils/setup-entities-db.js +175 -0
  487. package/lib/populate-entity-tables/types.d.ts +157 -0
  488. package/lib/populate-entity-tables/types.js +12 -0
  489. package/lib/populate-entity-tables/upsert-text-refs.d.ts +38 -0
  490. package/lib/populate-entity-tables/upsert-text-refs.js +78 -0
  491. package/lib/populate-ref-tables/classify-content-type.d.ts +16 -0
  492. package/lib/populate-ref-tables/classify-content-type.js +52 -0
  493. package/lib/populate-ref-tables/compute-content-hash.d.ts +22 -0
  494. package/lib/populate-ref-tables/compute-content-hash.js +26 -0
  495. package/lib/populate-ref-tables/compute-header-flags.d.ts +16 -0
  496. package/lib/populate-ref-tables/compute-header-flags.js +70 -0
  497. package/lib/populate-ref-tables/content-type-rules.d.ts +38 -0
  498. package/lib/populate-ref-tables/content-type-rules.js +133 -0
  499. package/lib/populate-ref-tables/create-header-table-caches.d.ts +25 -0
  500. package/lib/populate-ref-tables/create-header-table-caches.js +49 -0
  501. package/lib/populate-ref-tables/data-uri-url-refs-limit.d.ts +15 -0
  502. package/lib/populate-ref-tables/data-uri-url-refs-limit.js +15 -0
  503. package/lib/populate-ref-tables/decode-data-uri.d.ts +21 -0
  504. package/lib/populate-ref-tables/decode-data-uri.js +126 -0
  505. package/lib/populate-ref-tables/decompose-header-set.d.ts +29 -0
  506. package/lib/populate-ref-tables/decompose-header-set.js +157 -0
  507. package/lib/populate-ref-tables/decompose-url.d.ts +25 -0
  508. package/lib/populate-ref-tables/decompose-url.js +70 -0
  509. package/lib/populate-ref-tables/header-stability.d.ts +19 -0
  510. package/lib/populate-ref-tables/header-stability.js +22 -0
  511. package/lib/populate-ref-tables/header-value-cache-key.d.ts +17 -0
  512. package/lib/populate-ref-tables/header-value-cache-key.js +19 -0
  513. package/lib/populate-ref-tables/normalize-mime.d.ts +24 -0
  514. package/lib/populate-ref-tables/normalize-mime.js +36 -0
  515. package/lib/populate-ref-tables/populate-blob-refs.d.ts +38 -0
  516. package/lib/populate-ref-tables/populate-blob-refs.js +134 -0
  517. package/lib/populate-ref-tables/populate-content-type-refs.d.ts +27 -0
  518. package/lib/populate-ref-tables/populate-content-type-refs.js +70 -0
  519. package/lib/populate-ref-tables/populate-header-tables.d.ts +35 -0
  520. package/lib/populate-ref-tables/populate-header-tables.js +80 -0
  521. package/lib/populate-ref-tables/populate-json-refs.d.ts +29 -0
  522. package/lib/populate-ref-tables/populate-json-refs.js +101 -0
  523. package/lib/populate-ref-tables/populate-refs.d.ts +51 -0
  524. package/lib/populate-ref-tables/populate-refs.js +62 -0
  525. package/lib/populate-ref-tables/populate-text-refs.d.ts +32 -0
  526. package/lib/populate-ref-tables/populate-text-refs.js +133 -0
  527. package/lib/populate-ref-tables/populate-url-refs.d.ts +28 -0
  528. package/lib/populate-ref-tables/populate-url-refs.js +148 -0
  529. package/lib/populate-ref-tables/test-utils/count-rows.d.ts +15 -0
  530. package/lib/populate-ref-tables/test-utils/count-rows.js +17 -0
  531. package/lib/populate-ref-tables/types.d.ts +197 -0
  532. package/lib/populate-ref-tables/types.js +7 -0
  533. package/lib/populate-ref-tables/upsert-one-header-set.d.ts +34 -0
  534. package/lib/populate-ref-tables/upsert-one-header-set.js +208 -0
  535. package/lib/populate-ref-tables/volatile-header-names.d.ts +20 -0
  536. package/lib/populate-ref-tables/volatile-header-names.js +33 -0
  537. package/lib/redirect-table.d.ts +9 -0
  538. package/lib/redirect-table.js +28 -0
  539. package/lib/release-finished-statements.d.ts +42 -0
  540. package/lib/release-finished-statements.js +45 -0
  541. package/lib/resolve-redirect-chain.d.ts +33 -0
  542. package/lib/resolve-redirect-chain.js +27 -0
  543. package/lib/resource.d.ts +46 -0
  544. package/lib/resource.js +62 -0
  545. package/lib/retarget-legacy-fk-tables.d.ts +61 -0
  546. package/lib/retarget-legacy-fk-tables.js +137 -0
  547. package/lib/safe-path.d.ts +9 -0
  548. package/lib/safe-path.js +17 -0
  549. package/lib/scope/build-scope-map.d.ts +21 -0
  550. package/lib/scope/build-scope-map.js +32 -0
  551. package/lib/scope/find-scope-entry.d.ts +25 -0
  552. package/lib/scope/find-scope-entry.js +45 -0
  553. package/lib/template-classification/classify-archive-page-templates.d.ts +42 -0
  554. package/lib/template-classification/classify-archive-page-templates.js +70 -0
  555. package/lib/template-classification/classify-page-templates.d.ts +33 -0
  556. package/lib/template-classification/classify-page-templates.js +60 -0
  557. package/lib/template-classification/collect-page-ids-with-html.d.ts +24 -0
  558. package/lib/template-classification/collect-page-ids-with-html.js +30 -0
  559. package/lib/template-classification/collect-page-stylesheet-urls.d.ts +32 -0
  560. package/lib/template-classification/collect-page-stylesheet-urls.js +132 -0
  561. package/lib/template-classification/create-content-root-hint.d.ts +41 -0
  562. package/lib/template-classification/create-content-root-hint.js +51 -0
  563. package/lib/template-classification/create-page-cluster-factory.d.ts +36 -0
  564. package/lib/template-classification/create-page-cluster-factory.js +52 -0
  565. package/lib/template-classification/load-page-cluster-candidates.d.ts +32 -0
  566. package/lib/template-classification/load-page-cluster-candidates.js +93 -0
  567. package/lib/template-classification/page-cluster/resolve-page-cluster-keys.d.ts +34 -0
  568. package/lib/template-classification/page-cluster/resolve-page-cluster-keys.js +236 -0
  569. package/lib/template-classification/page-cluster/types.d.ts +123 -0
  570. package/lib/template-classification/page-cluster/types.js +1 -0
  571. package/lib/template-classification/types.d.ts +234 -0
  572. package/lib/template-classification/types.js +1 -0
  573. package/lib/test-utils/build-full-schema-test-db.d.ts +34 -0
  574. package/lib/test-utils/build-full-schema-test-db.js +74 -0
  575. package/lib/test-utils/compare-strings.d.ts +12 -0
  576. package/lib/test-utils/compare-strings.js +14 -0
  577. package/lib/test-utils/create-legacy-analysis-tables.d.ts +13 -0
  578. package/lib/test-utils/create-legacy-analysis-tables.js +37 -0
  579. package/lib/test-utils/fk-parent-tables.d.ts +15 -0
  580. package/lib/test-utils/fk-parent-tables.js +19 -0
  581. package/lib/test-utils/seed-content-item.d.ts +35 -0
  582. package/lib/test-utils/seed-content-item.js +42 -0
  583. package/lib/test-utils/setup-legacy-fk-db.d.ts +34 -0
  584. package/lib/test-utils/setup-legacy-fk-db.js +271 -0
  585. package/lib/transfer/attach-source-database.d.ts +31 -0
  586. package/lib/transfer/attach-source-database.js +40 -0
  587. package/lib/transfer/build-split-config.d.ts +30 -0
  588. package/lib/transfer/build-split-config.js +42 -0
  589. package/lib/transfer/clear-replaced-page-rows.d.ts +23 -0
  590. package/lib/transfer/clear-replaced-page-rows.js +61 -0
  591. package/lib/transfer/concat-archives.d.ts +55 -0
  592. package/lib/transfer/concat-archives.js +95 -0
  593. package/lib/transfer/concat-source-transfer-phases.d.ts +10 -0
  594. package/lib/transfer/concat-source-transfer-phases.js +17 -0
  595. package/lib/transfer/content-item-rank-sql.d.ts +21 -0
  596. package/lib/transfer/content-item-rank-sql.js +23 -0
  597. package/lib/transfer/copy-anchor-edges.d.ts +28 -0
  598. package/lib/transfer/copy-anchor-edges.js +42 -0
  599. package/lib/transfer/copy-dictionaries-for-concat.d.ts +39 -0
  600. package/lib/transfer/copy-dictionaries-for-concat.js +81 -0
  601. package/lib/transfer/copy-dictionaries-for-split.d.ts +29 -0
  602. package/lib/transfer/copy-dictionaries-for-split.js +224 -0
  603. package/lib/transfer/copy-header-set-children.d.ts +27 -0
  604. package/lib/transfer/copy-header-set-children.js +51 -0
  605. package/lib/transfer/copy-image-items.d.ts +21 -0
  606. package/lib/transfer/copy-image-items.js +62 -0
  607. package/lib/transfer/copy-inventory-source-lists.d.ts +30 -0
  608. package/lib/transfer/copy-inventory-source-lists.js +61 -0
  609. package/lib/transfer/copy-journal-tables.d.ts +30 -0
  610. package/lib/transfer/copy-journal-tables.js +56 -0
  611. package/lib/transfer/copy-page-console-logs.d.ts +16 -0
  612. package/lib/transfer/copy-page-console-logs.js +27 -0
  613. package/lib/transfer/copy-page-html-blobs.d.ts +31 -0
  614. package/lib/transfer/copy-page-html-blobs.js +43 -0
  615. package/lib/transfer/copy-page-meta.d.ts +27 -0
  616. package/lib/transfer/copy-page-meta.js +68 -0
  617. package/lib/transfer/copy-page-template-clusters.d.ts +31 -0
  618. package/lib/transfer/copy-page-template-clusters.js +56 -0
  619. package/lib/transfer/copy-page-template-labels.d.ts +34 -0
  620. package/lib/transfer/copy-page-template-labels.js +88 -0
  621. package/lib/transfer/copy-resource-items.d.ts +29 -0
  622. package/lib/transfer/copy-resource-items.js +89 -0
  623. package/lib/transfer/copy-resource-ref-edges.d.ts +20 -0
  624. package/lib/transfer/copy-resource-ref-edges.js +33 -0
  625. package/lib/transfer/copy-simple-page-scoped-tables.d.ts +15 -0
  626. package/lib/transfer/copy-simple-page-scoped-tables.js +34 -0
  627. package/lib/transfer/count-external-pages-in-scope.d.ts +26 -0
  628. package/lib/transfer/count-external-pages-in-scope.js +40 -0
  629. package/lib/transfer/create-transfer-temp-tables.d.ts +30 -0
  630. package/lib/transfer/create-transfer-temp-tables.js +59 -0
  631. package/lib/transfer/dictionary-copy-specs.d.ts +32 -0
  632. package/lib/transfer/dictionary-copy-specs.js +28 -0
  633. package/lib/transfer/drop-transfer-temp-tables.d.ts +9 -0
  634. package/lib/transfer/drop-transfer-temp-tables.js +18 -0
  635. package/lib/transfer/flatten-redirect-chains.d.ts +44 -0
  636. package/lib/transfer/flatten-redirect-chains.js +74 -0
  637. package/lib/transfer/insert-content-items.d.ts +35 -0
  638. package/lib/transfer/insert-content-items.js +116 -0
  639. package/lib/transfer/list-transfer-columns.d.ts +35 -0
  640. package/lib/transfer/list-transfer-columns.js +45 -0
  641. package/lib/transfer/mark-in-scope-content-items.d.ts +31 -0
  642. package/lib/transfer/mark-in-scope-content-items.js +54 -0
  643. package/lib/transfer/merge-archive-configs.d.ts +51 -0
  644. package/lib/transfer/merge-archive-configs.js +85 -0
  645. package/lib/transfer/plan-content-items-for-concat.d.ts +33 -0
  646. package/lib/transfer/plan-content-items-for-concat.js +51 -0
  647. package/lib/transfer/plan-content-items-for-split.d.ts +42 -0
  648. package/lib/transfer/plan-content-items-for-split.js +83 -0
  649. package/lib/transfer/plan-resource-items.d.ts +40 -0
  650. package/lib/transfer/plan-resource-items.js +69 -0
  651. package/lib/transfer/quote-transfer-identifier.d.ts +18 -0
  652. package/lib/transfer/quote-transfer-identifier.js +20 -0
  653. package/lib/transfer/reclassify-resource-externality.d.ts +37 -0
  654. package/lib/transfer/reclassify-resource-externality.js +66 -0
  655. package/lib/transfer/remap-redirect-dest-ids.d.ts +34 -0
  656. package/lib/transfer/remap-redirect-dest-ids.js +51 -0
  657. package/lib/transfer/replace-content-items.d.ts +39 -0
  658. package/lib/transfer/replace-content-items.js +83 -0
  659. package/lib/transfer/simple-page-scoped-tables.d.ts +25 -0
  660. package/lib/transfer/simple-page-scoped-tables.js +38 -0
  661. package/lib/transfer/split-archive.d.ts +55 -0
  662. package/lib/transfer/split-archive.js +98 -0
  663. package/lib/transfer/split-source-transfer-phases.d.ts +11 -0
  664. package/lib/transfer/split-source-transfer-phases.js +18 -0
  665. package/lib/transfer/transfer-action.d.ts +34 -0
  666. package/lib/transfer/transfer-action.js +32 -0
  667. package/lib/transfer/transfer-archive-rows.d.ts +65 -0
  668. package/lib/transfer/transfer-archive-rows.js +156 -0
  669. package/lib/transfer/transfer-dictionary-tables.d.ts +12 -0
  670. package/lib/transfer/transfer-dictionary-tables.js +22 -0
  671. package/lib/transfer/transfer-post-phases.d.ts +10 -0
  672. package/lib/transfer/transfer-post-phases.js +13 -0
  673. package/lib/transfer/transfer-source-alias.d.ts +9 -0
  674. package/lib/transfer/transfer-source-alias.js +9 -0
  675. package/lib/transfer/types.d.ts +111 -0
  676. package/lib/transfer/types.js +35 -0
  677. package/lib/types.d.ts +772 -0
  678. package/lib/types.js +1 -0
  679. package/lib/url-alias/compute-tier-a-alias-key.d.ts +44 -0
  680. package/lib/url-alias/compute-tier-a-alias-key.js +47 -0
  681. package/lib/url-alias/compute-tier-b-alias-key.d.ts +23 -0
  682. package/lib/url-alias/compute-tier-b-alias-key.js +26 -0
  683. package/lib/url-alias/group-alias-candidates.d.ts +27 -0
  684. package/lib/url-alias/group-alias-candidates.js +29 -0
  685. package/lib/url-alias/types.d.ts +12 -0
  686. package/lib/url-alias/types.js +1 -0
  687. package/lib/url-pattern/compute-shape-key.d.ts +58 -0
  688. package/lib/url-pattern/compute-shape-key.js +129 -0
  689. package/lib/url-pattern/decompose-url.d.ts +33 -0
  690. package/lib/url-pattern/decompose-url.js +79 -0
  691. package/lib/url-pattern/reconstruct-url.d.ts +15 -0
  692. package/lib/url-pattern/reconstruct-url.js +33 -0
  693. package/lib/url-sort/compute-url-sort-ranks.d.ts +30 -0
  694. package/lib/url-sort/compute-url-sort-ranks.js +33 -0
  695. package/lib/utils/array/each-splitted.d.ts +10 -0
  696. package/lib/utils/array/each-splitted.js +14 -0
  697. package/lib/utils/debug.d.ts +8 -0
  698. package/lib/utils/debug.js +8 -0
  699. package/lib/utils/error/emit-error-with-retry.d.ts +40 -0
  700. package/lib/utils/error/emit-error-with-retry.js +44 -0
  701. package/lib/utils/error/emit-error.d.ts +39 -0
  702. package/lib/utils/error/emit-error.js +41 -0
  703. package/lib/utils/object/parse-response-headers.d.ts +12 -0
  704. package/lib/utils/object/parse-response-headers.js +26 -0
  705. package/lib/utils/object/redact-request-headers.d.ts +15 -0
  706. package/lib/utils/object/redact-request-headers.js +27 -0
  707. package/lib/utils/types/types.d.ts +383 -0
  708. package/lib/utils/types/types.js +1 -0
  709. package/lib/verify-migration/capture-rejection.d.ts +24 -0
  710. package/lib/verify-migration/capture-rejection.js +31 -0
  711. package/lib/verify-migration/check-anchor-edges-count.d.ts +34 -0
  712. package/lib/verify-migration/check-anchor-edges-count.js +72 -0
  713. package/lib/verify-migration/check-anchor-edges-sum.d.ts +13 -0
  714. package/lib/verify-migration/check-anchor-edges-sum.js +27 -0
  715. package/lib/verify-migration/check-content-items-count.d.ts +16 -0
  716. package/lib/verify-migration/check-content-items-count.js +30 -0
  717. package/lib/verify-migration/check-content-type-preservation.d.ts +22 -0
  718. package/lib/verify-migration/check-content-type-preservation.js +40 -0
  719. package/lib/verify-migration/check-foreign-key-integrity.d.ts +31 -0
  720. package/lib/verify-migration/check-foreign-key-integrity.js +47 -0
  721. package/lib/verify-migration/check-image-items-count.d.ts +12 -0
  722. package/lib/verify-migration/check-image-items-count.js +26 -0
  723. package/lib/verify-migration/check-page-meta-count.d.ts +15 -0
  724. package/lib/verify-migration/check-page-meta-count.js +31 -0
  725. package/lib/verify-migration/check-reader-parity.d.ts +23 -0
  726. package/lib/verify-migration/check-reader-parity.js +182 -0
  727. package/lib/verify-migration/check-resource-items-count.d.ts +17 -0
  728. package/lib/verify-migration/check-resource-items-count.js +33 -0
  729. package/lib/verify-migration/check-url-round-trip.d.ts +43 -0
  730. package/lib/verify-migration/check-url-round-trip.js +112 -0
  731. package/lib/verify-migration/types.d.ts +70 -0
  732. package/lib/verify-migration/types.js +63 -0
  733. package/lib/verify-migration/verify-migration.d.ts +41 -0
  734. package/lib/verify-migration/verify-migration.js +120 -0
  735. package/lib/with-bulk-write-pragmas.d.ts +51 -0
  736. package/lib/with-bulk-write-pragmas.js +132 -0
  737. package/lib/write-queue.d.ts +32 -0
  738. package/lib/write-queue.js +55 -0
  739. package/package.json +350 -3
@@ -0,0 +1,326 @@
1
+ import fs from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ import { acquireArchiveLock, ArchiveLockError } from '../archive-lock.js';
4
+ import { Database } from '../database.js';
5
+ import { peekTarTopDir } from '../filesystem/peek-tar-top-dir.js';
6
+ import { rename } from '../filesystem/rename.js';
7
+ import { untar } from '../filesystem/untar.js';
8
+ import { IncompatibleArchiveError } from '../meta/types.js';
9
+ import { computeArchiveCacheKey } from './compute-archive-cache-key.js';
10
+ /**
11
+ * How long {@link extractArchiveToCache} waits in total for a peer
12
+ * extractor to finish (ms). 5 minutes is enough for the largest archives
13
+ * the crawler currently produces (~10 GB untars in ~10 s; 5 min gives
14
+ * 30× headroom) while still bounded so a deadlocked process eventually
15
+ * surfaces as a real error instead of hanging the viewer indefinitely.
16
+ */
17
+ const PEER_WAIT_TIMEOUT_MS = 5 * 60 * 1000;
18
+ /**
19
+ * Polling interval while waiting on a peer extractor (ms). Short enough
20
+ * to keep latency low when the peer is fast, long enough to avoid
21
+ * burning CPU on a busy `fs.stat` loop.
22
+ */
23
+ const PEER_WAIT_POLL_MS = 50;
24
+ /**
25
+ * Marker file dropped into a cache directory only after the extraction
26
+ * has fully completed AND the contents have passed a structural sanity
27
+ * check (db.sqlite present, schema compatible, migrations applied).
28
+ *
29
+ * Internal — the file name is an implementation detail. Tests should
30
+ * assert through externally observable behaviour (cache hit on second
31
+ * open) rather than importing this constant.
32
+ */
33
+ const READY_MARKER = '.nitpicker-cache-ready';
34
+ /**
35
+ * In-process deduplication for concurrent {@link extractArchiveToCache}
36
+ * calls targeting the same `cacheDir`.
37
+ *
38
+ * The file-based {@link acquireArchiveLock} (mkdir-based) cannot defend
39
+ * against same-process race windows because the lock holder writes its
40
+ * `pid.txt` in a separate `await` from the directory creation: a sibling
41
+ * promise that observes the `EEXIST` between those two steps sees an
42
+ * empty `pid.txt`, mistakes the lock for stale, and clobbers it.
43
+ *
44
+ * Single-process dedup eliminates that race entirely for the common
45
+ * "viewer + MCP in the same process" topology while still letting the
46
+ * inner file lock guard cross-process collisions.
47
+ */
48
+ const inFlightByCacheDir = new Map();
49
+ /**
50
+ * Untar a `.nitpicker` archive into the given cache directory.
51
+ *
52
+ * Concurrency / re-entry contract:
53
+ *
54
+ * - Two viewers opening the same archive race for the cache lock; the
55
+ * loser waits and recheck-loops until the winner writes the ready
56
+ * marker, then short-circuits.
57
+ * - Same-process concurrent callers dedupe through
58
+ * `inFlightByCacheDir` so they share one extraction promise.
59
+ *
60
+ * Per-archive staging:
61
+ *
62
+ * - The tar's inner directory is extracted into `${cacheDir}.staging/`
63
+ * first, then atomically renamed into `cacheDir`. Crucially, the
64
+ * staging path is keyed off `cacheDir` (not the tar's inner directory
65
+ * name) so two archives that happen to share an inner-dir name never
66
+ * collide on the same staging slot.
67
+ *
68
+ * Integrity guard:
69
+ *
70
+ * - After rename, we run the writer-side `Database.connect` so all
71
+ * migrations apply (column adds, new tables) and the version check
72
+ * fires before the ready marker is written. A corrupt or incompatible
73
+ * archive therefore poisons no cache entry — the cacheDir is removed
74
+ * and the next caller retries from scratch.
75
+ *
76
+ * TOCTOU guard:
77
+ *
78
+ * - The cache key is recomputed from the file's stat AFTER the untar
79
+ * completes. If the file changed mid-flight (a concurrent
80
+ * `crawl --append` rewrote it), the freshly-landed cache contents do
81
+ * not correspond to the original key, so we abort and remove the
82
+ * cacheDir so the caller upstream sees the inconsistency rather than
83
+ * serving misattributed data.
84
+ * @param archivePath - Absolute path to the source `.nitpicker` file.
85
+ * @param cacheRoot - Absolute path to the cache root directory.
86
+ * @param cacheDir - Absolute path the extracted contents should end up at.
87
+ * @param cacheKey - The cache key used to derive `cacheDir`. Recomputed
88
+ * after extraction to detect concurrent writers; must match.
89
+ * @param onExtractProgress - Called during the untar step with bytes read
90
+ * so far and the archive's total size (issue #294: a cold cache on a
91
+ * large archive can take tens of seconds with no other signal it isn't
92
+ * hung). Never called on a cache hit — {@link isCacheDirReady} short-
93
+ * circuits before `untar` runs — so callers can treat "never invoked" as
94
+ * the definition of a hit and skip printing a phase label until the
95
+ * first call actually arrives. Only the first concurrent caller for a
96
+ * given `cacheDir` sees callbacks; same-`cacheDir` callers deduped
97
+ * through {@link inFlightByCacheDir} just await the shared promise.
98
+ * @param onLog - Forwarded to the cache-miss writer connection
99
+ * (`runMigrationsOnCacheDir`'s `Database.connect`) so a legacy archive's
100
+ * self-healing schema migrations do not fall back to a bare
101
+ * `console.error` mid-redraw of a caller's display (issue #294). Omit for
102
+ * that fallback — see `Archive.openCached`'s own `onLog` doc.
103
+ * @returns Resolves once `cacheDir` is ready to be opened read-only.
104
+ */
105
+ export async function extractArchiveToCache(archivePath, cacheRoot, cacheDir, cacheKey, onExtractProgress, onLog) {
106
+ if (await isCacheDirReady(cacheDir)) {
107
+ return;
108
+ }
109
+ const existing = inFlightByCacheDir.get(cacheDir);
110
+ if (existing) {
111
+ return existing;
112
+ }
113
+ const promise = runExtraction(archivePath, cacheRoot, cacheDir, cacheKey, onExtractProgress, onLog).finally(() => {
114
+ inFlightByCacheDir.delete(cacheDir);
115
+ });
116
+ inFlightByCacheDir.set(cacheDir, promise);
117
+ return promise;
118
+ }
119
+ /**
120
+ * The actual extraction work. Split out from {@link extractArchiveToCache}
121
+ * so the in-process dedup map can wrap it without disturbing the
122
+ * extraction flow itself.
123
+ * @param archivePath - Absolute path to the source `.nitpicker` file.
124
+ * @param cacheRoot - Absolute path to the cache root directory.
125
+ * @param cacheDir - Absolute path the extracted contents should end up at.
126
+ * @param cacheKey - Pre-extraction cache key, re-verified post-extraction.
127
+ * @param onExtractProgress - See {@link extractArchiveToCache}.
128
+ * @param onLog - See {@link extractArchiveToCache}.
129
+ */
130
+ async function runExtraction(archivePath, cacheRoot, cacheDir, cacheKey, onExtractProgress, onLog) {
131
+ await fs.mkdir(cacheRoot, { recursive: true });
132
+ const releaseLock = await acquireLockWithPeerWait(cacheDir);
133
+ try {
134
+ if (await isCacheDirReady(cacheDir)) {
135
+ return;
136
+ }
137
+ // Half-populated cacheDir recovery: another extractor crashed
138
+ // before writing the marker. Rename it aside instead of `rm`-ing
139
+ // in place so any reader that somehow still holds an fd on the
140
+ // old contents (the cache layer doesn't refcount readers) is not
141
+ // pulled out from under. The renamed quarantine dir is left for
142
+ // OS-level temp cleanup.
143
+ await quarantineHalfPopulatedCacheDir(cacheDir);
144
+ const stagingDir = `${cacheDir}.staging`;
145
+ // Stage clean: a previous crashed run may have left
146
+ // `<cacheDir>.staging/` around. `untar` would happily merge into
147
+ // it (with `newer:true` cherrypicking entries) so wipe first.
148
+ await fs.rm(stagingDir, { recursive: true, force: true });
149
+ await fs.mkdir(stagingDir, { recursive: true });
150
+ await untar(archivePath, { cwd: stagingDir, onProgress: onExtractProgress });
151
+ // Concurrent-writer detection: if the source archive changed
152
+ // during our untar, the contents we just landed do NOT match the
153
+ // `cacheKey` that named the directory. Drop the staging dir and
154
+ // surface as an error so the caller (or a retry) can pick up the
155
+ // new key.
156
+ const postKey = await computeArchiveCacheKey(archivePath);
157
+ if (postKey !== cacheKey) {
158
+ await fs.rm(stagingDir, { recursive: true, force: true });
159
+ throw new Error(`Archive changed during cache extraction (key ${cacheKey} -> ${postKey}); ` +
160
+ `refusing to land mismatched contents at ${cacheDir}.`);
161
+ }
162
+ const innerDirName = await peekTarTopDir(archivePath);
163
+ const extractedInner = path.resolve(stagingDir, innerDirName);
164
+ // Move the inner directory into the cache slot. Any old quarantine
165
+ // is already aside, so the target is guaranteed empty.
166
+ await rename(extractedInner, cacheDir, true);
167
+ // The empty `stagingDir` wrapper is no longer useful — best effort
168
+ // remove (ignore errors, OS cleanup catches stragglers).
169
+ await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => { });
170
+ // Validate + migrate: open in WRITER mode so the full migration
171
+ // stack runs (initSchema / migrate*). On failure (db.sqlite
172
+ // missing, incompatible archive, broken schema), tear down the
173
+ // cache entry so the next caller does not get stuck on a
174
+ // "ready"-marked but broken cache.
175
+ try {
176
+ await runMigrationsOnCacheDir(cacheDir, onLog);
177
+ }
178
+ catch (error) {
179
+ await fs.rm(cacheDir, { recursive: true, force: true });
180
+ throw error;
181
+ }
182
+ await fs.writeFile(path.join(cacheDir, READY_MARKER), '', 'utf8');
183
+ }
184
+ finally {
185
+ await releaseLock();
186
+ }
187
+ }
188
+ /**
189
+ * Acquire the cache lock, blocking until any peer extractor finishes
190
+ * (rather than failing fast on `EEXIST` the way the writer-oriented
191
+ * {@link acquireArchiveLock} does). Returns the release function once
192
+ * the lock is held.
193
+ * @param cacheDir - Absolute path the lock guards.
194
+ */
195
+ async function acquireLockWithPeerWait(cacheDir) {
196
+ const startedAt = Date.now();
197
+ while (true) {
198
+ if (await isCacheDirReady(cacheDir)) {
199
+ // Peer finished and the marker is up. Return a no-op release
200
+ // so the caller can `finally`-await it without branching.
201
+ return async () => { };
202
+ }
203
+ try {
204
+ return await acquireArchiveLock(cacheDir);
205
+ }
206
+ catch (error) {
207
+ if (!(error instanceof ArchiveLockError)) {
208
+ throw error;
209
+ }
210
+ if (Date.now() - startedAt > PEER_WAIT_TIMEOUT_MS) {
211
+ throw error;
212
+ }
213
+ await sleep(PEER_WAIT_POLL_MS);
214
+ }
215
+ }
216
+ }
217
+ /**
218
+ * Open the migrated DB in writer mode so all migrations apply, then
219
+ * release the handle. Cache entries land migrated, so the subsequent
220
+ * `Archive.connect` read-only open never needs to mutate the cache dir
221
+ * (and never silently misses a newly-added column).
222
+ * @param cacheDir - Absolute path to the freshly-extracted cache dir.
223
+ * @param onLog - See {@link extractArchiveToCache}.
224
+ */
225
+ async function runMigrationsOnCacheDir(cacheDir, onLog) {
226
+ const dbPath = path.join(cacheDir, 'db.sqlite');
227
+ if (!(await fileExists(dbPath))) {
228
+ throw new Error(`Cache directory does not contain db.sqlite: ${cacheDir}`);
229
+ }
230
+ const db = await Database.connect({ filename: dbPath, readOnly: false, onLog });
231
+ try {
232
+ // `Database.connect` ran migrations during init. Closing here
233
+ // flushes WAL + drops the handle so the read-only re-open in
234
+ // `Archive.connect` does not race the writer connection.
235
+ await db.destroy();
236
+ }
237
+ catch (error) {
238
+ // Propagate after attempting cleanup of the connection (best
239
+ // effort — the cache dir tear-down happens in the caller).
240
+ if (error instanceof IncompatibleArchiveError) {
241
+ throw error;
242
+ }
243
+ throw error;
244
+ }
245
+ }
246
+ /**
247
+ * Move a half-populated cacheDir aside so the next extraction can land
248
+ * cleanly without yanking live readers' files out from under them. The
249
+ * quarantine path gets a unique suffix so two crashes in succession
250
+ * don't collide.
251
+ * @param cacheDir - Absolute path of the cache slot to free.
252
+ */
253
+ async function quarantineHalfPopulatedCacheDir(cacheDir) {
254
+ try {
255
+ await fs.access(cacheDir);
256
+ }
257
+ catch {
258
+ // Nothing to quarantine.
259
+ return;
260
+ }
261
+ if (await isCacheDirReady(cacheDir)) {
262
+ // Already settled — caller will short-circuit, no quarantine needed.
263
+ return;
264
+ }
265
+ const quarantinePath = `${cacheDir}.corrupt.${process.pid}.${nextQuarantineCounter()}`;
266
+ try {
267
+ await fs.rename(cacheDir, quarantinePath);
268
+ }
269
+ catch {
270
+ // If rename fails (e.g. cross-volume), fall back to direct
271
+ // removal — accepting the rm-while-reader risk this single time
272
+ // since we have no other option.
273
+ await fs.rm(cacheDir, { recursive: true, force: true });
274
+ }
275
+ }
276
+ /**
277
+ * Monotonic counter appended to quarantine directory names so multiple
278
+ * recoveries in a single process never collide on the same path. Resets
279
+ * to zero after `2 ** 32` increments; that bound is far larger than any
280
+ * realistic per-process quarantine count.
281
+ */
282
+ let quarantineCounter = 0;
283
+ /**
284
+ * Issue the next quarantine sequence number.
285
+ * @returns A non-negative integer, monotonically increasing per process.
286
+ */
287
+ function nextQuarantineCounter() {
288
+ quarantineCounter = (quarantineCounter + 1) % 0x1_00_00_00_00;
289
+ return quarantineCounter;
290
+ }
291
+ /**
292
+ * Async existence probe — avoids blocking the event loop on the warm
293
+ * cache-hit fast path.
294
+ * @param targetPath - Absolute path to probe.
295
+ * @returns `true` if the path is reachable via `fs.access`.
296
+ */
297
+ async function fileExists(targetPath) {
298
+ try {
299
+ await fs.access(targetPath);
300
+ return true;
301
+ }
302
+ catch {
303
+ return false;
304
+ }
305
+ }
306
+ /**
307
+ * Whether a cache directory contains a fully-validated extraction. The
308
+ * READY marker is the only trustworthy signal — sibling files alone
309
+ * cannot be trusted because a half-extracted dir can also contain a
310
+ * `db.sqlite`.
311
+ * @param cacheDir - Absolute path to a candidate cache directory.
312
+ * @returns `true` if the marker exists.
313
+ */
314
+ async function isCacheDirReady(cacheDir) {
315
+ return await fileExists(path.join(cacheDir, READY_MARKER));
316
+ }
317
+ /**
318
+ * Promise-based sleep used by the peer-wait poll. Kept inline so the
319
+ * file has no external sleep dependency and so the implementation can
320
+ * later swap for an AbortSignal-aware version without touching the
321
+ * call site.
322
+ * @param ms - Milliseconds to wait.
323
+ */
324
+ function sleep(ms) {
325
+ return new Promise((resolve) => setTimeout(resolve, ms));
326
+ }
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Resolve the directory where extracted `.nitpicker` archives are cached.
3
+ *
4
+ * Resolution order:
5
+ *
6
+ * 1. `NITPICKER_TAR_CACHE_DIR` env — explicit override (CI, testing, or
7
+ * operators who want the cache on a specific volume). Must be a
8
+ * path; values that look like boolean / sentinel words (e.g. `0`,
9
+ * `false`) are ignored to keep the cache from landing somewhere
10
+ * surprising when the user mistakes the env contract.
11
+ * 2. `<os.tmpdir()>/nitpicker/cache/` — default. Lives under the OS
12
+ * temp directory so the platform's own cleanup (macOS reboot, Linux
13
+ * `systemd-tmpfiles`, Windows Disk Cleanup) reclaims stale entries
14
+ * without bespoke logic on our side.
15
+ *
16
+ * The returned path is absolute. The caller is responsible for creating
17
+ * it on demand (this function is pure).
18
+ * @returns Absolute path to the cache root directory.
19
+ */
20
+ export declare function getArchiveCacheRoot(): string;
@@ -0,0 +1,53 @@
1
+ import os from 'node:os';
2
+ import path from 'node:path';
3
+ /**
4
+ * Env values that look like a "disable" sentinel rather than a path.
5
+ *
6
+ * A user who copy-pastes the `NITPICKER_DISABLE_TAR_CACHE` convention
7
+ * onto `NITPICKER_TAR_CACHE_DIR` (e.g. `NITPICKER_TAR_CACHE_DIR=0`
8
+ * thinking it means "use default") would otherwise silently land the
9
+ * cache at `$PWD/0/` — multi-gigabyte extracts polluting the project
10
+ * tree, never reclaimed by OS temp cleanup. Reject these explicitly
11
+ * and fall back to the default location instead.
12
+ */
13
+ const SENTINEL_LIKE_OVERRIDES = new Set([
14
+ '0',
15
+ '1',
16
+ 'false',
17
+ 'true',
18
+ 'no',
19
+ 'yes',
20
+ 'off',
21
+ 'on',
22
+ 'null',
23
+ 'undefined',
24
+ ]);
25
+ /**
26
+ * Resolve the directory where extracted `.nitpicker` archives are cached.
27
+ *
28
+ * Resolution order:
29
+ *
30
+ * 1. `NITPICKER_TAR_CACHE_DIR` env — explicit override (CI, testing, or
31
+ * operators who want the cache on a specific volume). Must be a
32
+ * path; values that look like boolean / sentinel words (e.g. `0`,
33
+ * `false`) are ignored to keep the cache from landing somewhere
34
+ * surprising when the user mistakes the env contract.
35
+ * 2. `<os.tmpdir()>/nitpicker/cache/` — default. Lives under the OS
36
+ * temp directory so the platform's own cleanup (macOS reboot, Linux
37
+ * `systemd-tmpfiles`, Windows Disk Cleanup) reclaims stale entries
38
+ * without bespoke logic on our side.
39
+ *
40
+ * The returned path is absolute. The caller is responsible for creating
41
+ * it on demand (this function is pure).
42
+ * @returns Absolute path to the cache root directory.
43
+ */
44
+ export function getArchiveCacheRoot() {
45
+ const envOverride = process.env.NITPICKER_TAR_CACHE_DIR;
46
+ if (envOverride && envOverride.trim().length > 0) {
47
+ const trimmed = envOverride.trim();
48
+ if (!SENTINEL_LIKE_OVERRIDES.has(trimmed.toLowerCase())) {
49
+ return path.resolve(trimmed);
50
+ }
51
+ }
52
+ return path.resolve(os.tmpdir(), 'nitpicker', 'cache');
53
+ }
@@ -0,0 +1,24 @@
1
+ /**
2
+ * Whether the tar cache is disabled by the `NITPICKER_DISABLE_TAR_CACHE`
3
+ * env var.
4
+ *
5
+ * Useful for two scenarios:
6
+ *
7
+ * 1. Debugging — bypass the cache to reproduce the cold-start behaviour
8
+ * against a fresh tmpDir.
9
+ * 2. Sandboxed CI — when the runner's tmpfs would not survive between
10
+ * steps anyway, the cache only adds first-step overhead.
11
+ *
12
+ * Accepted truthy values: `1`, `true`, `yes`, `on` (case-insensitive).
13
+ * Anything else (including unset) keeps the cache enabled.
14
+ * @returns `true` when the cache should be bypassed.
15
+ * @example
16
+ * ```ts
17
+ * if (isArchiveCacheDisabled()) {
18
+ * // Fall back to the writer path; cwd tmpDir + close-time cleanup.
19
+ * const archive = await Archive.open({ filePath });
20
+ * ...
21
+ * }
22
+ * ```
23
+ */
24
+ export declare function isArchiveCacheDisabled(): boolean;
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Whether the tar cache is disabled by the `NITPICKER_DISABLE_TAR_CACHE`
3
+ * env var.
4
+ *
5
+ * Useful for two scenarios:
6
+ *
7
+ * 1. Debugging — bypass the cache to reproduce the cold-start behaviour
8
+ * against a fresh tmpDir.
9
+ * 2. Sandboxed CI — when the runner's tmpfs would not survive between
10
+ * steps anyway, the cache only adds first-step overhead.
11
+ *
12
+ * Accepted truthy values: `1`, `true`, `yes`, `on` (case-insensitive).
13
+ * Anything else (including unset) keeps the cache enabled.
14
+ * @returns `true` when the cache should be bypassed.
15
+ * @example
16
+ * ```ts
17
+ * if (isArchiveCacheDisabled()) {
18
+ * // Fall back to the writer path; cwd tmpDir + close-time cleanup.
19
+ * const archive = await Archive.open({ filePath });
20
+ * ...
21
+ * }
22
+ * ```
23
+ */
24
+ export function isArchiveCacheDisabled() {
25
+ const raw = process.env.NITPICKER_DISABLE_TAR_CACHE;
26
+ if (!raw) {
27
+ return false;
28
+ }
29
+ const normalized = raw.trim().toLowerCase();
30
+ return (normalized === '1' ||
31
+ normalized === 'true' ||
32
+ normalized === 'yes' ||
33
+ normalized === 'on');
34
+ }
@@ -0,0 +1,23 @@
1
+ import type { ArchiveCacheEntry } from './types.js';
2
+ /**
3
+ * List the top-level entries found directly under an archive cache root,
4
+ * with a recursive size total and the most recent mtime found in each.
5
+ *
6
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself (see
7
+ * {@link import('./get-archive-cache-root.js').getArchiveCacheRoot} for the
8
+ * production root), so tests can point it at a throwaway directory instead
9
+ * of the real OS temp cache.
10
+ *
11
+ * Symbolic links are reported but never followed, both at the top level and
12
+ * while walking a directory's contents — this avoids escaping `cacheRoot`
13
+ * and crashing on dangling links.
14
+ * @param cacheRoot - Absolute path to the cache root to inspect.
15
+ * @returns One entry per top-level child of `cacheRoot`, in `fs.readdir`
16
+ * order. Returns `[]` if `cacheRoot` does not exist.
17
+ * @example
18
+ * ```ts
19
+ * const entries = await listArchiveCacheEntries(getArchiveCacheRoot());
20
+ * const totalBytes = entries.reduce((sum, e) => sum + e.sizeBytes, 0);
21
+ * ```
22
+ */
23
+ export declare function listArchiveCacheEntries(cacheRoot: string): Promise<ArchiveCacheEntry[]>;
@@ -0,0 +1,130 @@
1
+ import fs from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ /**
4
+ * List the top-level entries found directly under an archive cache root,
5
+ * with a recursive size total and the most recent mtime found in each.
6
+ *
7
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself (see
8
+ * {@link import('./get-archive-cache-root.js').getArchiveCacheRoot} for the
9
+ * production root), so tests can point it at a throwaway directory instead
10
+ * of the real OS temp cache.
11
+ *
12
+ * Symbolic links are reported but never followed, both at the top level and
13
+ * while walking a directory's contents — this avoids escaping `cacheRoot`
14
+ * and crashing on dangling links.
15
+ * @param cacheRoot - Absolute path to the cache root to inspect.
16
+ * @returns One entry per top-level child of `cacheRoot`, in `fs.readdir`
17
+ * order. Returns `[]` if `cacheRoot` does not exist.
18
+ * @example
19
+ * ```ts
20
+ * const entries = await listArchiveCacheEntries(getArchiveCacheRoot());
21
+ * const totalBytes = entries.reduce((sum, e) => sum + e.sizeBytes, 0);
22
+ * ```
23
+ */
24
+ export async function listArchiveCacheEntries(cacheRoot) {
25
+ let topLevel;
26
+ try {
27
+ topLevel = await fs.readdir(cacheRoot, { withFileTypes: true });
28
+ }
29
+ catch (error) {
30
+ if (error.code === 'ENOENT') {
31
+ return [];
32
+ }
33
+ throw error;
34
+ }
35
+ const entries = [];
36
+ for (const dirent of topLevel) {
37
+ const entryPath = path.join(cacheRoot, dirent.name);
38
+ if (dirent.isSymbolicLink()) {
39
+ entries.push({
40
+ kind: 'unknown',
41
+ name: dirent.name,
42
+ path: entryPath,
43
+ sizeBytes: 0,
44
+ mtimeMs: 0,
45
+ });
46
+ continue;
47
+ }
48
+ if (dirent.isDirectory()) {
49
+ const { sizeBytes, mtimeMs } = await computeDirectoryStats(entryPath);
50
+ entries.push({
51
+ kind: classifyEntryName(dirent.name),
52
+ name: dirent.name,
53
+ path: entryPath,
54
+ sizeBytes,
55
+ mtimeMs,
56
+ });
57
+ continue;
58
+ }
59
+ const stat = await fs.stat(entryPath);
60
+ entries.push({
61
+ kind: 'unknown',
62
+ name: dirent.name,
63
+ path: entryPath,
64
+ sizeBytes: stat.size,
65
+ mtimeMs: stat.mtimeMs,
66
+ });
67
+ }
68
+ return entries;
69
+ }
70
+ /**
71
+ * Matches the corrupt-quarantine suffix `extractArchiveToCache` appends
72
+ * (`${cacheDir}.corrupt.${pid}.${counter}`, both segments always numeric —
73
+ * see `quarantineHalfPopulatedCacheDir` in `extract-archive-to-cache.ts`).
74
+ * Anchored and numeric-specific rather than a loose `includes('.corrupt.')`
75
+ * so an archive whose own (sanitized) basename merely contains the
76
+ * substring `.corrupt.` — e.g. `my.corrupt.report.nitpicker` — is not
77
+ * misclassified as an orphan and offered up for deletion by `cache list`.
78
+ */
79
+ const CORRUPT_QUARANTINE_SUFFIX = /\.corrupt\.\d+\.\d+$/;
80
+ /**
81
+ * Name of the sibling scratch directory earlier versions created for
82
+ * analyze-plugin tables. Nothing writes it any more, so a leftover one is
83
+ * reported as an orphan that `cache clear` can reclaim.
84
+ */
85
+ const LEGACY_TABLE_CACHE_DIR_NAME = 'table';
86
+ /**
87
+ * Classify a cache-root child by name pattern alone (no filesystem access).
88
+ *
89
+ * This is a heuristic: `.staging`/corrupt-quarantine suffixes are appended
90
+ * by `extractArchiveToCache` onto an existing tar-cache dir name, so an
91
+ * archive whose own sanitized basename happens to end in exactly one of
92
+ * these literal suffixes is indistinguishable from a real orphan by name
93
+ * alone. Disambiguating fully would require re-deriving each entry's
94
+ * expected name from its source archive, which `list`/`clear` (by design,
95
+ * see grill-me scope) never has access to. Accepted as a rare, low-cost
96
+ * misclassification.
97
+ * @param name - Base name of the top-level entry.
98
+ */
99
+ function classifyEntryName(name) {
100
+ if (name === LEGACY_TABLE_CACHE_DIR_NAME ||
101
+ name.endsWith('.staging') ||
102
+ CORRUPT_QUARANTINE_SUFFIX.test(name)) {
103
+ return 'orphan';
104
+ }
105
+ return 'tar-cache';
106
+ }
107
+ /**
108
+ * Recursively sum file sizes and find the most recent mtime under a
109
+ * directory. Symbolic links are neither followed nor counted.
110
+ * @param dirPath - Absolute path to the directory to walk.
111
+ */
112
+ async function computeDirectoryStats(dirPath) {
113
+ let sizeBytes = 0;
114
+ let mtimeMs = 0;
115
+ const children = await fs.readdir(dirPath, { recursive: true, withFileTypes: true });
116
+ for (const child of children) {
117
+ if (!child.isFile()) {
118
+ continue;
119
+ }
120
+ const childPath = path.join(child.parentPath, child.name);
121
+ const stat = await fs.stat(childPath);
122
+ sizeBytes += stat.size;
123
+ mtimeMs = Math.max(mtimeMs, stat.mtimeMs);
124
+ }
125
+ if (mtimeMs === 0) {
126
+ const dirStat = await fs.stat(dirPath);
127
+ mtimeMs = dirStat.mtimeMs;
128
+ }
129
+ return { sizeBytes, mtimeMs };
130
+ }
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Async existence probe — avoids blocking the event loop on the common
3
+ * "check before remove" path shared by
4
+ * {@link import('./clear-archive-cache-root.js').clearArchiveCacheRoot} and
5
+ * {@link import('./clear-archive-cache-entry.js').clearArchiveCacheEntry}.
6
+ * @param targetPath - Absolute path to probe.
7
+ * @returns `true` if the path is reachable via `fs.access`.
8
+ */
9
+ export declare function pathExists(targetPath: string): Promise<boolean>;
@@ -0,0 +1,18 @@
1
+ import fs from 'node:fs/promises';
2
+ /**
3
+ * Async existence probe — avoids blocking the event loop on the common
4
+ * "check before remove" path shared by
5
+ * {@link import('./clear-archive-cache-root.js').clearArchiveCacheRoot} and
6
+ * {@link import('./clear-archive-cache-entry.js').clearArchiveCacheEntry}.
7
+ * @param targetPath - Absolute path to probe.
8
+ * @returns `true` if the path is reachable via `fs.access`.
9
+ */
10
+ export async function pathExists(targetPath) {
11
+ try {
12
+ await fs.access(targetPath);
13
+ return true;
14
+ }
15
+ catch {
16
+ return false;
17
+ }
18
+ }
@@ -0,0 +1,26 @@
1
+ /**
2
+ * Build the absolute path to a single archive's cache directory.
3
+ *
4
+ * The directory name is `<cacheKey>-<safeBasename>`:
5
+ *
6
+ * - `cacheKey` carries the freshness signal (size/mtime/ctime) and is
7
+ * what actually drives cache hit vs miss.
8
+ * - `safeBasename` is appended purely so a human running `ls` on the
9
+ * cache root can recognise which archive an entry belongs to. It is
10
+ * NOT used to disambiguate keys — two archives with identical inode
11
+ * metadata (e.g. an identical copy under a different name) intentionally
12
+ * share an entry under the first basename that landed there.
13
+ *
14
+ * The basename is sanitised: anything outside `[A-Za-z0-9._-]` becomes
15
+ * `_`. This keeps the path portable across filesystems (no spaces,
16
+ * unicode normalisation surprises, Windows-reserved chars) and removes
17
+ * any chance that a crafted archive name could escape the cache root
18
+ * (e.g. via `..` or path separators), independent of the upstream
19
+ * `path.basename` call that already drops directory components.
20
+ * @param cacheRoot - Absolute path returned by `getArchiveCacheRoot()`.
21
+ * @param cacheKey - The freshness key from `computeArchiveCacheKey()`.
22
+ * @param archivePath - Absolute path to the source `.nitpicker` file;
23
+ * only its basename contributes to the cache entry name.
24
+ * @returns Absolute path to the per-archive cache directory.
25
+ */
26
+ export declare function resolveArchiveCacheDir(cacheRoot: string, cacheKey: string, archivePath: string): string;