@nitpicker/archive 0.0.0-stage → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +191 -0
- package/README.md +105 -2
- package/lib/apply-connection-pragmas.d.ts +18 -0
- package/lib/apply-connection-pragmas.js +40 -0
- package/lib/archive-accessor.d.ts +265 -0
- package/lib/archive-accessor.js +477 -0
- package/lib/archive-lock.d.ts +45 -0
- package/lib/archive-lock.js +154 -0
- package/lib/archive.d.ts +736 -0
- package/lib/archive.js +1056 -0
- package/lib/body-hash/compute-body-hash.d.ts +32 -0
- package/lib/body-hash/compute-body-hash.js +35 -0
- package/lib/cache/clear-archive-cache-entry.d.ts +18 -0
- package/lib/cache/clear-archive-cache-entry.js +24 -0
- package/lib/cache/clear-archive-cache-root.d.ts +26 -0
- package/lib/cache/clear-archive-cache-root.js +32 -0
- package/lib/cache/compute-archive-cache-key.d.ts +39 -0
- package/lib/cache/compute-archive-cache-key.js +95 -0
- package/lib/cache/extract-archive-to-cache.d.ts +57 -0
- package/lib/cache/extract-archive-to-cache.js +326 -0
- package/lib/cache/get-archive-cache-root.d.ts +20 -0
- package/lib/cache/get-archive-cache-root.js +53 -0
- package/lib/cache/is-archive-cache-disabled.d.ts +24 -0
- package/lib/cache/is-archive-cache-disabled.js +34 -0
- package/lib/cache/list-archive-cache-entries.d.ts +23 -0
- package/lib/cache/list-archive-cache-entries.js +130 -0
- package/lib/cache/path-exists.d.ts +9 -0
- package/lib/cache/path-exists.js +18 -0
- package/lib/cache/resolve-archive-cache-dir.d.ts +26 -0
- package/lib/cache/resolve-archive-cache-dir.js +32 -0
- package/lib/cache/types.d.ts +33 -0
- package/lib/cache/types.js +1 -0
- package/lib/content-type/is-html-content-type.d.ts +17 -0
- package/lib/content-type/is-html-content-type.js +19 -0
- package/lib/content-type/normalize-content-type.d.ts +14 -0
- package/lib/content-type/normalize-content-type.js +20 -0
- package/lib/create-adjunct-tables.d.ts +66 -0
- package/lib/create-adjunct-tables.js +510 -0
- package/lib/create-entity-tables.d.ts +191 -0
- package/lib/create-entity-tables.js +387 -0
- package/lib/create-progress-reporter.d.ts +30 -0
- package/lib/create-progress-reporter.js +38 -0
- package/lib/create-ref-tables.d.ts +35 -0
- package/lib/create-ref-tables.js +185 -0
- package/lib/create-template-tables.d.ts +28 -0
- package/lib/create-template-tables.js +92 -0
- package/lib/database.d.ts +549 -0
- package/lib/database.js +804 -0
- package/lib/db-ops/_shared/clear-write-ref-caches.d.ts +27 -0
- package/lib/db-ops/_shared/clear-write-ref-caches.js +35 -0
- package/lib/db-ops/_shared/compress-payload.d.ts +26 -0
- package/lib/db-ops/_shared/compress-payload.js +30 -0
- package/lib/db-ops/_shared/create-write-ref-caches.d.ts +17 -0
- package/lib/db-ops/_shared/create-write-ref-caches.js +27 -0
- package/lib/db-ops/_shared/decode-json-ref.d.ts +17 -0
- package/lib/db-ops/_shared/decode-json-ref.js +31 -0
- package/lib/db-ops/_shared/keyset-paginate-by-id.d.ts +49 -0
- package/lib/db-ops/_shared/keyset-paginate-by-id.js +69 -0
- package/lib/db-ops/_shared/load-response-headers-by-set-ids.d.ts +20 -0
- package/lib/db-ops/_shared/load-response-headers-by-set-ids.js +53 -0
- package/lib/db-ops/_shared/resolve-content-item-id.d.ts +62 -0
- package/lib/db-ops/_shared/resolve-content-item-id.js +143 -0
- package/lib/db-ops/_shared/resolve-url-or-blob.d.ts +23 -0
- package/lib/db-ops/_shared/resolve-url-or-blob.js +29 -0
- package/lib/db-ops/_shared/retry-setting.d.ts +16 -0
- package/lib/db-ops/_shared/retry-setting.js +18 -0
- package/lib/db-ops/_shared/safe-parse-json.d.ts +11 -0
- package/lib/db-ops/_shared/safe-parse-json.js +18 -0
- package/lib/db-ops/_shared/types.d.ts +94 -0
- package/lib/db-ops/_shared/types.js +1 -0
- package/lib/db-ops/_shared/upsert-blob-ref.d.ts +25 -0
- package/lib/db-ops/_shared/upsert-blob-ref.js +48 -0
- package/lib/db-ops/_shared/upsert-content-type-ref.d.ts +30 -0
- package/lib/db-ops/_shared/upsert-content-type-ref.js +45 -0
- package/lib/db-ops/_shared/upsert-json-ref.d.ts +22 -0
- package/lib/db-ops/_shared/upsert-json-ref.js +41 -0
- package/lib/db-ops/_shared/upsert-response-headers.d.ts +35 -0
- package/lib/db-ops/_shared/upsert-response-headers.js +49 -0
- package/lib/db-ops/_shared/upsert-url-ref.d.ts +39 -0
- package/lib/db-ops/_shared/upsert-url-ref.js +62 -0
- package/lib/db-ops/anchors/get-anchors-on-page.d.ts +10 -0
- package/lib/db-ops/anchors/get-anchors-on-page.js +21 -0
- package/lib/db-ops/config/get-base-url.d.ts +8 -0
- package/lib/db-ops/config/get-base-url.js +14 -0
- package/lib/db-ops/config/get-config.d.ts +10 -0
- package/lib/db-ops/config/get-config.js +29 -0
- package/lib/db-ops/config/get-created-cwd.d.ts +11 -0
- package/lib/db-ops/config/get-created-cwd.js +13 -0
- package/lib/db-ops/config/get-name.d.ts +8 -0
- package/lib/db-ops/config/get-name.js +14 -0
- package/lib/db-ops/config/info-column-allowlist.d.ts +7 -0
- package/lib/db-ops/config/info-column-allowlist.js +29 -0
- package/lib/db-ops/config/info-json-columns.d.ts +5 -0
- package/lib/db-ops/config/info-json-columns.js +11 -0
- package/lib/db-ops/config/set-config.d.ts +12 -0
- package/lib/db-ops/config/set-config.js +21 -0
- package/lib/db-ops/config/update-config.d.ts +17 -0
- package/lib/db-ops/config/update-config.js +36 -0
- package/lib/db-ops/console-logs/compute-console-log-hash.d.ts +19 -0
- package/lib/db-ops/console-logs/compute-console-log-hash.js +30 -0
- package/lib/db-ops/console-logs/replace-console-logs.d.ts +34 -0
- package/lib/db-ops/console-logs/replace-console-logs.js +123 -0
- package/lib/db-ops/console-logs/stringify-console-log-args.d.ts +16 -0
- package/lib/db-ops/console-logs/stringify-console-log-args.js +26 -0
- package/lib/db-ops/console-logs/types.d.ts +38 -0
- package/lib/db-ops/console-logs/types.js +1 -0
- package/lib/db-ops/console-logs/upsert-console-log-item.d.ts +29 -0
- package/lib/db-ops/console-logs/upsert-console-log-item.js +52 -0
- package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts +18 -0
- package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js +23 -0
- package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts +12 -0
- package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.js +15 -0
- package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts +14 -0
- package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.js +30 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.d.ts +57 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.js +87 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts +21 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js +27 -0
- package/lib/db-ops/errors/insert-crawl-error.d.ts +15 -0
- package/lib/db-ops/errors/insert-crawl-error.js +21 -0
- package/lib/db-ops/errors/insert-page-error.d.ts +21 -0
- package/lib/db-ops/errors/insert-page-error.js +30 -0
- package/lib/db-ops/errors/list-dns-burned-host-candidates.d.ts +33 -0
- package/lib/db-ops/errors/list-dns-burned-host-candidates.js +158 -0
- package/lib/db-ops/html/read-page-html-blob.d.ts +17 -0
- package/lib/db-ops/html/read-page-html-blob.js +20 -0
- package/lib/db-ops/lifecycle/checkpoint.d.ts +8 -0
- package/lib/db-ops/lifecycle/checkpoint.js +9 -0
- package/lib/db-ops/lifecycle/destroy.d.ts +6 -0
- package/lib/db-ops/lifecycle/destroy.js +7 -0
- package/lib/db-ops/lifecycle/init.d.ts +58 -0
- package/lib/db-ops/lifecycle/init.js +122 -0
- package/lib/db-ops/list-reconcile/record-list-reconcile-run.d.ts +22 -0
- package/lib/db-ops/list-reconcile/record-list-reconcile-run.js +41 -0
- package/lib/db-ops/meta/get-audios-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-audios-of-page.js +15 -0
- package/lib/db-ops/meta/get-buttons-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-buttons-of-page.js +15 -0
- package/lib/db-ops/meta/get-canvases-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-canvases-of-page.js +15 -0
- package/lib/db-ops/meta/get-custom-elements-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-custom-elements-of-page.js +15 -0
- package/lib/db-ops/meta/get-headings-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-headings-of-page.js +15 -0
- package/lib/db-ops/meta/get-iframes-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-iframes-of-page.js +15 -0
- package/lib/db-ops/meta/get-jsonld-of-page.d.ts +13 -0
- package/lib/db-ops/meta/get-jsonld-of-page.js +27 -0
- package/lib/db-ops/meta/get-main-content-images-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-main-content-images-of-page.js +15 -0
- package/lib/db-ops/meta/get-main-content-tables-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-main-content-tables-of-page.js +15 -0
- package/lib/db-ops/meta/get-page-technologies-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-page-technologies-of-page.js +15 -0
- package/lib/db-ops/meta/get-technology-signals-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-technology-signals-of-page.js +15 -0
- package/lib/db-ops/meta/get-videos-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-videos-of-page.js +15 -0
- package/lib/db-ops/outages/close-network-outage.d.ts +12 -0
- package/lib/db-ops/outages/close-network-outage.js +15 -0
- package/lib/db-ops/outages/close-stale-open-network-outages.d.ts +26 -0
- package/lib/db-ops/outages/close-stale-open-network-outages.js +42 -0
- package/lib/db-ops/outages/compute-outage-clamp-timestamp.d.ts +26 -0
- package/lib/db-ops/outages/compute-outage-clamp-timestamp.js +31 -0
- package/lib/db-ops/outages/insert-network-outage.d.ts +14 -0
- package/lib/db-ops/outages/insert-network-outage.js +29 -0
- package/lib/db-ops/outages/list-network-outages.d.ts +28 -0
- package/lib/db-ops/outages/list-network-outages.js +42 -0
- package/lib/db-ops/pages/order/set-url-order.d.ts +13 -0
- package/lib/db-ops/pages/order/set-url-order.js +38 -0
- package/lib/db-ops/pages/read/build-page-query.d.ts +18 -0
- package/lib/db-ops/pages/read/build-page-query.js +40 -0
- package/lib/db-ops/pages/read/get-crawling-state.d.ts +117 -0
- package/lib/db-ops/pages/read/get-crawling-state.js +161 -0
- package/lib/db-ops/pages/read/get-existing-page-urls.d.ts +15 -0
- package/lib/db-ops/pages/read/get-existing-page-urls.js +30 -0
- package/lib/db-ops/pages/read/get-page-count.d.ts +12 -0
- package/lib/db-ops/pages/read/get-page-count.js +21 -0
- package/lib/db-ops/pages/read/get-page-source-by-url.d.ts +24 -0
- package/lib/db-ops/pages/read/get-page-source-by-url.js +28 -0
- package/lib/db-ops/pages/read/get-pages-with-rels.d.ts +38 -0
- package/lib/db-ops/pages/read/get-pages-with-rels.js +107 -0
- package/lib/db-ops/pages/read/get-pages.d.ts +11 -0
- package/lib/db-ops/pages/read/get-pages.js +51 -0
- package/lib/db-ops/pages/read/get-scraped-html-page-count.d.ts +18 -0
- package/lib/db-ops/pages/read/get-scraped-html-page-count.js +25 -0
- package/lib/db-ops/pages/read/reconstruct-page-rows.d.ts +31 -0
- package/lib/db-ops/pages/read/reconstruct-page-rows.js +32 -0
- package/lib/db-ops/pages/reset/clear-page-derived-rows.d.ts +20 -0
- package/lib/db-ops/pages/reset/clear-page-derived-rows.js +37 -0
- package/lib/db-ops/pages/reset/repromote-external-pages.d.ts +30 -0
- package/lib/db-ops/pages/reset/repromote-external-pages.js +91 -0
- package/lib/db-ops/pages/reset/reset-failed-pages.d.ts +88 -0
- package/lib/db-ops/pages/reset/reset-failed-pages.js +216 -0
- package/lib/db-ops/pages/reset/reset-pages-by-urls.d.ts +80 -0
- package/lib/db-ops/pages/reset/reset-pages-by-urls.js +151 -0
- package/lib/db-ops/pages/reset/retryable-image-scan-codes.d.ts +24 -0
- package/lib/db-ops/pages/reset/retryable-image-scan-codes.js +24 -0
- package/lib/db-ops/pages/write/insert-audios.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-audios.js +25 -0
- package/lib/db-ops/pages/write/insert-buttons.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-buttons.js +29 -0
- package/lib/db-ops/pages/write/insert-canvases.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-canvases.js +26 -0
- package/lib/db-ops/pages/write/insert-custom-elements.d.ts +27 -0
- package/lib/db-ops/pages/write/insert-custom-elements.js +40 -0
- package/lib/db-ops/pages/write/insert-headings.d.ts +15 -0
- package/lib/db-ops/pages/write/insert-headings.js +27 -0
- package/lib/db-ops/pages/write/insert-iframes.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-iframes.js +28 -0
- package/lib/db-ops/pages/write/insert-inventory-content-items.d.ts +38 -0
- package/lib/db-ops/pages/write/insert-inventory-content-items.js +60 -0
- package/lib/db-ops/pages/write/insert-inventory-seeds.d.ts +36 -0
- package/lib/db-ops/pages/write/insert-inventory-seeds.js +48 -0
- package/lib/db-ops/pages/write/insert-inventory-skipped-pages.d.ts +42 -0
- package/lib/db-ops/pages/write/insert-inventory-skipped-pages.js +56 -0
- package/lib/db-ops/pages/write/insert-jsonld.d.ts +17 -0
- package/lib/db-ops/pages/write/insert-jsonld.js +49 -0
- package/lib/db-ops/pages/write/insert-main-content-images.d.ts +20 -0
- package/lib/db-ops/pages/write/insert-main-content-images.js +32 -0
- package/lib/db-ops/pages/write/insert-main-content-tables.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-main-content-tables.js +29 -0
- package/lib/db-ops/pages/write/insert-page.d.ts +56 -0
- package/lib/db-ops/pages/write/insert-page.js +259 -0
- package/lib/db-ops/pages/write/insert-technologies.d.ts +25 -0
- package/lib/db-ops/pages/write/insert-technologies.js +45 -0
- package/lib/db-ops/pages/write/insert-videos.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-videos.js +28 -0
- package/lib/db-ops/pages/write/link-redirect-sources.d.ts +42 -0
- package/lib/db-ops/pages/write/link-redirect-sources.js +101 -0
- package/lib/db-ops/pages/write/record-redirect.d.ts +35 -0
- package/lib/db-ops/pages/write/record-redirect.js +100 -0
- package/lib/db-ops/pages/write/set-skipped-page.d.ts +13 -0
- package/lib/db-ops/pages/write/set-skipped-page.js +24 -0
- package/lib/db-ops/pages/write/update-page.d.ts +40 -0
- package/lib/db-ops/pages/write/update-page.js +419 -0
- package/lib/db-ops/pages/write/write-page-html-blob.d.ts +19 -0
- package/lib/db-ops/pages/write/write-page-html-blob.js +41 -0
- package/lib/db-ops/referrers/get-redirects-for-pages.d.ts +9 -0
- package/lib/db-ops/referrers/get-redirects-for-pages.js +15 -0
- package/lib/db-ops/referrers/get-referrers-of-page.d.ts +17 -0
- package/lib/db-ops/referrers/get-referrers-of-page.js +32 -0
- package/lib/db-ops/referrers/get-referrers-of-resource.d.ts +8 -0
- package/lib/db-ops/referrers/get-referrers-of-resource.js +15 -0
- package/lib/db-ops/resources/build-resource-query.d.ts +25 -0
- package/lib/db-ops/resources/build-resource-query.js +29 -0
- package/lib/db-ops/resources/get-existing-resource-urls.d.ts +9 -0
- package/lib/db-ops/resources/get-existing-resource-urls.js +24 -0
- package/lib/db-ops/resources/get-resource-by-url.d.ts +13 -0
- package/lib/db-ops/resources/get-resource-by-url.js +22 -0
- package/lib/db-ops/resources/get-resource-url-list.d.ts +17 -0
- package/lib/db-ops/resources/get-resource-url-list.js +26 -0
- package/lib/db-ops/resources/get-resources.d.ts +8 -0
- package/lib/db-ops/resources/get-resources.js +11 -0
- package/lib/db-ops/resources/insert-inventory-resources.d.ts +24 -0
- package/lib/db-ops/resources/insert-inventory-resources.js +64 -0
- package/lib/db-ops/resources/insert-resource-referrers.d.ts +15 -0
- package/lib/db-ops/resources/insert-resource-referrers.js +54 -0
- package/lib/db-ops/resources/insert-resource.d.ts +34 -0
- package/lib/db-ops/resources/insert-resource.js +73 -0
- package/lib/db-ops/resources/reconstruct-resource-rows.d.ts +26 -0
- package/lib/db-ops/resources/reconstruct-resource-rows.js +30 -0
- package/lib/db-ops/templates/assign-template-labels.d.ts +60 -0
- package/lib/db-ops/templates/assign-template-labels.js +187 -0
- package/lib/db-ops/templates/replace-page-templates.d.ts +44 -0
- package/lib/db-ops/templates/replace-page-templates.js +174 -0
- package/lib/db-ops/templates/serialize-template-cluster-reason.d.ts +16 -0
- package/lib/db-ops/templates/serialize-template-cluster-reason.js +17 -0
- package/lib/db-ops/templates/types.d.ts +103 -0
- package/lib/db-ops/templates/types.js +1 -0
- package/lib/debug.d.ts +8 -0
- package/lib/debug.js +9 -0
- package/lib/decode-html-blob.d.ts +20 -0
- package/lib/decode-html-blob.js +23 -0
- package/lib/decode-page-html-blob-bytes.d.ts +25 -0
- package/lib/decode-page-html-blob-bytes.js +29 -0
- package/lib/decode-stored-blob-bytes.d.ts +25 -0
- package/lib/decode-stored-blob-bytes.js +47 -0
- package/lib/derive-lineage-from-parent.d.ts +37 -0
- package/lib/derive-lineage-from-parent.js +42 -0
- package/lib/drop-legacy-tables.d.ts +45 -0
- package/lib/drop-legacy-tables.js +56 -0
- package/lib/error-kind/classify-error-kind.d.ts +20 -0
- package/lib/error-kind/classify-error-kind.js +136 -0
- package/lib/error-kind/network-related-error-kinds.d.ts +23 -0
- package/lib/error-kind/network-related-error-kinds.js +28 -0
- package/lib/error-kind/permanent-error-kinds.d.ts +48 -0
- package/lib/error-kind/permanent-error-kinds.js +54 -0
- package/lib/error-kind/types.d.ts +54 -0
- package/lib/error-kind/types.js +1 -0
- package/lib/filesystem/append-text.d.ts +9 -0
- package/lib/filesystem/append-text.js +14 -0
- package/lib/filesystem/copy-dir-sync.d.ts +6 -0
- package/lib/filesystem/copy-dir-sync.js +9 -0
- package/lib/filesystem/copy-dir.d.ts +7 -0
- package/lib/filesystem/copy-dir.js +13 -0
- package/lib/filesystem/copy-file-with-progress.d.ts +20 -0
- package/lib/filesystem/copy-file-with-progress.js +34 -0
- package/lib/filesystem/exists.d.ts +6 -0
- package/lib/filesystem/exists.js +9 -0
- package/lib/filesystem/get-file-list.d.ts +8 -0
- package/lib/filesystem/get-file-list.js +12 -0
- package/lib/filesystem/is-dir.d.ts +6 -0
- package/lib/filesystem/is-dir.js +10 -0
- package/lib/filesystem/mkdir.d.ts +8 -0
- package/lib/filesystem/mkdir.js +15 -0
- package/lib/filesystem/output-binary.d.ts +12 -0
- package/lib/filesystem/output-binary.js +17 -0
- package/lib/filesystem/output-json.d.ts +9 -0
- package/lib/filesystem/output-json.js +14 -0
- package/lib/filesystem/output-text.d.ts +11 -0
- package/lib/filesystem/output-text.js +32 -0
- package/lib/filesystem/parse-pax-path.d.ts +18 -0
- package/lib/filesystem/parse-pax-path.js +47 -0
- package/lib/filesystem/parse-tar-size-field.d.ts +13 -0
- package/lib/filesystem/parse-tar-size-field.js +40 -0
- package/lib/filesystem/peek-tar-top-dir.d.ts +35 -0
- package/lib/filesystem/peek-tar-top-dir.js +215 -0
- package/lib/filesystem/read-json.d.ts +7 -0
- package/lib/filesystem/read-json.js +11 -0
- package/lib/filesystem/read-text.d.ts +6 -0
- package/lib/filesystem/read-text.js +10 -0
- package/lib/filesystem/readline.d.ts +11 -0
- package/lib/filesystem/readline.js +26 -0
- package/lib/filesystem/remove.d.ts +5 -0
- package/lib/filesystem/remove.js +10 -0
- package/lib/filesystem/rename.d.ts +16 -0
- package/lib/filesystem/rename.js +48 -0
- package/lib/filesystem/tar.d.ts +23 -0
- package/lib/filesystem/tar.js +78 -0
- package/lib/filesystem/untar.d.ts +33 -0
- package/lib/filesystem/untar.js +84 -0
- package/lib/get-failed-page-messages.d.ts +56 -0
- package/lib/get-failed-page-messages.js +146 -0
- package/lib/get-json.d.ts +10 -0
- package/lib/get-json.js +25 -0
- package/lib/html-snapshot/match-selector-html.d.ts +25 -0
- package/lib/html-snapshot/match-selector-html.js +28 -0
- package/lib/html-snapshot/match-selector-snapshots.d.ts +33 -0
- package/lib/html-snapshot/match-selector-snapshots.js +33 -0
- package/lib/html-snapshot/search-html-document.d.ts +29 -0
- package/lib/html-snapshot/search-html-document.js +27 -0
- package/lib/html-snapshot/search-html-snapshots.d.ts +36 -0
- package/lib/html-snapshot/search-html-snapshots.js +35 -0
- package/lib/html-snapshot/types.d.ts +111 -0
- package/lib/html-snapshot/types.js +7 -0
- package/lib/html-snapshot/validate-search-pattern.d.ts +15 -0
- package/lib/html-snapshot/validate-search-pattern.js +17 -0
- package/lib/init-schema.d.ts +48 -0
- package/lib/init-schema.js +124 -0
- package/lib/is-inventory-source.d.ts +21 -0
- package/lib/is-inventory-source.js +22 -0
- package/lib/is-within-outage-window.d.ts +49 -0
- package/lib/is-within-outage-window.js +33 -0
- package/lib/isolated-cluster/create-isolated-cluster-run.d.ts +24 -0
- package/lib/isolated-cluster/create-isolated-cluster-run.js +26 -0
- package/lib/libsql-dialect.d.ts +115 -0
- package/lib/libsql-dialect.js +116 -0
- package/lib/limited-page-ids.d.ts +9 -0
- package/lib/limited-page-ids.js +18 -0
- package/lib/meta/assert-compatible-version.d.ts +60 -0
- package/lib/meta/assert-compatible-version.js +93 -0
- package/lib/meta/classify-jsonld-type.d.ts +23 -0
- package/lib/meta/classify-jsonld-type.js +43 -0
- package/lib/meta/compare-semver.d.ts +23 -0
- package/lib/meta/compare-semver.js +51 -0
- package/lib/meta/compute-main-contents-denormalized.d.ts +41 -0
- package/lib/meta/compute-main-contents-denormalized.js +88 -0
- package/lib/meta/compute-page-denormalized.d.ts +20 -0
- package/lib/meta/compute-page-denormalized.js +35 -0
- package/lib/meta/derive-flat-from-meta.d.ts +34 -0
- package/lib/meta/derive-flat-from-meta.js +158 -0
- package/lib/meta/derive-meta-extras.d.ts +20 -0
- package/lib/meta/derive-meta-extras.js +23 -0
- package/lib/meta/extract-tags-for-archive.d.ts +17 -0
- package/lib/meta/extract-tags-for-archive.js +36 -0
- package/lib/meta/parse-main-content-class-list.d.ts +19 -0
- package/lib/meta/parse-main-content-class-list.js +34 -0
- package/lib/meta/summarize-jsonld.d.ts +17 -0
- package/lib/meta/summarize-jsonld.js +29 -0
- package/lib/meta/technologies/combine-technology-confidence.d.ts +46 -0
- package/lib/meta/technologies/combine-technology-confidence.js +93 -0
- package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.d.ts +45 -0
- package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.js +53 -0
- package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.d.ts +33 -0
- package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.js +46 -0
- package/lib/meta/technologies/extract-technologies-for-archive.d.ts +33 -0
- package/lib/meta/technologies/extract-technologies-for-archive.js +30 -0
- package/lib/meta/technologies/match-js-license-comment-signal.d.ts +21 -0
- package/lib/meta/technologies/match-js-license-comment-signal.js +23 -0
- package/lib/meta/technologies/match-structural-technology-signals.d.ts +26 -0
- package/lib/meta/technologies/match-structural-technology-signals.js +48 -0
- package/lib/meta/technologies/normalize-wappalyzer-entries.d.ts +25 -0
- package/lib/meta/technologies/normalize-wappalyzer-entries.js +35 -0
- package/lib/meta/technologies/technology-signal-definitions.d.ts +44 -0
- package/lib/meta/technologies/technology-signal-definitions.js +60 -0
- package/lib/meta/technologies/types.d.ts +70 -0
- package/lib/meta/technologies/types.js +13 -0
- package/lib/meta/types.d.ts +1490 -0
- package/lib/meta/types.js +65 -0
- package/lib/migrate-content-items-alias-of-id.d.ts +52 -0
- package/lib/migrate-content-items-alias-of-id.js +69 -0
- package/lib/migrate-content-items-dedupe-cap-event-id.d.ts +46 -0
- package/lib/migrate-content-items-dedupe-cap-event-id.js +62 -0
- package/lib/migrate-content-items-is-metadata-only.d.ts +44 -0
- package/lib/migrate-content-items-is-metadata-only.js +60 -0
- package/lib/migrate-entity-tables.d.ts +55 -0
- package/lib/migrate-entity-tables.js +66 -0
- package/lib/migrate-info-created-cwd.d.ts +17 -0
- package/lib/migrate-info-created-cwd.js +36 -0
- package/lib/migrate-info-main-content-selector.d.ts +16 -0
- package/lib/migrate-info-main-content-selector.js +35 -0
- package/lib/migrate-info-request-header-names.d.ts +16 -0
- package/lib/migrate-info-request-header-names.js +35 -0
- package/lib/migrate-info-roots.d.ts +20 -0
- package/lib/migrate-info-roots.js +52 -0
- package/lib/migrate-inventory-runs-to-list-reconcile-runs.d.ts +34 -0
- package/lib/migrate-inventory-runs-to-list-reconcile-runs.js +51 -0
- package/lib/migrate-list-reconcile-runs-exclude-skipped.d.ts +25 -0
- package/lib/migrate-list-reconcile-runs-exclude-skipped.js +44 -0
- package/lib/migrate-list-reconcile-runs-invalid-skipped.d.ts +16 -0
- package/lib/migrate-list-reconcile-runs-invalid-skipped.js +35 -0
- package/lib/migrate-main-contents-columns.d.ts +26 -0
- package/lib/migrate-main-contents-columns.js +61 -0
- package/lib/migrate-page-meta-body-hash.d.ts +44 -0
- package/lib/migrate-page-meta-body-hash.js +63 -0
- package/lib/migrate-page-meta-console-error-count.d.ts +27 -0
- package/lib/migrate-page-meta-console-error-count.js +45 -0
- package/lib/migrate-page-meta-custom-element-count.d.ts +33 -0
- package/lib/migrate-page-meta-custom-element-count.js +51 -0
- package/lib/migrate-page-meta-image-scan.d.ts +27 -0
- package/lib/migrate-page-meta-image-scan.js +52 -0
- package/lib/migrate-page-tags-to-page-technologies.d.ts +39 -0
- package/lib/migrate-page-tags-to-page-technologies.js +79 -0
- package/lib/migrate-ref-tables.d.ts +25 -0
- package/lib/migrate-ref-tables.js +38 -0
- package/lib/native-executor/check-native-executor-config.d.ts +17 -0
- package/lib/native-executor/check-native-executor-config.js +20 -0
- package/lib/page-meta-column-maps.d.ts +32 -0
- package/lib/page-meta-column-maps.js +43 -0
- package/lib/page.d.ts +546 -0
- package/lib/page.js +764 -0
- package/lib/peek-archive-lock.d.ts +40 -0
- package/lib/peek-archive-lock.js +62 -0
- package/lib/populate-entity-tables/collapse-anchor-rows.d.ts +41 -0
- package/lib/populate-entity-tables/collapse-anchor-rows.js +87 -0
- package/lib/populate-entity-tables/derive-dom-path.d.ts +35 -0
- package/lib/populate-entity-tables/derive-dom-path.js +72 -0
- package/lib/populate-entity-tables/is-blob-ref-value.d.ts +16 -0
- package/lib/populate-entity-tables/is-blob-ref-value.js +19 -0
- package/lib/populate-entity-tables/match-images-to-dom-paths.d.ts +66 -0
- package/lib/populate-entity-tables/match-images-to-dom-paths.js +96 -0
- package/lib/populate-entity-tables/populate-anchor-edges.d.ts +33 -0
- package/lib/populate-entity-tables/populate-anchor-edges.js +153 -0
- package/lib/populate-entity-tables/populate-content-items.d.ts +40 -0
- package/lib/populate-entity-tables/populate-content-items.js +141 -0
- package/lib/populate-entity-tables/populate-entities.d.ts +81 -0
- package/lib/populate-entity-tables/populate-entities.js +110 -0
- package/lib/populate-entity-tables/populate-image-items.d.ts +91 -0
- package/lib/populate-entity-tables/populate-image-items.js +226 -0
- package/lib/populate-entity-tables/populate-page-meta.d.ts +33 -0
- package/lib/populate-entity-tables/populate-page-meta.js +267 -0
- package/lib/populate-entity-tables/populate-resource-items.d.ts +22 -0
- package/lib/populate-entity-tables/populate-resource-items.js +114 -0
- package/lib/populate-entity-tables/populate-resource-ref-edges.d.ts +31 -0
- package/lib/populate-entity-tables/populate-resource-ref-edges.js +33 -0
- package/lib/populate-entity-tables/resolve-blob-refs.d.ts +31 -0
- package/lib/populate-entity-tables/resolve-blob-refs.js +100 -0
- package/lib/populate-entity-tables/resolve-content-type-refs.d.ts +22 -0
- package/lib/populate-entity-tables/resolve-content-type-refs.js +27 -0
- package/lib/populate-entity-tables/resolve-header-sets.d.ts +49 -0
- package/lib/populate-entity-tables/resolve-header-sets.js +122 -0
- package/lib/populate-entity-tables/resolve-json-refs.d.ts +25 -0
- package/lib/populate-entity-tables/resolve-json-refs.js +67 -0
- package/lib/populate-entity-tables/resolve-text-refs.d.ts +30 -0
- package/lib/populate-entity-tables/resolve-text-refs.js +61 -0
- package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.d.ts +21 -0
- package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.js +27 -0
- package/lib/populate-entity-tables/resolve-url-refs.d.ts +33 -0
- package/lib/populate-entity-tables/resolve-url-refs.js +60 -0
- package/lib/populate-entity-tables/test-utils/count-rows.d.ts +17 -0
- package/lib/populate-entity-tables/test-utils/count-rows.js +20 -0
- package/lib/populate-entity-tables/test-utils/seed-content-items.d.ts +25 -0
- package/lib/populate-entity-tables/test-utils/seed-content-items.js +42 -0
- package/lib/populate-entity-tables/test-utils/setup-entities-db.d.ts +30 -0
- package/lib/populate-entity-tables/test-utils/setup-entities-db.js +175 -0
- package/lib/populate-entity-tables/types.d.ts +157 -0
- package/lib/populate-entity-tables/types.js +12 -0
- package/lib/populate-entity-tables/upsert-text-refs.d.ts +38 -0
- package/lib/populate-entity-tables/upsert-text-refs.js +78 -0
- package/lib/populate-ref-tables/classify-content-type.d.ts +16 -0
- package/lib/populate-ref-tables/classify-content-type.js +52 -0
- package/lib/populate-ref-tables/compute-content-hash.d.ts +22 -0
- package/lib/populate-ref-tables/compute-content-hash.js +26 -0
- package/lib/populate-ref-tables/compute-header-flags.d.ts +16 -0
- package/lib/populate-ref-tables/compute-header-flags.js +70 -0
- package/lib/populate-ref-tables/content-type-rules.d.ts +38 -0
- package/lib/populate-ref-tables/content-type-rules.js +133 -0
- package/lib/populate-ref-tables/create-header-table-caches.d.ts +25 -0
- package/lib/populate-ref-tables/create-header-table-caches.js +49 -0
- package/lib/populate-ref-tables/data-uri-url-refs-limit.d.ts +15 -0
- package/lib/populate-ref-tables/data-uri-url-refs-limit.js +15 -0
- package/lib/populate-ref-tables/decode-data-uri.d.ts +21 -0
- package/lib/populate-ref-tables/decode-data-uri.js +126 -0
- package/lib/populate-ref-tables/decompose-header-set.d.ts +29 -0
- package/lib/populate-ref-tables/decompose-header-set.js +157 -0
- package/lib/populate-ref-tables/decompose-url.d.ts +25 -0
- package/lib/populate-ref-tables/decompose-url.js +70 -0
- package/lib/populate-ref-tables/header-stability.d.ts +19 -0
- package/lib/populate-ref-tables/header-stability.js +22 -0
- package/lib/populate-ref-tables/header-value-cache-key.d.ts +17 -0
- package/lib/populate-ref-tables/header-value-cache-key.js +19 -0
- package/lib/populate-ref-tables/normalize-mime.d.ts +24 -0
- package/lib/populate-ref-tables/normalize-mime.js +36 -0
- package/lib/populate-ref-tables/populate-blob-refs.d.ts +38 -0
- package/lib/populate-ref-tables/populate-blob-refs.js +134 -0
- package/lib/populate-ref-tables/populate-content-type-refs.d.ts +27 -0
- package/lib/populate-ref-tables/populate-content-type-refs.js +70 -0
- package/lib/populate-ref-tables/populate-header-tables.d.ts +35 -0
- package/lib/populate-ref-tables/populate-header-tables.js +80 -0
- package/lib/populate-ref-tables/populate-json-refs.d.ts +29 -0
- package/lib/populate-ref-tables/populate-json-refs.js +101 -0
- package/lib/populate-ref-tables/populate-refs.d.ts +51 -0
- package/lib/populate-ref-tables/populate-refs.js +62 -0
- package/lib/populate-ref-tables/populate-text-refs.d.ts +32 -0
- package/lib/populate-ref-tables/populate-text-refs.js +133 -0
- package/lib/populate-ref-tables/populate-url-refs.d.ts +28 -0
- package/lib/populate-ref-tables/populate-url-refs.js +148 -0
- package/lib/populate-ref-tables/test-utils/count-rows.d.ts +15 -0
- package/lib/populate-ref-tables/test-utils/count-rows.js +17 -0
- package/lib/populate-ref-tables/types.d.ts +197 -0
- package/lib/populate-ref-tables/types.js +7 -0
- package/lib/populate-ref-tables/upsert-one-header-set.d.ts +34 -0
- package/lib/populate-ref-tables/upsert-one-header-set.js +208 -0
- package/lib/populate-ref-tables/volatile-header-names.d.ts +20 -0
- package/lib/populate-ref-tables/volatile-header-names.js +33 -0
- package/lib/redirect-table.d.ts +9 -0
- package/lib/redirect-table.js +28 -0
- package/lib/release-finished-statements.d.ts +42 -0
- package/lib/release-finished-statements.js +45 -0
- package/lib/resolve-redirect-chain.d.ts +33 -0
- package/lib/resolve-redirect-chain.js +27 -0
- package/lib/resource.d.ts +46 -0
- package/lib/resource.js +62 -0
- package/lib/retarget-legacy-fk-tables.d.ts +61 -0
- package/lib/retarget-legacy-fk-tables.js +137 -0
- package/lib/safe-path.d.ts +9 -0
- package/lib/safe-path.js +17 -0
- package/lib/scope/build-scope-map.d.ts +21 -0
- package/lib/scope/build-scope-map.js +32 -0
- package/lib/scope/find-scope-entry.d.ts +25 -0
- package/lib/scope/find-scope-entry.js +45 -0
- package/lib/template-classification/classify-archive-page-templates.d.ts +42 -0
- package/lib/template-classification/classify-archive-page-templates.js +70 -0
- package/lib/template-classification/classify-page-templates.d.ts +33 -0
- package/lib/template-classification/classify-page-templates.js +60 -0
- package/lib/template-classification/collect-page-ids-with-html.d.ts +24 -0
- package/lib/template-classification/collect-page-ids-with-html.js +30 -0
- package/lib/template-classification/collect-page-stylesheet-urls.d.ts +32 -0
- package/lib/template-classification/collect-page-stylesheet-urls.js +130 -0
- package/lib/template-classification/create-content-root-hint.d.ts +41 -0
- package/lib/template-classification/create-content-root-hint.js +51 -0
- package/lib/template-classification/create-page-cluster-factory.d.ts +36 -0
- package/lib/template-classification/create-page-cluster-factory.js +52 -0
- package/lib/template-classification/load-page-cluster-candidates.d.ts +32 -0
- package/lib/template-classification/load-page-cluster-candidates.js +93 -0
- package/lib/template-classification/page-cluster/resolve-page-cluster-keys.d.ts +34 -0
- package/lib/template-classification/page-cluster/resolve-page-cluster-keys.js +236 -0
- package/lib/template-classification/page-cluster/types.d.ts +123 -0
- package/lib/template-classification/page-cluster/types.js +1 -0
- package/lib/template-classification/types.d.ts +234 -0
- package/lib/template-classification/types.js +1 -0
- package/lib/test-utils/build-full-schema-test-db.d.ts +34 -0
- package/lib/test-utils/build-full-schema-test-db.js +74 -0
- package/lib/test-utils/create-legacy-analysis-tables.d.ts +13 -0
- package/lib/test-utils/create-legacy-analysis-tables.js +37 -0
- package/lib/test-utils/fk-parent-tables.d.ts +15 -0
- package/lib/test-utils/fk-parent-tables.js +19 -0
- package/lib/test-utils/seed-content-item.d.ts +35 -0
- package/lib/test-utils/seed-content-item.js +42 -0
- package/lib/test-utils/setup-legacy-fk-db.d.ts +34 -0
- package/lib/test-utils/setup-legacy-fk-db.js +271 -0
- package/lib/transfer/attach-source-database.d.ts +31 -0
- package/lib/transfer/attach-source-database.js +40 -0
- package/lib/transfer/build-split-config.d.ts +30 -0
- package/lib/transfer/build-split-config.js +42 -0
- package/lib/transfer/clear-replaced-page-rows.d.ts +23 -0
- package/lib/transfer/clear-replaced-page-rows.js +61 -0
- package/lib/transfer/concat-archives.d.ts +55 -0
- package/lib/transfer/concat-archives.js +95 -0
- package/lib/transfer/concat-source-transfer-phases.d.ts +10 -0
- package/lib/transfer/concat-source-transfer-phases.js +17 -0
- package/lib/transfer/content-item-rank-sql.d.ts +21 -0
- package/lib/transfer/content-item-rank-sql.js +23 -0
- package/lib/transfer/copy-anchor-edges.d.ts +28 -0
- package/lib/transfer/copy-anchor-edges.js +42 -0
- package/lib/transfer/copy-dictionaries-for-concat.d.ts +39 -0
- package/lib/transfer/copy-dictionaries-for-concat.js +81 -0
- package/lib/transfer/copy-dictionaries-for-split.d.ts +29 -0
- package/lib/transfer/copy-dictionaries-for-split.js +224 -0
- package/lib/transfer/copy-header-set-children.d.ts +27 -0
- package/lib/transfer/copy-header-set-children.js +51 -0
- package/lib/transfer/copy-image-items.d.ts +21 -0
- package/lib/transfer/copy-image-items.js +62 -0
- package/lib/transfer/copy-inventory-source-lists.d.ts +30 -0
- package/lib/transfer/copy-inventory-source-lists.js +61 -0
- package/lib/transfer/copy-journal-tables.d.ts +30 -0
- package/lib/transfer/copy-journal-tables.js +56 -0
- package/lib/transfer/copy-page-console-logs.d.ts +16 -0
- package/lib/transfer/copy-page-console-logs.js +27 -0
- package/lib/transfer/copy-page-html-blobs.d.ts +31 -0
- package/lib/transfer/copy-page-html-blobs.js +43 -0
- package/lib/transfer/copy-page-meta.d.ts +27 -0
- package/lib/transfer/copy-page-meta.js +68 -0
- package/lib/transfer/copy-page-template-clusters.d.ts +31 -0
- package/lib/transfer/copy-page-template-clusters.js +56 -0
- package/lib/transfer/copy-page-template-labels.d.ts +34 -0
- package/lib/transfer/copy-page-template-labels.js +88 -0
- package/lib/transfer/copy-resource-items.d.ts +29 -0
- package/lib/transfer/copy-resource-items.js +89 -0
- package/lib/transfer/copy-resource-ref-edges.d.ts +20 -0
- package/lib/transfer/copy-resource-ref-edges.js +33 -0
- package/lib/transfer/copy-simple-page-scoped-tables.d.ts +15 -0
- package/lib/transfer/copy-simple-page-scoped-tables.js +34 -0
- package/lib/transfer/count-external-pages-in-scope.d.ts +26 -0
- package/lib/transfer/count-external-pages-in-scope.js +40 -0
- package/lib/transfer/create-transfer-temp-tables.d.ts +30 -0
- package/lib/transfer/create-transfer-temp-tables.js +59 -0
- package/lib/transfer/dictionary-copy-specs.d.ts +32 -0
- package/lib/transfer/dictionary-copy-specs.js +28 -0
- package/lib/transfer/drop-transfer-temp-tables.d.ts +9 -0
- package/lib/transfer/drop-transfer-temp-tables.js +18 -0
- package/lib/transfer/flatten-redirect-chains.d.ts +44 -0
- package/lib/transfer/flatten-redirect-chains.js +74 -0
- package/lib/transfer/insert-content-items.d.ts +35 -0
- package/lib/transfer/insert-content-items.js +116 -0
- package/lib/transfer/list-transfer-columns.d.ts +35 -0
- package/lib/transfer/list-transfer-columns.js +45 -0
- package/lib/transfer/mark-in-scope-content-items.d.ts +31 -0
- package/lib/transfer/mark-in-scope-content-items.js +54 -0
- package/lib/transfer/merge-archive-configs.d.ts +51 -0
- package/lib/transfer/merge-archive-configs.js +85 -0
- package/lib/transfer/plan-content-items-for-concat.d.ts +33 -0
- package/lib/transfer/plan-content-items-for-concat.js +51 -0
- package/lib/transfer/plan-content-items-for-split.d.ts +42 -0
- package/lib/transfer/plan-content-items-for-split.js +83 -0
- package/lib/transfer/plan-resource-items.d.ts +40 -0
- package/lib/transfer/plan-resource-items.js +69 -0
- package/lib/transfer/quote-transfer-identifier.d.ts +18 -0
- package/lib/transfer/quote-transfer-identifier.js +20 -0
- package/lib/transfer/reclassify-resource-externality.d.ts +37 -0
- package/lib/transfer/reclassify-resource-externality.js +66 -0
- package/lib/transfer/remap-redirect-dest-ids.d.ts +34 -0
- package/lib/transfer/remap-redirect-dest-ids.js +51 -0
- package/lib/transfer/replace-content-items.d.ts +39 -0
- package/lib/transfer/replace-content-items.js +83 -0
- package/lib/transfer/simple-page-scoped-tables.d.ts +25 -0
- package/lib/transfer/simple-page-scoped-tables.js +38 -0
- package/lib/transfer/split-archive.d.ts +55 -0
- package/lib/transfer/split-archive.js +98 -0
- package/lib/transfer/split-source-transfer-phases.d.ts +11 -0
- package/lib/transfer/split-source-transfer-phases.js +18 -0
- package/lib/transfer/transfer-action.d.ts +34 -0
- package/lib/transfer/transfer-action.js +32 -0
- package/lib/transfer/transfer-archive-rows.d.ts +65 -0
- package/lib/transfer/transfer-archive-rows.js +156 -0
- package/lib/transfer/transfer-dictionary-tables.d.ts +12 -0
- package/lib/transfer/transfer-dictionary-tables.js +22 -0
- package/lib/transfer/transfer-post-phases.d.ts +10 -0
- package/lib/transfer/transfer-post-phases.js +13 -0
- package/lib/transfer/transfer-source-alias.d.ts +9 -0
- package/lib/transfer/transfer-source-alias.js +9 -0
- package/lib/transfer/types.d.ts +111 -0
- package/lib/transfer/types.js +35 -0
- package/lib/types.d.ts +772 -0
- package/lib/types.js +1 -0
- package/lib/url-alias/compute-tier-a-alias-key.d.ts +44 -0
- package/lib/url-alias/compute-tier-a-alias-key.js +47 -0
- package/lib/url-alias/compute-tier-b-alias-key.d.ts +23 -0
- package/lib/url-alias/compute-tier-b-alias-key.js +26 -0
- package/lib/url-alias/group-alias-candidates.d.ts +27 -0
- package/lib/url-alias/group-alias-candidates.js +29 -0
- package/lib/url-alias/types.d.ts +12 -0
- package/lib/url-alias/types.js +1 -0
- package/lib/url-pattern/compute-shape-key.d.ts +58 -0
- package/lib/url-pattern/compute-shape-key.js +129 -0
- package/lib/url-pattern/decompose-url.d.ts +33 -0
- package/lib/url-pattern/decompose-url.js +79 -0
- package/lib/url-pattern/reconstruct-url.d.ts +15 -0
- package/lib/url-pattern/reconstruct-url.js +33 -0
- package/lib/url-sort/compute-url-sort-ranks.d.ts +30 -0
- package/lib/url-sort/compute-url-sort-ranks.js +33 -0
- package/lib/utils/array/each-splitted.d.ts +10 -0
- package/lib/utils/array/each-splitted.js +14 -0
- package/lib/utils/debug.d.ts +8 -0
- package/lib/utils/debug.js +8 -0
- package/lib/utils/error/emit-error-with-retry.d.ts +40 -0
- package/lib/utils/error/emit-error-with-retry.js +44 -0
- package/lib/utils/error/emit-error.d.ts +39 -0
- package/lib/utils/error/emit-error.js +41 -0
- package/lib/utils/object/parse-response-headers.d.ts +12 -0
- package/lib/utils/object/parse-response-headers.js +26 -0
- package/lib/utils/object/redact-request-headers.d.ts +15 -0
- package/lib/utils/object/redact-request-headers.js +27 -0
- package/lib/utils/types/types.d.ts +383 -0
- package/lib/utils/types/types.js +1 -0
- package/lib/verify-migration/capture-rejection.d.ts +24 -0
- package/lib/verify-migration/capture-rejection.js +31 -0
- package/lib/verify-migration/check-anchor-edges-count.d.ts +34 -0
- package/lib/verify-migration/check-anchor-edges-count.js +72 -0
- package/lib/verify-migration/check-anchor-edges-sum.d.ts +13 -0
- package/lib/verify-migration/check-anchor-edges-sum.js +27 -0
- package/lib/verify-migration/check-content-items-count.d.ts +16 -0
- package/lib/verify-migration/check-content-items-count.js +30 -0
- package/lib/verify-migration/check-content-type-preservation.d.ts +22 -0
- package/lib/verify-migration/check-content-type-preservation.js +40 -0
- package/lib/verify-migration/check-foreign-key-integrity.d.ts +31 -0
- package/lib/verify-migration/check-foreign-key-integrity.js +47 -0
- package/lib/verify-migration/check-image-items-count.d.ts +12 -0
- package/lib/verify-migration/check-image-items-count.js +26 -0
- package/lib/verify-migration/check-page-meta-count.d.ts +15 -0
- package/lib/verify-migration/check-page-meta-count.js +31 -0
- package/lib/verify-migration/check-reader-parity.d.ts +23 -0
- package/lib/verify-migration/check-reader-parity.js +182 -0
- package/lib/verify-migration/check-resource-items-count.d.ts +17 -0
- package/lib/verify-migration/check-resource-items-count.js +33 -0
- package/lib/verify-migration/check-url-round-trip.d.ts +43 -0
- package/lib/verify-migration/check-url-round-trip.js +112 -0
- package/lib/verify-migration/types.d.ts +70 -0
- package/lib/verify-migration/types.js +63 -0
- package/lib/verify-migration/verify-migration.d.ts +41 -0
- package/lib/verify-migration/verify-migration.js +120 -0
- package/lib/with-bulk-write-pragmas.d.ts +51 -0
- package/lib/with-bulk-write-pragmas.js +132 -0
- package/lib/write-queue.d.ts +32 -0
- package/lib/write-queue.js +55 -0
- package/package.json +360 -5
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { acquireArchiveLock, ArchiveLockError } from '../archive-lock.js';
|
|
4
|
+
import { Database } from '../database.js';
|
|
5
|
+
import { peekTarTopDir } from '../filesystem/peek-tar-top-dir.js';
|
|
6
|
+
import { rename } from '../filesystem/rename.js';
|
|
7
|
+
import { untar } from '../filesystem/untar.js';
|
|
8
|
+
import { IncompatibleArchiveError } from '../meta/types.js';
|
|
9
|
+
import { computeArchiveCacheKey } from './compute-archive-cache-key.js';
|
|
10
|
+
/**
|
|
11
|
+
* How long {@link extractArchiveToCache} waits in total for a peer
|
|
12
|
+
* extractor to finish (ms). 5 minutes is enough for the largest archives
|
|
13
|
+
* the crawler currently produces (~10 GB untars in ~10 s; 5 min gives
|
|
14
|
+
* 30× headroom) while still bounded so a deadlocked process eventually
|
|
15
|
+
* surfaces as a real error instead of hanging the viewer indefinitely.
|
|
16
|
+
*/
|
|
17
|
+
const PEER_WAIT_TIMEOUT_MS = 5 * 60 * 1000;
|
|
18
|
+
/**
|
|
19
|
+
* Polling interval while waiting on a peer extractor (ms). Short enough
|
|
20
|
+
* to keep latency low when the peer is fast, long enough to avoid
|
|
21
|
+
* burning CPU on a busy `fs.stat` loop.
|
|
22
|
+
*/
|
|
23
|
+
const PEER_WAIT_POLL_MS = 50;
|
|
24
|
+
/**
|
|
25
|
+
* Marker file dropped into a cache directory only after the extraction
|
|
26
|
+
* has fully completed AND the contents have passed a structural sanity
|
|
27
|
+
* check (db.sqlite present, schema compatible, migrations applied).
|
|
28
|
+
*
|
|
29
|
+
* Internal — the file name is an implementation detail. Tests should
|
|
30
|
+
* assert through externally observable behaviour (cache hit on second
|
|
31
|
+
* open) rather than importing this constant.
|
|
32
|
+
*/
|
|
33
|
+
const READY_MARKER = '.nitpicker-cache-ready';
|
|
34
|
+
/**
|
|
35
|
+
* In-process deduplication for concurrent {@link extractArchiveToCache}
|
|
36
|
+
* calls targeting the same `cacheDir`.
|
|
37
|
+
*
|
|
38
|
+
* The file-based {@link acquireArchiveLock} (mkdir-based) cannot defend
|
|
39
|
+
* against same-process race windows because the lock holder writes its
|
|
40
|
+
* `pid.txt` in a separate `await` from the directory creation: a sibling
|
|
41
|
+
* promise that observes the `EEXIST` between those two steps sees an
|
|
42
|
+
* empty `pid.txt`, mistakes the lock for stale, and clobbers it.
|
|
43
|
+
*
|
|
44
|
+
* Single-process dedup eliminates that race entirely for the common
|
|
45
|
+
* "viewer + MCP in the same process" topology while still letting the
|
|
46
|
+
* inner file lock guard cross-process collisions.
|
|
47
|
+
*/
|
|
48
|
+
const inFlightByCacheDir = new Map();
|
|
49
|
+
/**
|
|
50
|
+
* Untar a `.nitpicker` archive into the given cache directory.
|
|
51
|
+
*
|
|
52
|
+
* Concurrency / re-entry contract:
|
|
53
|
+
*
|
|
54
|
+
* - Two viewers opening the same archive race for the cache lock; the
|
|
55
|
+
* loser waits and recheck-loops until the winner writes the ready
|
|
56
|
+
* marker, then short-circuits.
|
|
57
|
+
* - Same-process concurrent callers dedupe through
|
|
58
|
+
* `inFlightByCacheDir` so they share one extraction promise.
|
|
59
|
+
*
|
|
60
|
+
* Per-archive staging:
|
|
61
|
+
*
|
|
62
|
+
* - The tar's inner directory is extracted into `${cacheDir}.staging/`
|
|
63
|
+
* first, then atomically renamed into `cacheDir`. Crucially, the
|
|
64
|
+
* staging path is keyed off `cacheDir` (not the tar's inner directory
|
|
65
|
+
* name) so two archives that happen to share an inner-dir name never
|
|
66
|
+
* collide on the same staging slot.
|
|
67
|
+
*
|
|
68
|
+
* Integrity guard:
|
|
69
|
+
*
|
|
70
|
+
* - After rename, we run the writer-side `Database.connect` so all
|
|
71
|
+
* migrations apply (column adds, new tables) and the version check
|
|
72
|
+
* fires before the ready marker is written. A corrupt or incompatible
|
|
73
|
+
* archive therefore poisons no cache entry — the cacheDir is removed
|
|
74
|
+
* and the next caller retries from scratch.
|
|
75
|
+
*
|
|
76
|
+
* TOCTOU guard:
|
|
77
|
+
*
|
|
78
|
+
* - The cache key is recomputed from the file's stat AFTER the untar
|
|
79
|
+
* completes. If the file changed mid-flight (a concurrent
|
|
80
|
+
* `crawl --append` rewrote it), the freshly-landed cache contents do
|
|
81
|
+
* not correspond to the original key, so we abort and remove the
|
|
82
|
+
* cacheDir so the caller upstream sees the inconsistency rather than
|
|
83
|
+
* serving misattributed data.
|
|
84
|
+
* @param archivePath - Absolute path to the source `.nitpicker` file.
|
|
85
|
+
* @param cacheRoot - Absolute path to the cache root directory.
|
|
86
|
+
* @param cacheDir - Absolute path the extracted contents should end up at.
|
|
87
|
+
* @param cacheKey - The cache key used to derive `cacheDir`. Recomputed
|
|
88
|
+
* after extraction to detect concurrent writers; must match.
|
|
89
|
+
* @param onExtractProgress - Called during the untar step with bytes read
|
|
90
|
+
* so far and the archive's total size (issue #294: a cold cache on a
|
|
91
|
+
* large archive can take tens of seconds with no other signal it isn't
|
|
92
|
+
* hung). Never called on a cache hit — {@link isCacheDirReady} short-
|
|
93
|
+
* circuits before `untar` runs — so callers can treat "never invoked" as
|
|
94
|
+
* the definition of a hit and skip printing a phase label until the
|
|
95
|
+
* first call actually arrives. Only the first concurrent caller for a
|
|
96
|
+
* given `cacheDir` sees callbacks; same-`cacheDir` callers deduped
|
|
97
|
+
* through {@link inFlightByCacheDir} just await the shared promise.
|
|
98
|
+
* @param onLog - Forwarded to the cache-miss writer connection
|
|
99
|
+
* (`runMigrationsOnCacheDir`'s `Database.connect`) so a legacy archive's
|
|
100
|
+
* self-healing schema migrations do not fall back to a bare
|
|
101
|
+
* `console.error` mid-redraw of a caller's display (issue #294). Omit for
|
|
102
|
+
* that fallback — see `Archive.openCached`'s own `onLog` doc.
|
|
103
|
+
* @returns Resolves once `cacheDir` is ready to be opened read-only.
|
|
104
|
+
*/
|
|
105
|
+
export async function extractArchiveToCache(archivePath, cacheRoot, cacheDir, cacheKey, onExtractProgress, onLog) {
|
|
106
|
+
if (await isCacheDirReady(cacheDir)) {
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
const existing = inFlightByCacheDir.get(cacheDir);
|
|
110
|
+
if (existing) {
|
|
111
|
+
return existing;
|
|
112
|
+
}
|
|
113
|
+
const promise = runExtraction(archivePath, cacheRoot, cacheDir, cacheKey, onExtractProgress, onLog).finally(() => {
|
|
114
|
+
inFlightByCacheDir.delete(cacheDir);
|
|
115
|
+
});
|
|
116
|
+
inFlightByCacheDir.set(cacheDir, promise);
|
|
117
|
+
return promise;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* The actual extraction work. Split out from {@link extractArchiveToCache}
|
|
121
|
+
* so the in-process dedup map can wrap it without disturbing the
|
|
122
|
+
* extraction flow itself.
|
|
123
|
+
* @param archivePath - Absolute path to the source `.nitpicker` file.
|
|
124
|
+
* @param cacheRoot - Absolute path to the cache root directory.
|
|
125
|
+
* @param cacheDir - Absolute path the extracted contents should end up at.
|
|
126
|
+
* @param cacheKey - Pre-extraction cache key, re-verified post-extraction.
|
|
127
|
+
* @param onExtractProgress - See {@link extractArchiveToCache}.
|
|
128
|
+
* @param onLog - See {@link extractArchiveToCache}.
|
|
129
|
+
*/
|
|
130
|
+
async function runExtraction(archivePath, cacheRoot, cacheDir, cacheKey, onExtractProgress, onLog) {
|
|
131
|
+
await fs.mkdir(cacheRoot, { recursive: true });
|
|
132
|
+
const releaseLock = await acquireLockWithPeerWait(cacheDir);
|
|
133
|
+
try {
|
|
134
|
+
if (await isCacheDirReady(cacheDir)) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
// Half-populated cacheDir recovery: another extractor crashed
|
|
138
|
+
// before writing the marker. Rename it aside instead of `rm`-ing
|
|
139
|
+
// in place so any reader that somehow still holds an fd on the
|
|
140
|
+
// old contents (the cache layer doesn't refcount readers) is not
|
|
141
|
+
// pulled out from under. The renamed quarantine dir is left for
|
|
142
|
+
// OS-level temp cleanup.
|
|
143
|
+
await quarantineHalfPopulatedCacheDir(cacheDir);
|
|
144
|
+
const stagingDir = `${cacheDir}.staging`;
|
|
145
|
+
// Stage clean: a previous crashed run may have left
|
|
146
|
+
// `<cacheDir>.staging/` around. `untar` would happily merge into
|
|
147
|
+
// it (with `newer:true` cherrypicking entries) so wipe first.
|
|
148
|
+
await fs.rm(stagingDir, { recursive: true, force: true });
|
|
149
|
+
await fs.mkdir(stagingDir, { recursive: true });
|
|
150
|
+
await untar(archivePath, { cwd: stagingDir, onProgress: onExtractProgress });
|
|
151
|
+
// Concurrent-writer detection: if the source archive changed
|
|
152
|
+
// during our untar, the contents we just landed do NOT match the
|
|
153
|
+
// `cacheKey` that named the directory. Drop the staging dir and
|
|
154
|
+
// surface as an error so the caller (or a retry) can pick up the
|
|
155
|
+
// new key.
|
|
156
|
+
const postKey = await computeArchiveCacheKey(archivePath);
|
|
157
|
+
if (postKey !== cacheKey) {
|
|
158
|
+
await fs.rm(stagingDir, { recursive: true, force: true });
|
|
159
|
+
throw new Error(`Archive changed during cache extraction (key ${cacheKey} -> ${postKey}); ` +
|
|
160
|
+
`refusing to land mismatched contents at ${cacheDir}.`);
|
|
161
|
+
}
|
|
162
|
+
const innerDirName = await peekTarTopDir(archivePath);
|
|
163
|
+
const extractedInner = path.resolve(stagingDir, innerDirName);
|
|
164
|
+
// Move the inner directory into the cache slot. Any old quarantine
|
|
165
|
+
// is already aside, so the target is guaranteed empty.
|
|
166
|
+
await rename(extractedInner, cacheDir, true);
|
|
167
|
+
// The empty `stagingDir` wrapper is no longer useful — best effort
|
|
168
|
+
// remove (ignore errors, OS cleanup catches stragglers).
|
|
169
|
+
await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => { });
|
|
170
|
+
// Validate + migrate: open in WRITER mode so the full migration
|
|
171
|
+
// stack runs (initSchema / migrate*). On failure (db.sqlite
|
|
172
|
+
// missing, incompatible archive, broken schema), tear down the
|
|
173
|
+
// cache entry so the next caller does not get stuck on a
|
|
174
|
+
// "ready"-marked but broken cache.
|
|
175
|
+
try {
|
|
176
|
+
await runMigrationsOnCacheDir(cacheDir, onLog);
|
|
177
|
+
}
|
|
178
|
+
catch (error) {
|
|
179
|
+
await fs.rm(cacheDir, { recursive: true, force: true });
|
|
180
|
+
throw error;
|
|
181
|
+
}
|
|
182
|
+
await fs.writeFile(path.join(cacheDir, READY_MARKER), '', 'utf8');
|
|
183
|
+
}
|
|
184
|
+
finally {
|
|
185
|
+
await releaseLock();
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Acquire the cache lock, blocking until any peer extractor finishes
|
|
190
|
+
* (rather than failing fast on `EEXIST` the way the writer-oriented
|
|
191
|
+
* {@link acquireArchiveLock} does). Returns the release function once
|
|
192
|
+
* the lock is held.
|
|
193
|
+
* @param cacheDir - Absolute path the lock guards.
|
|
194
|
+
*/
|
|
195
|
+
async function acquireLockWithPeerWait(cacheDir) {
|
|
196
|
+
const startedAt = Date.now();
|
|
197
|
+
while (true) {
|
|
198
|
+
if (await isCacheDirReady(cacheDir)) {
|
|
199
|
+
// Peer finished and the marker is up. Return a no-op release
|
|
200
|
+
// so the caller can `finally`-await it without branching.
|
|
201
|
+
return async () => { };
|
|
202
|
+
}
|
|
203
|
+
try {
|
|
204
|
+
return await acquireArchiveLock(cacheDir);
|
|
205
|
+
}
|
|
206
|
+
catch (error) {
|
|
207
|
+
if (!(error instanceof ArchiveLockError)) {
|
|
208
|
+
throw error;
|
|
209
|
+
}
|
|
210
|
+
if (Date.now() - startedAt > PEER_WAIT_TIMEOUT_MS) {
|
|
211
|
+
throw error;
|
|
212
|
+
}
|
|
213
|
+
await sleep(PEER_WAIT_POLL_MS);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Open the migrated DB in writer mode so all migrations apply, then
|
|
219
|
+
* release the handle. Cache entries land migrated, so the subsequent
|
|
220
|
+
* `Archive.connect` read-only open never needs to mutate the cache dir
|
|
221
|
+
* (and never silently misses a newly-added column).
|
|
222
|
+
* @param cacheDir - Absolute path to the freshly-extracted cache dir.
|
|
223
|
+
* @param onLog - See {@link extractArchiveToCache}.
|
|
224
|
+
*/
|
|
225
|
+
async function runMigrationsOnCacheDir(cacheDir, onLog) {
|
|
226
|
+
const dbPath = path.join(cacheDir, 'db.sqlite');
|
|
227
|
+
if (!(await fileExists(dbPath))) {
|
|
228
|
+
throw new Error(`Cache directory does not contain db.sqlite: ${cacheDir}`);
|
|
229
|
+
}
|
|
230
|
+
const db = await Database.connect({ filename: dbPath, readOnly: false, onLog });
|
|
231
|
+
try {
|
|
232
|
+
// `Database.connect` ran migrations during init. Closing here
|
|
233
|
+
// flushes WAL + drops the handle so the read-only re-open in
|
|
234
|
+
// `Archive.connect` does not race the writer connection.
|
|
235
|
+
await db.destroy();
|
|
236
|
+
}
|
|
237
|
+
catch (error) {
|
|
238
|
+
// Propagate after attempting cleanup of the connection (best
|
|
239
|
+
// effort — the cache dir tear-down happens in the caller).
|
|
240
|
+
if (error instanceof IncompatibleArchiveError) {
|
|
241
|
+
throw error;
|
|
242
|
+
}
|
|
243
|
+
throw error;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Move a half-populated cacheDir aside so the next extraction can land
|
|
248
|
+
* cleanly without yanking live readers' files out from under them. The
|
|
249
|
+
* quarantine path gets a unique suffix so two crashes in succession
|
|
250
|
+
* don't collide.
|
|
251
|
+
* @param cacheDir - Absolute path of the cache slot to free.
|
|
252
|
+
*/
|
|
253
|
+
async function quarantineHalfPopulatedCacheDir(cacheDir) {
|
|
254
|
+
try {
|
|
255
|
+
await fs.access(cacheDir);
|
|
256
|
+
}
|
|
257
|
+
catch {
|
|
258
|
+
// Nothing to quarantine.
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
if (await isCacheDirReady(cacheDir)) {
|
|
262
|
+
// Already settled — caller will short-circuit, no quarantine needed.
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
const quarantinePath = `${cacheDir}.corrupt.${process.pid}.${nextQuarantineCounter()}`;
|
|
266
|
+
try {
|
|
267
|
+
await fs.rename(cacheDir, quarantinePath);
|
|
268
|
+
}
|
|
269
|
+
catch {
|
|
270
|
+
// If rename fails (e.g. cross-volume), fall back to direct
|
|
271
|
+
// removal — accepting the rm-while-reader risk this single time
|
|
272
|
+
// since we have no other option.
|
|
273
|
+
await fs.rm(cacheDir, { recursive: true, force: true });
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Monotonic counter appended to quarantine directory names so multiple
|
|
278
|
+
* recoveries in a single process never collide on the same path. Resets
|
|
279
|
+
* to zero after `2 ** 32` increments; that bound is far larger than any
|
|
280
|
+
* realistic per-process quarantine count.
|
|
281
|
+
*/
|
|
282
|
+
let quarantineCounter = 0;
|
|
283
|
+
/**
|
|
284
|
+
* Issue the next quarantine sequence number.
|
|
285
|
+
* @returns A non-negative integer, monotonically increasing per process.
|
|
286
|
+
*/
|
|
287
|
+
function nextQuarantineCounter() {
|
|
288
|
+
quarantineCounter = (quarantineCounter + 1) % 0x1_00_00_00_00;
|
|
289
|
+
return quarantineCounter;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Async existence probe — avoids blocking the event loop on the warm
|
|
293
|
+
* cache-hit fast path.
|
|
294
|
+
* @param targetPath - Absolute path to probe.
|
|
295
|
+
* @returns `true` if the path is reachable via `fs.access`.
|
|
296
|
+
*/
|
|
297
|
+
async function fileExists(targetPath) {
|
|
298
|
+
try {
|
|
299
|
+
await fs.access(targetPath);
|
|
300
|
+
return true;
|
|
301
|
+
}
|
|
302
|
+
catch {
|
|
303
|
+
return false;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Whether a cache directory contains a fully-validated extraction. The
|
|
308
|
+
* READY marker is the only trustworthy signal — sibling files alone
|
|
309
|
+
* cannot be trusted because a half-extracted dir can also contain a
|
|
310
|
+
* `db.sqlite`.
|
|
311
|
+
* @param cacheDir - Absolute path to a candidate cache directory.
|
|
312
|
+
* @returns `true` if the marker exists.
|
|
313
|
+
*/
|
|
314
|
+
async function isCacheDirReady(cacheDir) {
|
|
315
|
+
return await fileExists(path.join(cacheDir, READY_MARKER));
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Promise-based sleep used by the peer-wait poll. Kept inline so the
|
|
319
|
+
* file has no external sleep dependency and so the implementation can
|
|
320
|
+
* later swap for an AbortSignal-aware version without touching the
|
|
321
|
+
* call site.
|
|
322
|
+
* @param ms - Milliseconds to wait.
|
|
323
|
+
*/
|
|
324
|
+
function sleep(ms) {
|
|
325
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
326
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve the directory where extracted `.nitpicker` archives are cached.
|
|
3
|
+
*
|
|
4
|
+
* Resolution order:
|
|
5
|
+
*
|
|
6
|
+
* 1. `NITPICKER_TAR_CACHE_DIR` env — explicit override (CI, testing, or
|
|
7
|
+
* operators who want the cache on a specific volume). Must be a
|
|
8
|
+
* path; values that look like boolean / sentinel words (e.g. `0`,
|
|
9
|
+
* `false`) are ignored to keep the cache from landing somewhere
|
|
10
|
+
* surprising when the user mistakes the env contract.
|
|
11
|
+
* 2. `<os.tmpdir()>/nitpicker/cache/` — default. Lives under the OS
|
|
12
|
+
* temp directory so the platform's own cleanup (macOS reboot, Linux
|
|
13
|
+
* `systemd-tmpfiles`, Windows Disk Cleanup) reclaims stale entries
|
|
14
|
+
* without bespoke logic on our side.
|
|
15
|
+
*
|
|
16
|
+
* The returned path is absolute. The caller is responsible for creating
|
|
17
|
+
* it on demand (this function is pure).
|
|
18
|
+
* @returns Absolute path to the cache root directory.
|
|
19
|
+
*/
|
|
20
|
+
export declare function getArchiveCacheRoot(): string;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import os from 'node:os';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
/**
|
|
4
|
+
* Env values that look like a "disable" sentinel rather than a path.
|
|
5
|
+
*
|
|
6
|
+
* A user who copy-pastes the `NITPICKER_DISABLE_TAR_CACHE` convention
|
|
7
|
+
* onto `NITPICKER_TAR_CACHE_DIR` (e.g. `NITPICKER_TAR_CACHE_DIR=0`
|
|
8
|
+
* thinking it means "use default") would otherwise silently land the
|
|
9
|
+
* cache at `$PWD/0/` — multi-gigabyte extracts polluting the project
|
|
10
|
+
* tree, never reclaimed by OS temp cleanup. Reject these explicitly
|
|
11
|
+
* and fall back to the default location instead.
|
|
12
|
+
*/
|
|
13
|
+
const SENTINEL_LIKE_OVERRIDES = new Set([
|
|
14
|
+
'0',
|
|
15
|
+
'1',
|
|
16
|
+
'false',
|
|
17
|
+
'true',
|
|
18
|
+
'no',
|
|
19
|
+
'yes',
|
|
20
|
+
'off',
|
|
21
|
+
'on',
|
|
22
|
+
'null',
|
|
23
|
+
'undefined',
|
|
24
|
+
]);
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the directory where extracted `.nitpicker` archives are cached.
|
|
27
|
+
*
|
|
28
|
+
* Resolution order:
|
|
29
|
+
*
|
|
30
|
+
* 1. `NITPICKER_TAR_CACHE_DIR` env — explicit override (CI, testing, or
|
|
31
|
+
* operators who want the cache on a specific volume). Must be a
|
|
32
|
+
* path; values that look like boolean / sentinel words (e.g. `0`,
|
|
33
|
+
* `false`) are ignored to keep the cache from landing somewhere
|
|
34
|
+
* surprising when the user mistakes the env contract.
|
|
35
|
+
* 2. `<os.tmpdir()>/nitpicker/cache/` — default. Lives under the OS
|
|
36
|
+
* temp directory so the platform's own cleanup (macOS reboot, Linux
|
|
37
|
+
* `systemd-tmpfiles`, Windows Disk Cleanup) reclaims stale entries
|
|
38
|
+
* without bespoke logic on our side.
|
|
39
|
+
*
|
|
40
|
+
* The returned path is absolute. The caller is responsible for creating
|
|
41
|
+
* it on demand (this function is pure).
|
|
42
|
+
* @returns Absolute path to the cache root directory.
|
|
43
|
+
*/
|
|
44
|
+
export function getArchiveCacheRoot() {
|
|
45
|
+
const envOverride = process.env.NITPICKER_TAR_CACHE_DIR;
|
|
46
|
+
if (envOverride && envOverride.trim().length > 0) {
|
|
47
|
+
const trimmed = envOverride.trim();
|
|
48
|
+
if (!SENTINEL_LIKE_OVERRIDES.has(trimmed.toLowerCase())) {
|
|
49
|
+
return path.resolve(trimmed);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return path.resolve(os.tmpdir(), 'nitpicker', 'cache');
|
|
53
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether the tar cache is disabled by the `NITPICKER_DISABLE_TAR_CACHE`
|
|
3
|
+
* env var.
|
|
4
|
+
*
|
|
5
|
+
* Useful for two scenarios:
|
|
6
|
+
*
|
|
7
|
+
* 1. Debugging — bypass the cache to reproduce the cold-start behaviour
|
|
8
|
+
* against a fresh tmpDir.
|
|
9
|
+
* 2. Sandboxed CI — when the runner's tmpfs would not survive between
|
|
10
|
+
* steps anyway, the cache only adds first-step overhead.
|
|
11
|
+
*
|
|
12
|
+
* Accepted truthy values: `1`, `true`, `yes`, `on` (case-insensitive).
|
|
13
|
+
* Anything else (including unset) keeps the cache enabled.
|
|
14
|
+
* @returns `true` when the cache should be bypassed.
|
|
15
|
+
* @example
|
|
16
|
+
* ```ts
|
|
17
|
+
* if (isArchiveCacheDisabled()) {
|
|
18
|
+
* // Fall back to the writer path; cwd tmpDir + close-time cleanup.
|
|
19
|
+
* const archive = await Archive.open({ filePath });
|
|
20
|
+
* ...
|
|
21
|
+
* }
|
|
22
|
+
* ```
|
|
23
|
+
*/
|
|
24
|
+
export declare function isArchiveCacheDisabled(): boolean;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether the tar cache is disabled by the `NITPICKER_DISABLE_TAR_CACHE`
|
|
3
|
+
* env var.
|
|
4
|
+
*
|
|
5
|
+
* Useful for two scenarios:
|
|
6
|
+
*
|
|
7
|
+
* 1. Debugging — bypass the cache to reproduce the cold-start behaviour
|
|
8
|
+
* against a fresh tmpDir.
|
|
9
|
+
* 2. Sandboxed CI — when the runner's tmpfs would not survive between
|
|
10
|
+
* steps anyway, the cache only adds first-step overhead.
|
|
11
|
+
*
|
|
12
|
+
* Accepted truthy values: `1`, `true`, `yes`, `on` (case-insensitive).
|
|
13
|
+
* Anything else (including unset) keeps the cache enabled.
|
|
14
|
+
* @returns `true` when the cache should be bypassed.
|
|
15
|
+
* @example
|
|
16
|
+
* ```ts
|
|
17
|
+
* if (isArchiveCacheDisabled()) {
|
|
18
|
+
* // Fall back to the writer path; cwd tmpDir + close-time cleanup.
|
|
19
|
+
* const archive = await Archive.open({ filePath });
|
|
20
|
+
* ...
|
|
21
|
+
* }
|
|
22
|
+
* ```
|
|
23
|
+
*/
|
|
24
|
+
export function isArchiveCacheDisabled() {
|
|
25
|
+
const raw = process.env.NITPICKER_DISABLE_TAR_CACHE;
|
|
26
|
+
if (!raw) {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
const normalized = raw.trim().toLowerCase();
|
|
30
|
+
return (normalized === '1' ||
|
|
31
|
+
normalized === 'true' ||
|
|
32
|
+
normalized === 'yes' ||
|
|
33
|
+
normalized === 'on');
|
|
34
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { ArchiveCacheEntry } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* List the top-level entries found directly under an archive cache root,
|
|
4
|
+
* with a recursive size total and the most recent mtime found in each.
|
|
5
|
+
*
|
|
6
|
+
* Pure: takes `cacheRoot` as a parameter and never resolves it itself (see
|
|
7
|
+
* {@link import('./get-archive-cache-root.js').getArchiveCacheRoot} for the
|
|
8
|
+
* production root), so tests can point it at a throwaway directory instead
|
|
9
|
+
* of the real OS temp cache.
|
|
10
|
+
*
|
|
11
|
+
* Symbolic links are reported but never followed, both at the top level and
|
|
12
|
+
* while walking a directory's contents — this avoids escaping `cacheRoot`
|
|
13
|
+
* and crashing on dangling links.
|
|
14
|
+
* @param cacheRoot - Absolute path to the cache root to inspect.
|
|
15
|
+
* @returns One entry per top-level child of `cacheRoot`, in `fs.readdir`
|
|
16
|
+
* order. Returns `[]` if `cacheRoot` does not exist.
|
|
17
|
+
* @example
|
|
18
|
+
* ```ts
|
|
19
|
+
* const entries = await listArchiveCacheEntries(getArchiveCacheRoot());
|
|
20
|
+
* const totalBytes = entries.reduce((sum, e) => sum + e.sizeBytes, 0);
|
|
21
|
+
* ```
|
|
22
|
+
*/
|
|
23
|
+
export declare function listArchiveCacheEntries(cacheRoot: string): Promise<ArchiveCacheEntry[]>;
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
/**
|
|
4
|
+
* List the top-level entries found directly under an archive cache root,
|
|
5
|
+
* with a recursive size total and the most recent mtime found in each.
|
|
6
|
+
*
|
|
7
|
+
* Pure: takes `cacheRoot` as a parameter and never resolves it itself (see
|
|
8
|
+
* {@link import('./get-archive-cache-root.js').getArchiveCacheRoot} for the
|
|
9
|
+
* production root), so tests can point it at a throwaway directory instead
|
|
10
|
+
* of the real OS temp cache.
|
|
11
|
+
*
|
|
12
|
+
* Symbolic links are reported but never followed, both at the top level and
|
|
13
|
+
* while walking a directory's contents — this avoids escaping `cacheRoot`
|
|
14
|
+
* and crashing on dangling links.
|
|
15
|
+
* @param cacheRoot - Absolute path to the cache root to inspect.
|
|
16
|
+
* @returns One entry per top-level child of `cacheRoot`, in `fs.readdir`
|
|
17
|
+
* order. Returns `[]` if `cacheRoot` does not exist.
|
|
18
|
+
* @example
|
|
19
|
+
* ```ts
|
|
20
|
+
* const entries = await listArchiveCacheEntries(getArchiveCacheRoot());
|
|
21
|
+
* const totalBytes = entries.reduce((sum, e) => sum + e.sizeBytes, 0);
|
|
22
|
+
* ```
|
|
23
|
+
*/
|
|
24
|
+
export async function listArchiveCacheEntries(cacheRoot) {
|
|
25
|
+
let topLevel;
|
|
26
|
+
try {
|
|
27
|
+
topLevel = await fs.readdir(cacheRoot, { withFileTypes: true });
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (error.code === 'ENOENT') {
|
|
31
|
+
return [];
|
|
32
|
+
}
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
const entries = [];
|
|
36
|
+
for (const dirent of topLevel) {
|
|
37
|
+
const entryPath = path.join(cacheRoot, dirent.name);
|
|
38
|
+
if (dirent.isSymbolicLink()) {
|
|
39
|
+
entries.push({
|
|
40
|
+
kind: 'unknown',
|
|
41
|
+
name: dirent.name,
|
|
42
|
+
path: entryPath,
|
|
43
|
+
sizeBytes: 0,
|
|
44
|
+
mtimeMs: 0,
|
|
45
|
+
});
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
if (dirent.isDirectory()) {
|
|
49
|
+
const { sizeBytes, mtimeMs } = await computeDirectoryStats(entryPath);
|
|
50
|
+
entries.push({
|
|
51
|
+
kind: classifyEntryName(dirent.name),
|
|
52
|
+
name: dirent.name,
|
|
53
|
+
path: entryPath,
|
|
54
|
+
sizeBytes,
|
|
55
|
+
mtimeMs,
|
|
56
|
+
});
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const stat = await fs.stat(entryPath);
|
|
60
|
+
entries.push({
|
|
61
|
+
kind: 'unknown',
|
|
62
|
+
name: dirent.name,
|
|
63
|
+
path: entryPath,
|
|
64
|
+
sizeBytes: stat.size,
|
|
65
|
+
mtimeMs: stat.mtimeMs,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
return entries;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Matches the corrupt-quarantine suffix `extractArchiveToCache` appends
|
|
72
|
+
* (`${cacheDir}.corrupt.${pid}.${counter}`, both segments always numeric —
|
|
73
|
+
* see `quarantineHalfPopulatedCacheDir` in `extract-archive-to-cache.ts`).
|
|
74
|
+
* Anchored and numeric-specific rather than a loose `includes('.corrupt.')`
|
|
75
|
+
* so an archive whose own (sanitized) basename merely contains the
|
|
76
|
+
* substring `.corrupt.` — e.g. `my.corrupt.report.nitpicker` — is not
|
|
77
|
+
* misclassified as an orphan and offered up for deletion by `cache list`.
|
|
78
|
+
*/
|
|
79
|
+
const CORRUPT_QUARANTINE_SUFFIX = /\.corrupt\.\d+\.\d+$/;
|
|
80
|
+
/**
|
|
81
|
+
* Name of the sibling scratch directory earlier versions created for
|
|
82
|
+
* analyze-plugin tables. Nothing writes it any more, so a leftover one is
|
|
83
|
+
* reported as an orphan that `cache clear` can reclaim.
|
|
84
|
+
*/
|
|
85
|
+
const LEGACY_TABLE_CACHE_DIR_NAME = 'table';
|
|
86
|
+
/**
|
|
87
|
+
* Classify a cache-root child by name pattern alone (no filesystem access).
|
|
88
|
+
*
|
|
89
|
+
* This is a heuristic: `.staging`/corrupt-quarantine suffixes are appended
|
|
90
|
+
* by `extractArchiveToCache` onto an existing tar-cache dir name, so an
|
|
91
|
+
* archive whose own sanitized basename happens to end in exactly one of
|
|
92
|
+
* these literal suffixes is indistinguishable from a real orphan by name
|
|
93
|
+
* alone. Disambiguating fully would require re-deriving each entry's
|
|
94
|
+
* expected name from its source archive, which `list`/`clear` (by design,
|
|
95
|
+
* see grill-me scope) never has access to. Accepted as a rare, low-cost
|
|
96
|
+
* misclassification.
|
|
97
|
+
* @param name - Base name of the top-level entry.
|
|
98
|
+
*/
|
|
99
|
+
function classifyEntryName(name) {
|
|
100
|
+
if (name === LEGACY_TABLE_CACHE_DIR_NAME ||
|
|
101
|
+
name.endsWith('.staging') ||
|
|
102
|
+
CORRUPT_QUARANTINE_SUFFIX.test(name)) {
|
|
103
|
+
return 'orphan';
|
|
104
|
+
}
|
|
105
|
+
return 'tar-cache';
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Recursively sum file sizes and find the most recent mtime under a
|
|
109
|
+
* directory. Symbolic links are neither followed nor counted.
|
|
110
|
+
* @param dirPath - Absolute path to the directory to walk.
|
|
111
|
+
*/
|
|
112
|
+
async function computeDirectoryStats(dirPath) {
|
|
113
|
+
let sizeBytes = 0;
|
|
114
|
+
let mtimeMs = 0;
|
|
115
|
+
const children = await fs.readdir(dirPath, { recursive: true, withFileTypes: true });
|
|
116
|
+
for (const child of children) {
|
|
117
|
+
if (!child.isFile()) {
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
const childPath = path.join(child.parentPath, child.name);
|
|
121
|
+
const stat = await fs.stat(childPath);
|
|
122
|
+
sizeBytes += stat.size;
|
|
123
|
+
mtimeMs = Math.max(mtimeMs, stat.mtimeMs);
|
|
124
|
+
}
|
|
125
|
+
if (mtimeMs === 0) {
|
|
126
|
+
const dirStat = await fs.stat(dirPath);
|
|
127
|
+
mtimeMs = dirStat.mtimeMs;
|
|
128
|
+
}
|
|
129
|
+
return { sizeBytes, mtimeMs };
|
|
130
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Async existence probe — avoids blocking the event loop on the common
|
|
3
|
+
* "check before remove" path shared by
|
|
4
|
+
* {@link import('./clear-archive-cache-root.js').clearArchiveCacheRoot} and
|
|
5
|
+
* {@link import('./clear-archive-cache-entry.js').clearArchiveCacheEntry}.
|
|
6
|
+
* @param targetPath - Absolute path to probe.
|
|
7
|
+
* @returns `true` if the path is reachable via `fs.access`.
|
|
8
|
+
*/
|
|
9
|
+
export declare function pathExists(targetPath: string): Promise<boolean>;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
/**
|
|
3
|
+
* Async existence probe — avoids blocking the event loop on the common
|
|
4
|
+
* "check before remove" path shared by
|
|
5
|
+
* {@link import('./clear-archive-cache-root.js').clearArchiveCacheRoot} and
|
|
6
|
+
* {@link import('./clear-archive-cache-entry.js').clearArchiveCacheEntry}.
|
|
7
|
+
* @param targetPath - Absolute path to probe.
|
|
8
|
+
* @returns `true` if the path is reachable via `fs.access`.
|
|
9
|
+
*/
|
|
10
|
+
export async function pathExists(targetPath) {
|
|
11
|
+
try {
|
|
12
|
+
await fs.access(targetPath);
|
|
13
|
+
return true;
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build the absolute path to a single archive's cache directory.
|
|
3
|
+
*
|
|
4
|
+
* The directory name is `<cacheKey>-<safeBasename>`:
|
|
5
|
+
*
|
|
6
|
+
* - `cacheKey` carries the freshness signal (size/mtime/ctime) and is
|
|
7
|
+
* what actually drives cache hit vs miss.
|
|
8
|
+
* - `safeBasename` is appended purely so a human running `ls` on the
|
|
9
|
+
* cache root can recognise which archive an entry belongs to. It is
|
|
10
|
+
* NOT used to disambiguate keys — two archives with identical inode
|
|
11
|
+
* metadata (e.g. an identical copy under a different name) intentionally
|
|
12
|
+
* share an entry under the first basename that landed there.
|
|
13
|
+
*
|
|
14
|
+
* The basename is sanitised: anything outside `[A-Za-z0-9._-]` becomes
|
|
15
|
+
* `_`. This keeps the path portable across filesystems (no spaces,
|
|
16
|
+
* unicode normalisation surprises, Windows-reserved chars) and removes
|
|
17
|
+
* any chance that a crafted archive name could escape the cache root
|
|
18
|
+
* (e.g. via `..` or path separators), independent of the upstream
|
|
19
|
+
* `path.basename` call that already drops directory components.
|
|
20
|
+
* @param cacheRoot - Absolute path returned by `getArchiveCacheRoot()`.
|
|
21
|
+
* @param cacheKey - The freshness key from `computeArchiveCacheKey()`.
|
|
22
|
+
* @param archivePath - Absolute path to the source `.nitpicker` file;
|
|
23
|
+
* only its basename contributes to the cache entry name.
|
|
24
|
+
* @returns Absolute path to the per-archive cache directory.
|
|
25
|
+
*/
|
|
26
|
+
export declare function resolveArchiveCacheDir(cacheRoot: string, cacheKey: string, archivePath: string): string;
|