@nitpicker/archive 0.0.0-stage → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +191 -0
- package/README.md +105 -2
- package/lib/apply-connection-pragmas.d.ts +18 -0
- package/lib/apply-connection-pragmas.js +40 -0
- package/lib/archive-accessor.d.ts +265 -0
- package/lib/archive-accessor.js +477 -0
- package/lib/archive-lock.d.ts +45 -0
- package/lib/archive-lock.js +154 -0
- package/lib/archive.d.ts +736 -0
- package/lib/archive.js +1056 -0
- package/lib/body-hash/compute-body-hash.d.ts +32 -0
- package/lib/body-hash/compute-body-hash.js +35 -0
- package/lib/cache/clear-archive-cache-entry.d.ts +18 -0
- package/lib/cache/clear-archive-cache-entry.js +24 -0
- package/lib/cache/clear-archive-cache-root.d.ts +26 -0
- package/lib/cache/clear-archive-cache-root.js +32 -0
- package/lib/cache/compute-archive-cache-key.d.ts +39 -0
- package/lib/cache/compute-archive-cache-key.js +95 -0
- package/lib/cache/extract-archive-to-cache.d.ts +57 -0
- package/lib/cache/extract-archive-to-cache.js +326 -0
- package/lib/cache/get-archive-cache-root.d.ts +20 -0
- package/lib/cache/get-archive-cache-root.js +53 -0
- package/lib/cache/is-archive-cache-disabled.d.ts +24 -0
- package/lib/cache/is-archive-cache-disabled.js +34 -0
- package/lib/cache/list-archive-cache-entries.d.ts +23 -0
- package/lib/cache/list-archive-cache-entries.js +130 -0
- package/lib/cache/path-exists.d.ts +9 -0
- package/lib/cache/path-exists.js +18 -0
- package/lib/cache/resolve-archive-cache-dir.d.ts +26 -0
- package/lib/cache/resolve-archive-cache-dir.js +32 -0
- package/lib/cache/types.d.ts +33 -0
- package/lib/cache/types.js +1 -0
- package/lib/content-type/is-html-content-type.d.ts +17 -0
- package/lib/content-type/is-html-content-type.js +19 -0
- package/lib/content-type/normalize-content-type.d.ts +14 -0
- package/lib/content-type/normalize-content-type.js +20 -0
- package/lib/create-adjunct-tables.d.ts +66 -0
- package/lib/create-adjunct-tables.js +510 -0
- package/lib/create-entity-tables.d.ts +191 -0
- package/lib/create-entity-tables.js +387 -0
- package/lib/create-progress-reporter.d.ts +30 -0
- package/lib/create-progress-reporter.js +38 -0
- package/lib/create-ref-tables.d.ts +35 -0
- package/lib/create-ref-tables.js +185 -0
- package/lib/create-template-tables.d.ts +28 -0
- package/lib/create-template-tables.js +92 -0
- package/lib/database.d.ts +549 -0
- package/lib/database.js +804 -0
- package/lib/db-ops/_shared/clear-write-ref-caches.d.ts +27 -0
- package/lib/db-ops/_shared/clear-write-ref-caches.js +35 -0
- package/lib/db-ops/_shared/compress-payload.d.ts +26 -0
- package/lib/db-ops/_shared/compress-payload.js +30 -0
- package/lib/db-ops/_shared/create-write-ref-caches.d.ts +17 -0
- package/lib/db-ops/_shared/create-write-ref-caches.js +27 -0
- package/lib/db-ops/_shared/decode-json-ref.d.ts +17 -0
- package/lib/db-ops/_shared/decode-json-ref.js +31 -0
- package/lib/db-ops/_shared/keyset-paginate-by-id.d.ts +49 -0
- package/lib/db-ops/_shared/keyset-paginate-by-id.js +69 -0
- package/lib/db-ops/_shared/load-response-headers-by-set-ids.d.ts +20 -0
- package/lib/db-ops/_shared/load-response-headers-by-set-ids.js +53 -0
- package/lib/db-ops/_shared/resolve-content-item-id.d.ts +62 -0
- package/lib/db-ops/_shared/resolve-content-item-id.js +143 -0
- package/lib/db-ops/_shared/resolve-url-or-blob.d.ts +23 -0
- package/lib/db-ops/_shared/resolve-url-or-blob.js +29 -0
- package/lib/db-ops/_shared/retry-setting.d.ts +16 -0
- package/lib/db-ops/_shared/retry-setting.js +18 -0
- package/lib/db-ops/_shared/safe-parse-json.d.ts +11 -0
- package/lib/db-ops/_shared/safe-parse-json.js +18 -0
- package/lib/db-ops/_shared/types.d.ts +94 -0
- package/lib/db-ops/_shared/types.js +1 -0
- package/lib/db-ops/_shared/upsert-blob-ref.d.ts +25 -0
- package/lib/db-ops/_shared/upsert-blob-ref.js +48 -0
- package/lib/db-ops/_shared/upsert-content-type-ref.d.ts +30 -0
- package/lib/db-ops/_shared/upsert-content-type-ref.js +45 -0
- package/lib/db-ops/_shared/upsert-json-ref.d.ts +22 -0
- package/lib/db-ops/_shared/upsert-json-ref.js +41 -0
- package/lib/db-ops/_shared/upsert-response-headers.d.ts +35 -0
- package/lib/db-ops/_shared/upsert-response-headers.js +49 -0
- package/lib/db-ops/_shared/upsert-url-ref.d.ts +39 -0
- package/lib/db-ops/_shared/upsert-url-ref.js +62 -0
- package/lib/db-ops/anchors/get-anchors-on-page.d.ts +10 -0
- package/lib/db-ops/anchors/get-anchors-on-page.js +21 -0
- package/lib/db-ops/config/get-base-url.d.ts +8 -0
- package/lib/db-ops/config/get-base-url.js +14 -0
- package/lib/db-ops/config/get-config.d.ts +10 -0
- package/lib/db-ops/config/get-config.js +29 -0
- package/lib/db-ops/config/get-created-cwd.d.ts +11 -0
- package/lib/db-ops/config/get-created-cwd.js +13 -0
- package/lib/db-ops/config/get-name.d.ts +8 -0
- package/lib/db-ops/config/get-name.js +14 -0
- package/lib/db-ops/config/info-column-allowlist.d.ts +7 -0
- package/lib/db-ops/config/info-column-allowlist.js +29 -0
- package/lib/db-ops/config/info-json-columns.d.ts +5 -0
- package/lib/db-ops/config/info-json-columns.js +11 -0
- package/lib/db-ops/config/set-config.d.ts +12 -0
- package/lib/db-ops/config/set-config.js +21 -0
- package/lib/db-ops/config/update-config.d.ts +17 -0
- package/lib/db-ops/config/update-config.js +36 -0
- package/lib/db-ops/console-logs/compute-console-log-hash.d.ts +19 -0
- package/lib/db-ops/console-logs/compute-console-log-hash.js +30 -0
- package/lib/db-ops/console-logs/replace-console-logs.d.ts +34 -0
- package/lib/db-ops/console-logs/replace-console-logs.js +123 -0
- package/lib/db-ops/console-logs/stringify-console-log-args.d.ts +16 -0
- package/lib/db-ops/console-logs/stringify-console-log-args.js +26 -0
- package/lib/db-ops/console-logs/types.d.ts +38 -0
- package/lib/db-ops/console-logs/types.js +1 -0
- package/lib/db-ops/console-logs/upsert-console-log-item.d.ts +29 -0
- package/lib/db-ops/console-logs/upsert-console-log-item.js +52 -0
- package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts +18 -0
- package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js +23 -0
- package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts +12 -0
- package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.js +15 -0
- package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts +14 -0
- package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.js +30 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.d.ts +57 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.js +87 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts +21 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js +27 -0
- package/lib/db-ops/errors/insert-crawl-error.d.ts +15 -0
- package/lib/db-ops/errors/insert-crawl-error.js +21 -0
- package/lib/db-ops/errors/insert-page-error.d.ts +21 -0
- package/lib/db-ops/errors/insert-page-error.js +30 -0
- package/lib/db-ops/errors/list-dns-burned-host-candidates.d.ts +33 -0
- package/lib/db-ops/errors/list-dns-burned-host-candidates.js +158 -0
- package/lib/db-ops/html/read-page-html-blob.d.ts +17 -0
- package/lib/db-ops/html/read-page-html-blob.js +20 -0
- package/lib/db-ops/lifecycle/checkpoint.d.ts +8 -0
- package/lib/db-ops/lifecycle/checkpoint.js +9 -0
- package/lib/db-ops/lifecycle/destroy.d.ts +6 -0
- package/lib/db-ops/lifecycle/destroy.js +7 -0
- package/lib/db-ops/lifecycle/init.d.ts +58 -0
- package/lib/db-ops/lifecycle/init.js +122 -0
- package/lib/db-ops/list-reconcile/record-list-reconcile-run.d.ts +22 -0
- package/lib/db-ops/list-reconcile/record-list-reconcile-run.js +41 -0
- package/lib/db-ops/meta/get-audios-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-audios-of-page.js +15 -0
- package/lib/db-ops/meta/get-buttons-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-buttons-of-page.js +15 -0
- package/lib/db-ops/meta/get-canvases-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-canvases-of-page.js +15 -0
- package/lib/db-ops/meta/get-custom-elements-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-custom-elements-of-page.js +15 -0
- package/lib/db-ops/meta/get-headings-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-headings-of-page.js +15 -0
- package/lib/db-ops/meta/get-iframes-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-iframes-of-page.js +15 -0
- package/lib/db-ops/meta/get-jsonld-of-page.d.ts +13 -0
- package/lib/db-ops/meta/get-jsonld-of-page.js +27 -0
- package/lib/db-ops/meta/get-main-content-images-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-main-content-images-of-page.js +15 -0
- package/lib/db-ops/meta/get-main-content-tables-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-main-content-tables-of-page.js +15 -0
- package/lib/db-ops/meta/get-page-technologies-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-page-technologies-of-page.js +15 -0
- package/lib/db-ops/meta/get-technology-signals-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-technology-signals-of-page.js +15 -0
- package/lib/db-ops/meta/get-videos-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-videos-of-page.js +15 -0
- package/lib/db-ops/outages/close-network-outage.d.ts +12 -0
- package/lib/db-ops/outages/close-network-outage.js +15 -0
- package/lib/db-ops/outages/close-stale-open-network-outages.d.ts +26 -0
- package/lib/db-ops/outages/close-stale-open-network-outages.js +42 -0
- package/lib/db-ops/outages/compute-outage-clamp-timestamp.d.ts +26 -0
- package/lib/db-ops/outages/compute-outage-clamp-timestamp.js +31 -0
- package/lib/db-ops/outages/insert-network-outage.d.ts +14 -0
- package/lib/db-ops/outages/insert-network-outage.js +29 -0
- package/lib/db-ops/outages/list-network-outages.d.ts +28 -0
- package/lib/db-ops/outages/list-network-outages.js +42 -0
- package/lib/db-ops/pages/order/set-url-order.d.ts +13 -0
- package/lib/db-ops/pages/order/set-url-order.js +38 -0
- package/lib/db-ops/pages/read/build-page-query.d.ts +18 -0
- package/lib/db-ops/pages/read/build-page-query.js +40 -0
- package/lib/db-ops/pages/read/get-crawling-state.d.ts +117 -0
- package/lib/db-ops/pages/read/get-crawling-state.js +161 -0
- package/lib/db-ops/pages/read/get-existing-page-urls.d.ts +15 -0
- package/lib/db-ops/pages/read/get-existing-page-urls.js +30 -0
- package/lib/db-ops/pages/read/get-page-count.d.ts +12 -0
- package/lib/db-ops/pages/read/get-page-count.js +21 -0
- package/lib/db-ops/pages/read/get-page-source-by-url.d.ts +24 -0
- package/lib/db-ops/pages/read/get-page-source-by-url.js +28 -0
- package/lib/db-ops/pages/read/get-pages-with-rels.d.ts +38 -0
- package/lib/db-ops/pages/read/get-pages-with-rels.js +107 -0
- package/lib/db-ops/pages/read/get-pages.d.ts +11 -0
- package/lib/db-ops/pages/read/get-pages.js +51 -0
- package/lib/db-ops/pages/read/get-scraped-html-page-count.d.ts +18 -0
- package/lib/db-ops/pages/read/get-scraped-html-page-count.js +25 -0
- package/lib/db-ops/pages/read/reconstruct-page-rows.d.ts +31 -0
- package/lib/db-ops/pages/read/reconstruct-page-rows.js +32 -0
- package/lib/db-ops/pages/reset/clear-page-derived-rows.d.ts +20 -0
- package/lib/db-ops/pages/reset/clear-page-derived-rows.js +37 -0
- package/lib/db-ops/pages/reset/repromote-external-pages.d.ts +30 -0
- package/lib/db-ops/pages/reset/repromote-external-pages.js +91 -0
- package/lib/db-ops/pages/reset/reset-failed-pages.d.ts +88 -0
- package/lib/db-ops/pages/reset/reset-failed-pages.js +216 -0
- package/lib/db-ops/pages/reset/reset-pages-by-urls.d.ts +80 -0
- package/lib/db-ops/pages/reset/reset-pages-by-urls.js +151 -0
- package/lib/db-ops/pages/reset/retryable-image-scan-codes.d.ts +24 -0
- package/lib/db-ops/pages/reset/retryable-image-scan-codes.js +24 -0
- package/lib/db-ops/pages/write/insert-audios.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-audios.js +25 -0
- package/lib/db-ops/pages/write/insert-buttons.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-buttons.js +29 -0
- package/lib/db-ops/pages/write/insert-canvases.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-canvases.js +26 -0
- package/lib/db-ops/pages/write/insert-custom-elements.d.ts +27 -0
- package/lib/db-ops/pages/write/insert-custom-elements.js +40 -0
- package/lib/db-ops/pages/write/insert-headings.d.ts +15 -0
- package/lib/db-ops/pages/write/insert-headings.js +27 -0
- package/lib/db-ops/pages/write/insert-iframes.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-iframes.js +28 -0
- package/lib/db-ops/pages/write/insert-inventory-content-items.d.ts +38 -0
- package/lib/db-ops/pages/write/insert-inventory-content-items.js +60 -0
- package/lib/db-ops/pages/write/insert-inventory-seeds.d.ts +36 -0
- package/lib/db-ops/pages/write/insert-inventory-seeds.js +48 -0
- package/lib/db-ops/pages/write/insert-inventory-skipped-pages.d.ts +42 -0
- package/lib/db-ops/pages/write/insert-inventory-skipped-pages.js +56 -0
- package/lib/db-ops/pages/write/insert-jsonld.d.ts +17 -0
- package/lib/db-ops/pages/write/insert-jsonld.js +49 -0
- package/lib/db-ops/pages/write/insert-main-content-images.d.ts +20 -0
- package/lib/db-ops/pages/write/insert-main-content-images.js +32 -0
- package/lib/db-ops/pages/write/insert-main-content-tables.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-main-content-tables.js +29 -0
- package/lib/db-ops/pages/write/insert-page.d.ts +56 -0
- package/lib/db-ops/pages/write/insert-page.js +259 -0
- package/lib/db-ops/pages/write/insert-technologies.d.ts +25 -0
- package/lib/db-ops/pages/write/insert-technologies.js +45 -0
- package/lib/db-ops/pages/write/insert-videos.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-videos.js +28 -0
- package/lib/db-ops/pages/write/link-redirect-sources.d.ts +42 -0
- package/lib/db-ops/pages/write/link-redirect-sources.js +101 -0
- package/lib/db-ops/pages/write/record-redirect.d.ts +35 -0
- package/lib/db-ops/pages/write/record-redirect.js +100 -0
- package/lib/db-ops/pages/write/set-skipped-page.d.ts +13 -0
- package/lib/db-ops/pages/write/set-skipped-page.js +24 -0
- package/lib/db-ops/pages/write/update-page.d.ts +40 -0
- package/lib/db-ops/pages/write/update-page.js +419 -0
- package/lib/db-ops/pages/write/write-page-html-blob.d.ts +19 -0
- package/lib/db-ops/pages/write/write-page-html-blob.js +41 -0
- package/lib/db-ops/referrers/get-redirects-for-pages.d.ts +9 -0
- package/lib/db-ops/referrers/get-redirects-for-pages.js +15 -0
- package/lib/db-ops/referrers/get-referrers-of-page.d.ts +17 -0
- package/lib/db-ops/referrers/get-referrers-of-page.js +32 -0
- package/lib/db-ops/referrers/get-referrers-of-resource.d.ts +8 -0
- package/lib/db-ops/referrers/get-referrers-of-resource.js +15 -0
- package/lib/db-ops/resources/build-resource-query.d.ts +25 -0
- package/lib/db-ops/resources/build-resource-query.js +29 -0
- package/lib/db-ops/resources/get-existing-resource-urls.d.ts +9 -0
- package/lib/db-ops/resources/get-existing-resource-urls.js +24 -0
- package/lib/db-ops/resources/get-resource-by-url.d.ts +13 -0
- package/lib/db-ops/resources/get-resource-by-url.js +22 -0
- package/lib/db-ops/resources/get-resource-url-list.d.ts +17 -0
- package/lib/db-ops/resources/get-resource-url-list.js +26 -0
- package/lib/db-ops/resources/get-resources.d.ts +8 -0
- package/lib/db-ops/resources/get-resources.js +11 -0
- package/lib/db-ops/resources/insert-inventory-resources.d.ts +24 -0
- package/lib/db-ops/resources/insert-inventory-resources.js +64 -0
- package/lib/db-ops/resources/insert-resource-referrers.d.ts +15 -0
- package/lib/db-ops/resources/insert-resource-referrers.js +54 -0
- package/lib/db-ops/resources/insert-resource.d.ts +34 -0
- package/lib/db-ops/resources/insert-resource.js +73 -0
- package/lib/db-ops/resources/reconstruct-resource-rows.d.ts +26 -0
- package/lib/db-ops/resources/reconstruct-resource-rows.js +30 -0
- package/lib/db-ops/templates/assign-template-labels.d.ts +60 -0
- package/lib/db-ops/templates/assign-template-labels.js +187 -0
- package/lib/db-ops/templates/replace-page-templates.d.ts +44 -0
- package/lib/db-ops/templates/replace-page-templates.js +174 -0
- package/lib/db-ops/templates/serialize-template-cluster-reason.d.ts +16 -0
- package/lib/db-ops/templates/serialize-template-cluster-reason.js +17 -0
- package/lib/db-ops/templates/types.d.ts +103 -0
- package/lib/db-ops/templates/types.js +1 -0
- package/lib/debug.d.ts +8 -0
- package/lib/debug.js +9 -0
- package/lib/decode-html-blob.d.ts +20 -0
- package/lib/decode-html-blob.js +23 -0
- package/lib/decode-page-html-blob-bytes.d.ts +25 -0
- package/lib/decode-page-html-blob-bytes.js +29 -0
- package/lib/decode-stored-blob-bytes.d.ts +25 -0
- package/lib/decode-stored-blob-bytes.js +47 -0
- package/lib/derive-lineage-from-parent.d.ts +37 -0
- package/lib/derive-lineage-from-parent.js +42 -0
- package/lib/drop-legacy-tables.d.ts +45 -0
- package/lib/drop-legacy-tables.js +56 -0
- package/lib/error-kind/classify-error-kind.d.ts +20 -0
- package/lib/error-kind/classify-error-kind.js +136 -0
- package/lib/error-kind/network-related-error-kinds.d.ts +23 -0
- package/lib/error-kind/network-related-error-kinds.js +28 -0
- package/lib/error-kind/permanent-error-kinds.d.ts +48 -0
- package/lib/error-kind/permanent-error-kinds.js +54 -0
- package/lib/error-kind/types.d.ts +54 -0
- package/lib/error-kind/types.js +1 -0
- package/lib/filesystem/append-text.d.ts +9 -0
- package/lib/filesystem/append-text.js +14 -0
- package/lib/filesystem/copy-dir-sync.d.ts +6 -0
- package/lib/filesystem/copy-dir-sync.js +9 -0
- package/lib/filesystem/copy-dir.d.ts +7 -0
- package/lib/filesystem/copy-dir.js +13 -0
- package/lib/filesystem/copy-file-with-progress.d.ts +20 -0
- package/lib/filesystem/copy-file-with-progress.js +34 -0
- package/lib/filesystem/exists.d.ts +6 -0
- package/lib/filesystem/exists.js +9 -0
- package/lib/filesystem/get-file-list.d.ts +8 -0
- package/lib/filesystem/get-file-list.js +12 -0
- package/lib/filesystem/is-dir.d.ts +6 -0
- package/lib/filesystem/is-dir.js +10 -0
- package/lib/filesystem/mkdir.d.ts +8 -0
- package/lib/filesystem/mkdir.js +15 -0
- package/lib/filesystem/output-binary.d.ts +12 -0
- package/lib/filesystem/output-binary.js +17 -0
- package/lib/filesystem/output-json.d.ts +9 -0
- package/lib/filesystem/output-json.js +14 -0
- package/lib/filesystem/output-text.d.ts +11 -0
- package/lib/filesystem/output-text.js +32 -0
- package/lib/filesystem/parse-pax-path.d.ts +18 -0
- package/lib/filesystem/parse-pax-path.js +47 -0
- package/lib/filesystem/parse-tar-size-field.d.ts +13 -0
- package/lib/filesystem/parse-tar-size-field.js +40 -0
- package/lib/filesystem/peek-tar-top-dir.d.ts +35 -0
- package/lib/filesystem/peek-tar-top-dir.js +215 -0
- package/lib/filesystem/read-json.d.ts +7 -0
- package/lib/filesystem/read-json.js +11 -0
- package/lib/filesystem/read-text.d.ts +6 -0
- package/lib/filesystem/read-text.js +10 -0
- package/lib/filesystem/readline.d.ts +11 -0
- package/lib/filesystem/readline.js +26 -0
- package/lib/filesystem/remove.d.ts +5 -0
- package/lib/filesystem/remove.js +10 -0
- package/lib/filesystem/rename.d.ts +16 -0
- package/lib/filesystem/rename.js +48 -0
- package/lib/filesystem/tar.d.ts +23 -0
- package/lib/filesystem/tar.js +78 -0
- package/lib/filesystem/untar.d.ts +33 -0
- package/lib/filesystem/untar.js +84 -0
- package/lib/get-failed-page-messages.d.ts +56 -0
- package/lib/get-failed-page-messages.js +146 -0
- package/lib/get-json.d.ts +10 -0
- package/lib/get-json.js +25 -0
- package/lib/html-snapshot/match-selector-html.d.ts +25 -0
- package/lib/html-snapshot/match-selector-html.js +28 -0
- package/lib/html-snapshot/match-selector-snapshots.d.ts +33 -0
- package/lib/html-snapshot/match-selector-snapshots.js +33 -0
- package/lib/html-snapshot/search-html-document.d.ts +29 -0
- package/lib/html-snapshot/search-html-document.js +27 -0
- package/lib/html-snapshot/search-html-snapshots.d.ts +36 -0
- package/lib/html-snapshot/search-html-snapshots.js +35 -0
- package/lib/html-snapshot/types.d.ts +111 -0
- package/lib/html-snapshot/types.js +7 -0
- package/lib/html-snapshot/validate-search-pattern.d.ts +15 -0
- package/lib/html-snapshot/validate-search-pattern.js +17 -0
- package/lib/init-schema.d.ts +48 -0
- package/lib/init-schema.js +124 -0
- package/lib/is-inventory-source.d.ts +21 -0
- package/lib/is-inventory-source.js +22 -0
- package/lib/is-within-outage-window.d.ts +49 -0
- package/lib/is-within-outage-window.js +33 -0
- package/lib/isolated-cluster/create-isolated-cluster-run.d.ts +24 -0
- package/lib/isolated-cluster/create-isolated-cluster-run.js +26 -0
- package/lib/libsql-dialect.d.ts +115 -0
- package/lib/libsql-dialect.js +116 -0
- package/lib/limited-page-ids.d.ts +9 -0
- package/lib/limited-page-ids.js +18 -0
- package/lib/meta/assert-compatible-version.d.ts +60 -0
- package/lib/meta/assert-compatible-version.js +93 -0
- package/lib/meta/classify-jsonld-type.d.ts +23 -0
- package/lib/meta/classify-jsonld-type.js +43 -0
- package/lib/meta/compare-semver.d.ts +23 -0
- package/lib/meta/compare-semver.js +51 -0
- package/lib/meta/compute-main-contents-denormalized.d.ts +41 -0
- package/lib/meta/compute-main-contents-denormalized.js +88 -0
- package/lib/meta/compute-page-denormalized.d.ts +20 -0
- package/lib/meta/compute-page-denormalized.js +35 -0
- package/lib/meta/derive-flat-from-meta.d.ts +34 -0
- package/lib/meta/derive-flat-from-meta.js +158 -0
- package/lib/meta/derive-meta-extras.d.ts +20 -0
- package/lib/meta/derive-meta-extras.js +23 -0
- package/lib/meta/extract-tags-for-archive.d.ts +17 -0
- package/lib/meta/extract-tags-for-archive.js +36 -0
- package/lib/meta/parse-main-content-class-list.d.ts +19 -0
- package/lib/meta/parse-main-content-class-list.js +34 -0
- package/lib/meta/summarize-jsonld.d.ts +17 -0
- package/lib/meta/summarize-jsonld.js +29 -0
- package/lib/meta/technologies/combine-technology-confidence.d.ts +46 -0
- package/lib/meta/technologies/combine-technology-confidence.js +93 -0
- package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.d.ts +45 -0
- package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.js +53 -0
- package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.d.ts +33 -0
- package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.js +46 -0
- package/lib/meta/technologies/extract-technologies-for-archive.d.ts +33 -0
- package/lib/meta/technologies/extract-technologies-for-archive.js +30 -0
- package/lib/meta/technologies/match-js-license-comment-signal.d.ts +21 -0
- package/lib/meta/technologies/match-js-license-comment-signal.js +23 -0
- package/lib/meta/technologies/match-structural-technology-signals.d.ts +26 -0
- package/lib/meta/technologies/match-structural-technology-signals.js +48 -0
- package/lib/meta/technologies/normalize-wappalyzer-entries.d.ts +25 -0
- package/lib/meta/technologies/normalize-wappalyzer-entries.js +35 -0
- package/lib/meta/technologies/technology-signal-definitions.d.ts +44 -0
- package/lib/meta/technologies/technology-signal-definitions.js +60 -0
- package/lib/meta/technologies/types.d.ts +70 -0
- package/lib/meta/technologies/types.js +13 -0
- package/lib/meta/types.d.ts +1490 -0
- package/lib/meta/types.js +65 -0
- package/lib/migrate-content-items-alias-of-id.d.ts +52 -0
- package/lib/migrate-content-items-alias-of-id.js +69 -0
- package/lib/migrate-content-items-dedupe-cap-event-id.d.ts +46 -0
- package/lib/migrate-content-items-dedupe-cap-event-id.js +62 -0
- package/lib/migrate-content-items-is-metadata-only.d.ts +44 -0
- package/lib/migrate-content-items-is-metadata-only.js +60 -0
- package/lib/migrate-entity-tables.d.ts +55 -0
- package/lib/migrate-entity-tables.js +66 -0
- package/lib/migrate-info-created-cwd.d.ts +17 -0
- package/lib/migrate-info-created-cwd.js +36 -0
- package/lib/migrate-info-main-content-selector.d.ts +16 -0
- package/lib/migrate-info-main-content-selector.js +35 -0
- package/lib/migrate-info-request-header-names.d.ts +16 -0
- package/lib/migrate-info-request-header-names.js +35 -0
- package/lib/migrate-info-roots.d.ts +20 -0
- package/lib/migrate-info-roots.js +52 -0
- package/lib/migrate-inventory-runs-to-list-reconcile-runs.d.ts +34 -0
- package/lib/migrate-inventory-runs-to-list-reconcile-runs.js +51 -0
- package/lib/migrate-list-reconcile-runs-exclude-skipped.d.ts +25 -0
- package/lib/migrate-list-reconcile-runs-exclude-skipped.js +44 -0
- package/lib/migrate-list-reconcile-runs-invalid-skipped.d.ts +16 -0
- package/lib/migrate-list-reconcile-runs-invalid-skipped.js +35 -0
- package/lib/migrate-main-contents-columns.d.ts +26 -0
- package/lib/migrate-main-contents-columns.js +61 -0
- package/lib/migrate-page-meta-body-hash.d.ts +44 -0
- package/lib/migrate-page-meta-body-hash.js +63 -0
- package/lib/migrate-page-meta-console-error-count.d.ts +27 -0
- package/lib/migrate-page-meta-console-error-count.js +45 -0
- package/lib/migrate-page-meta-custom-element-count.d.ts +33 -0
- package/lib/migrate-page-meta-custom-element-count.js +51 -0
- package/lib/migrate-page-meta-image-scan.d.ts +27 -0
- package/lib/migrate-page-meta-image-scan.js +52 -0
- package/lib/migrate-page-tags-to-page-technologies.d.ts +39 -0
- package/lib/migrate-page-tags-to-page-technologies.js +79 -0
- package/lib/migrate-ref-tables.d.ts +25 -0
- package/lib/migrate-ref-tables.js +38 -0
- package/lib/native-executor/check-native-executor-config.d.ts +17 -0
- package/lib/native-executor/check-native-executor-config.js +20 -0
- package/lib/page-meta-column-maps.d.ts +32 -0
- package/lib/page-meta-column-maps.js +43 -0
- package/lib/page.d.ts +546 -0
- package/lib/page.js +764 -0
- package/lib/peek-archive-lock.d.ts +40 -0
- package/lib/peek-archive-lock.js +62 -0
- package/lib/populate-entity-tables/collapse-anchor-rows.d.ts +41 -0
- package/lib/populate-entity-tables/collapse-anchor-rows.js +87 -0
- package/lib/populate-entity-tables/derive-dom-path.d.ts +35 -0
- package/lib/populate-entity-tables/derive-dom-path.js +72 -0
- package/lib/populate-entity-tables/is-blob-ref-value.d.ts +16 -0
- package/lib/populate-entity-tables/is-blob-ref-value.js +19 -0
- package/lib/populate-entity-tables/match-images-to-dom-paths.d.ts +66 -0
- package/lib/populate-entity-tables/match-images-to-dom-paths.js +96 -0
- package/lib/populate-entity-tables/populate-anchor-edges.d.ts +33 -0
- package/lib/populate-entity-tables/populate-anchor-edges.js +153 -0
- package/lib/populate-entity-tables/populate-content-items.d.ts +40 -0
- package/lib/populate-entity-tables/populate-content-items.js +141 -0
- package/lib/populate-entity-tables/populate-entities.d.ts +81 -0
- package/lib/populate-entity-tables/populate-entities.js +110 -0
- package/lib/populate-entity-tables/populate-image-items.d.ts +91 -0
- package/lib/populate-entity-tables/populate-image-items.js +226 -0
- package/lib/populate-entity-tables/populate-page-meta.d.ts +33 -0
- package/lib/populate-entity-tables/populate-page-meta.js +267 -0
- package/lib/populate-entity-tables/populate-resource-items.d.ts +22 -0
- package/lib/populate-entity-tables/populate-resource-items.js +114 -0
- package/lib/populate-entity-tables/populate-resource-ref-edges.d.ts +31 -0
- package/lib/populate-entity-tables/populate-resource-ref-edges.js +33 -0
- package/lib/populate-entity-tables/resolve-blob-refs.d.ts +31 -0
- package/lib/populate-entity-tables/resolve-blob-refs.js +100 -0
- package/lib/populate-entity-tables/resolve-content-type-refs.d.ts +22 -0
- package/lib/populate-entity-tables/resolve-content-type-refs.js +27 -0
- package/lib/populate-entity-tables/resolve-header-sets.d.ts +49 -0
- package/lib/populate-entity-tables/resolve-header-sets.js +122 -0
- package/lib/populate-entity-tables/resolve-json-refs.d.ts +25 -0
- package/lib/populate-entity-tables/resolve-json-refs.js +67 -0
- package/lib/populate-entity-tables/resolve-text-refs.d.ts +30 -0
- package/lib/populate-entity-tables/resolve-text-refs.js +61 -0
- package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.d.ts +21 -0
- package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.js +27 -0
- package/lib/populate-entity-tables/resolve-url-refs.d.ts +33 -0
- package/lib/populate-entity-tables/resolve-url-refs.js +60 -0
- package/lib/populate-entity-tables/test-utils/count-rows.d.ts +17 -0
- package/lib/populate-entity-tables/test-utils/count-rows.js +20 -0
- package/lib/populate-entity-tables/test-utils/seed-content-items.d.ts +25 -0
- package/lib/populate-entity-tables/test-utils/seed-content-items.js +42 -0
- package/lib/populate-entity-tables/test-utils/setup-entities-db.d.ts +30 -0
- package/lib/populate-entity-tables/test-utils/setup-entities-db.js +175 -0
- package/lib/populate-entity-tables/types.d.ts +157 -0
- package/lib/populate-entity-tables/types.js +12 -0
- package/lib/populate-entity-tables/upsert-text-refs.d.ts +38 -0
- package/lib/populate-entity-tables/upsert-text-refs.js +78 -0
- package/lib/populate-ref-tables/classify-content-type.d.ts +16 -0
- package/lib/populate-ref-tables/classify-content-type.js +52 -0
- package/lib/populate-ref-tables/compute-content-hash.d.ts +22 -0
- package/lib/populate-ref-tables/compute-content-hash.js +26 -0
- package/lib/populate-ref-tables/compute-header-flags.d.ts +16 -0
- package/lib/populate-ref-tables/compute-header-flags.js +70 -0
- package/lib/populate-ref-tables/content-type-rules.d.ts +38 -0
- package/lib/populate-ref-tables/content-type-rules.js +133 -0
- package/lib/populate-ref-tables/create-header-table-caches.d.ts +25 -0
- package/lib/populate-ref-tables/create-header-table-caches.js +49 -0
- package/lib/populate-ref-tables/data-uri-url-refs-limit.d.ts +15 -0
- package/lib/populate-ref-tables/data-uri-url-refs-limit.js +15 -0
- package/lib/populate-ref-tables/decode-data-uri.d.ts +21 -0
- package/lib/populate-ref-tables/decode-data-uri.js +126 -0
- package/lib/populate-ref-tables/decompose-header-set.d.ts +29 -0
- package/lib/populate-ref-tables/decompose-header-set.js +157 -0
- package/lib/populate-ref-tables/decompose-url.d.ts +25 -0
- package/lib/populate-ref-tables/decompose-url.js +70 -0
- package/lib/populate-ref-tables/header-stability.d.ts +19 -0
- package/lib/populate-ref-tables/header-stability.js +22 -0
- package/lib/populate-ref-tables/header-value-cache-key.d.ts +17 -0
- package/lib/populate-ref-tables/header-value-cache-key.js +19 -0
- package/lib/populate-ref-tables/normalize-mime.d.ts +24 -0
- package/lib/populate-ref-tables/normalize-mime.js +36 -0
- package/lib/populate-ref-tables/populate-blob-refs.d.ts +38 -0
- package/lib/populate-ref-tables/populate-blob-refs.js +134 -0
- package/lib/populate-ref-tables/populate-content-type-refs.d.ts +27 -0
- package/lib/populate-ref-tables/populate-content-type-refs.js +70 -0
- package/lib/populate-ref-tables/populate-header-tables.d.ts +35 -0
- package/lib/populate-ref-tables/populate-header-tables.js +80 -0
- package/lib/populate-ref-tables/populate-json-refs.d.ts +29 -0
- package/lib/populate-ref-tables/populate-json-refs.js +101 -0
- package/lib/populate-ref-tables/populate-refs.d.ts +51 -0
- package/lib/populate-ref-tables/populate-refs.js +62 -0
- package/lib/populate-ref-tables/populate-text-refs.d.ts +32 -0
- package/lib/populate-ref-tables/populate-text-refs.js +133 -0
- package/lib/populate-ref-tables/populate-url-refs.d.ts +28 -0
- package/lib/populate-ref-tables/populate-url-refs.js +148 -0
- package/lib/populate-ref-tables/test-utils/count-rows.d.ts +15 -0
- package/lib/populate-ref-tables/test-utils/count-rows.js +17 -0
- package/lib/populate-ref-tables/types.d.ts +197 -0
- package/lib/populate-ref-tables/types.js +7 -0
- package/lib/populate-ref-tables/upsert-one-header-set.d.ts +34 -0
- package/lib/populate-ref-tables/upsert-one-header-set.js +208 -0
- package/lib/populate-ref-tables/volatile-header-names.d.ts +20 -0
- package/lib/populate-ref-tables/volatile-header-names.js +33 -0
- package/lib/redirect-table.d.ts +9 -0
- package/lib/redirect-table.js +28 -0
- package/lib/release-finished-statements.d.ts +42 -0
- package/lib/release-finished-statements.js +45 -0
- package/lib/resolve-redirect-chain.d.ts +33 -0
- package/lib/resolve-redirect-chain.js +27 -0
- package/lib/resource.d.ts +46 -0
- package/lib/resource.js +62 -0
- package/lib/retarget-legacy-fk-tables.d.ts +61 -0
- package/lib/retarget-legacy-fk-tables.js +137 -0
- package/lib/safe-path.d.ts +9 -0
- package/lib/safe-path.js +17 -0
- package/lib/scope/build-scope-map.d.ts +21 -0
- package/lib/scope/build-scope-map.js +32 -0
- package/lib/scope/find-scope-entry.d.ts +25 -0
- package/lib/scope/find-scope-entry.js +45 -0
- package/lib/template-classification/classify-archive-page-templates.d.ts +42 -0
- package/lib/template-classification/classify-archive-page-templates.js +70 -0
- package/lib/template-classification/classify-page-templates.d.ts +33 -0
- package/lib/template-classification/classify-page-templates.js +60 -0
- package/lib/template-classification/collect-page-ids-with-html.d.ts +24 -0
- package/lib/template-classification/collect-page-ids-with-html.js +30 -0
- package/lib/template-classification/collect-page-stylesheet-urls.d.ts +32 -0
- package/lib/template-classification/collect-page-stylesheet-urls.js +130 -0
- package/lib/template-classification/create-content-root-hint.d.ts +41 -0
- package/lib/template-classification/create-content-root-hint.js +51 -0
- package/lib/template-classification/create-page-cluster-factory.d.ts +36 -0
- package/lib/template-classification/create-page-cluster-factory.js +52 -0
- package/lib/template-classification/load-page-cluster-candidates.d.ts +32 -0
- package/lib/template-classification/load-page-cluster-candidates.js +93 -0
- package/lib/template-classification/page-cluster/resolve-page-cluster-keys.d.ts +34 -0
- package/lib/template-classification/page-cluster/resolve-page-cluster-keys.js +236 -0
- package/lib/template-classification/page-cluster/types.d.ts +123 -0
- package/lib/template-classification/page-cluster/types.js +1 -0
- package/lib/template-classification/types.d.ts +234 -0
- package/lib/template-classification/types.js +1 -0
- package/lib/test-utils/build-full-schema-test-db.d.ts +34 -0
- package/lib/test-utils/build-full-schema-test-db.js +74 -0
- package/lib/test-utils/create-legacy-analysis-tables.d.ts +13 -0
- package/lib/test-utils/create-legacy-analysis-tables.js +37 -0
- package/lib/test-utils/fk-parent-tables.d.ts +15 -0
- package/lib/test-utils/fk-parent-tables.js +19 -0
- package/lib/test-utils/seed-content-item.d.ts +35 -0
- package/lib/test-utils/seed-content-item.js +42 -0
- package/lib/test-utils/setup-legacy-fk-db.d.ts +34 -0
- package/lib/test-utils/setup-legacy-fk-db.js +271 -0
- package/lib/transfer/attach-source-database.d.ts +31 -0
- package/lib/transfer/attach-source-database.js +40 -0
- package/lib/transfer/build-split-config.d.ts +30 -0
- package/lib/transfer/build-split-config.js +42 -0
- package/lib/transfer/clear-replaced-page-rows.d.ts +23 -0
- package/lib/transfer/clear-replaced-page-rows.js +61 -0
- package/lib/transfer/concat-archives.d.ts +55 -0
- package/lib/transfer/concat-archives.js +95 -0
- package/lib/transfer/concat-source-transfer-phases.d.ts +10 -0
- package/lib/transfer/concat-source-transfer-phases.js +17 -0
- package/lib/transfer/content-item-rank-sql.d.ts +21 -0
- package/lib/transfer/content-item-rank-sql.js +23 -0
- package/lib/transfer/copy-anchor-edges.d.ts +28 -0
- package/lib/transfer/copy-anchor-edges.js +42 -0
- package/lib/transfer/copy-dictionaries-for-concat.d.ts +39 -0
- package/lib/transfer/copy-dictionaries-for-concat.js +81 -0
- package/lib/transfer/copy-dictionaries-for-split.d.ts +29 -0
- package/lib/transfer/copy-dictionaries-for-split.js +224 -0
- package/lib/transfer/copy-header-set-children.d.ts +27 -0
- package/lib/transfer/copy-header-set-children.js +51 -0
- package/lib/transfer/copy-image-items.d.ts +21 -0
- package/lib/transfer/copy-image-items.js +62 -0
- package/lib/transfer/copy-inventory-source-lists.d.ts +30 -0
- package/lib/transfer/copy-inventory-source-lists.js +61 -0
- package/lib/transfer/copy-journal-tables.d.ts +30 -0
- package/lib/transfer/copy-journal-tables.js +56 -0
- package/lib/transfer/copy-page-console-logs.d.ts +16 -0
- package/lib/transfer/copy-page-console-logs.js +27 -0
- package/lib/transfer/copy-page-html-blobs.d.ts +31 -0
- package/lib/transfer/copy-page-html-blobs.js +43 -0
- package/lib/transfer/copy-page-meta.d.ts +27 -0
- package/lib/transfer/copy-page-meta.js +68 -0
- package/lib/transfer/copy-page-template-clusters.d.ts +31 -0
- package/lib/transfer/copy-page-template-clusters.js +56 -0
- package/lib/transfer/copy-page-template-labels.d.ts +34 -0
- package/lib/transfer/copy-page-template-labels.js +88 -0
- package/lib/transfer/copy-resource-items.d.ts +29 -0
- package/lib/transfer/copy-resource-items.js +89 -0
- package/lib/transfer/copy-resource-ref-edges.d.ts +20 -0
- package/lib/transfer/copy-resource-ref-edges.js +33 -0
- package/lib/transfer/copy-simple-page-scoped-tables.d.ts +15 -0
- package/lib/transfer/copy-simple-page-scoped-tables.js +34 -0
- package/lib/transfer/count-external-pages-in-scope.d.ts +26 -0
- package/lib/transfer/count-external-pages-in-scope.js +40 -0
- package/lib/transfer/create-transfer-temp-tables.d.ts +30 -0
- package/lib/transfer/create-transfer-temp-tables.js +59 -0
- package/lib/transfer/dictionary-copy-specs.d.ts +32 -0
- package/lib/transfer/dictionary-copy-specs.js +28 -0
- package/lib/transfer/drop-transfer-temp-tables.d.ts +9 -0
- package/lib/transfer/drop-transfer-temp-tables.js +18 -0
- package/lib/transfer/flatten-redirect-chains.d.ts +44 -0
- package/lib/transfer/flatten-redirect-chains.js +74 -0
- package/lib/transfer/insert-content-items.d.ts +35 -0
- package/lib/transfer/insert-content-items.js +116 -0
- package/lib/transfer/list-transfer-columns.d.ts +35 -0
- package/lib/transfer/list-transfer-columns.js +45 -0
- package/lib/transfer/mark-in-scope-content-items.d.ts +31 -0
- package/lib/transfer/mark-in-scope-content-items.js +54 -0
- package/lib/transfer/merge-archive-configs.d.ts +51 -0
- package/lib/transfer/merge-archive-configs.js +85 -0
- package/lib/transfer/plan-content-items-for-concat.d.ts +33 -0
- package/lib/transfer/plan-content-items-for-concat.js +51 -0
- package/lib/transfer/plan-content-items-for-split.d.ts +42 -0
- package/lib/transfer/plan-content-items-for-split.js +83 -0
- package/lib/transfer/plan-resource-items.d.ts +40 -0
- package/lib/transfer/plan-resource-items.js +69 -0
- package/lib/transfer/quote-transfer-identifier.d.ts +18 -0
- package/lib/transfer/quote-transfer-identifier.js +20 -0
- package/lib/transfer/reclassify-resource-externality.d.ts +37 -0
- package/lib/transfer/reclassify-resource-externality.js +66 -0
- package/lib/transfer/remap-redirect-dest-ids.d.ts +34 -0
- package/lib/transfer/remap-redirect-dest-ids.js +51 -0
- package/lib/transfer/replace-content-items.d.ts +39 -0
- package/lib/transfer/replace-content-items.js +83 -0
- package/lib/transfer/simple-page-scoped-tables.d.ts +25 -0
- package/lib/transfer/simple-page-scoped-tables.js +38 -0
- package/lib/transfer/split-archive.d.ts +55 -0
- package/lib/transfer/split-archive.js +98 -0
- package/lib/transfer/split-source-transfer-phases.d.ts +11 -0
- package/lib/transfer/split-source-transfer-phases.js +18 -0
- package/lib/transfer/transfer-action.d.ts +34 -0
- package/lib/transfer/transfer-action.js +32 -0
- package/lib/transfer/transfer-archive-rows.d.ts +65 -0
- package/lib/transfer/transfer-archive-rows.js +156 -0
- package/lib/transfer/transfer-dictionary-tables.d.ts +12 -0
- package/lib/transfer/transfer-dictionary-tables.js +22 -0
- package/lib/transfer/transfer-post-phases.d.ts +10 -0
- package/lib/transfer/transfer-post-phases.js +13 -0
- package/lib/transfer/transfer-source-alias.d.ts +9 -0
- package/lib/transfer/transfer-source-alias.js +9 -0
- package/lib/transfer/types.d.ts +111 -0
- package/lib/transfer/types.js +35 -0
- package/lib/types.d.ts +772 -0
- package/lib/types.js +1 -0
- package/lib/url-alias/compute-tier-a-alias-key.d.ts +44 -0
- package/lib/url-alias/compute-tier-a-alias-key.js +47 -0
- package/lib/url-alias/compute-tier-b-alias-key.d.ts +23 -0
- package/lib/url-alias/compute-tier-b-alias-key.js +26 -0
- package/lib/url-alias/group-alias-candidates.d.ts +27 -0
- package/lib/url-alias/group-alias-candidates.js +29 -0
- package/lib/url-alias/types.d.ts +12 -0
- package/lib/url-alias/types.js +1 -0
- package/lib/url-pattern/compute-shape-key.d.ts +58 -0
- package/lib/url-pattern/compute-shape-key.js +129 -0
- package/lib/url-pattern/decompose-url.d.ts +33 -0
- package/lib/url-pattern/decompose-url.js +79 -0
- package/lib/url-pattern/reconstruct-url.d.ts +15 -0
- package/lib/url-pattern/reconstruct-url.js +33 -0
- package/lib/url-sort/compute-url-sort-ranks.d.ts +30 -0
- package/lib/url-sort/compute-url-sort-ranks.js +33 -0
- package/lib/utils/array/each-splitted.d.ts +10 -0
- package/lib/utils/array/each-splitted.js +14 -0
- package/lib/utils/debug.d.ts +8 -0
- package/lib/utils/debug.js +8 -0
- package/lib/utils/error/emit-error-with-retry.d.ts +40 -0
- package/lib/utils/error/emit-error-with-retry.js +44 -0
- package/lib/utils/error/emit-error.d.ts +39 -0
- package/lib/utils/error/emit-error.js +41 -0
- package/lib/utils/object/parse-response-headers.d.ts +12 -0
- package/lib/utils/object/parse-response-headers.js +26 -0
- package/lib/utils/object/redact-request-headers.d.ts +15 -0
- package/lib/utils/object/redact-request-headers.js +27 -0
- package/lib/utils/types/types.d.ts +383 -0
- package/lib/utils/types/types.js +1 -0
- package/lib/verify-migration/capture-rejection.d.ts +24 -0
- package/lib/verify-migration/capture-rejection.js +31 -0
- package/lib/verify-migration/check-anchor-edges-count.d.ts +34 -0
- package/lib/verify-migration/check-anchor-edges-count.js +72 -0
- package/lib/verify-migration/check-anchor-edges-sum.d.ts +13 -0
- package/lib/verify-migration/check-anchor-edges-sum.js +27 -0
- package/lib/verify-migration/check-content-items-count.d.ts +16 -0
- package/lib/verify-migration/check-content-items-count.js +30 -0
- package/lib/verify-migration/check-content-type-preservation.d.ts +22 -0
- package/lib/verify-migration/check-content-type-preservation.js +40 -0
- package/lib/verify-migration/check-foreign-key-integrity.d.ts +31 -0
- package/lib/verify-migration/check-foreign-key-integrity.js +47 -0
- package/lib/verify-migration/check-image-items-count.d.ts +12 -0
- package/lib/verify-migration/check-image-items-count.js +26 -0
- package/lib/verify-migration/check-page-meta-count.d.ts +15 -0
- package/lib/verify-migration/check-page-meta-count.js +31 -0
- package/lib/verify-migration/check-reader-parity.d.ts +23 -0
- package/lib/verify-migration/check-reader-parity.js +182 -0
- package/lib/verify-migration/check-resource-items-count.d.ts +17 -0
- package/lib/verify-migration/check-resource-items-count.js +33 -0
- package/lib/verify-migration/check-url-round-trip.d.ts +43 -0
- package/lib/verify-migration/check-url-round-trip.js +112 -0
- package/lib/verify-migration/types.d.ts +70 -0
- package/lib/verify-migration/types.js +63 -0
- package/lib/verify-migration/verify-migration.d.ts +41 -0
- package/lib/verify-migration/verify-migration.js +120 -0
- package/lib/with-bulk-write-pragmas.d.ts +51 -0
- package/lib/with-bulk-write-pragmas.js +132 -0
- package/lib/write-queue.d.ts +32 -0
- package/lib/write-queue.js +55 -0
- package/package.json +360 -5
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
import { raceWithTimeout } from '@d-zero/shared/race-with-timeout';
|
|
2
|
+
import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
|
|
3
|
+
import { log } from './debug.js';
|
|
4
|
+
import { readJSON } from './filesystem/read-json.js';
|
|
5
|
+
import { readText } from './filesystem/read-text.js';
|
|
6
|
+
import Page from './page.js';
|
|
7
|
+
import Resource from './resource.js';
|
|
8
|
+
import { safePath } from './safe-path.js';
|
|
9
|
+
/**
|
|
10
|
+
* Provides read-only access to an archive's database and stored data files.
|
|
11
|
+
*
|
|
12
|
+
* This class is the base for the `Archive` class and is also returned
|
|
13
|
+
* by `Archive.connect` for read-only access to an existing archive.
|
|
14
|
+
* It supports querying pages, anchors, referrers, resources, and custom data.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* Default timeout for {@link ArchiveAccessor.close}'s `db.destroy()` step.
|
|
18
|
+
*
|
|
19
|
+
* `knex.destroy()` will otherwise wait the full `acquireTimeoutMillis`
|
|
20
|
+
* (10 minutes in this repo) for in-flight queries to drain. For a viewer
|
|
21
|
+
* shut down by Ctrl-C while the live crawler holds a long write lock that
|
|
22
|
+
* is an unacceptable user experience, so we bound the wait and treat the
|
|
23
|
+
* accessor as closed after the timeout regardless.
|
|
24
|
+
*/
|
|
25
|
+
const DEFAULT_CLOSE_TIMEOUT_MS = 5000;
|
|
26
|
+
export class ArchiveAccessor extends EventEmitter {
|
|
27
|
+
/**
|
|
28
|
+
* Promise tracking an in-progress (or completed) close. `null` means the
|
|
29
|
+
* accessor is open and idle; a settled promise means we are closed (the
|
|
30
|
+
* accessor stays "closed" even if `db.destroy()` rejected, because there
|
|
31
|
+
* is nothing safe to retry — see {@link ArchiveAccessor.close}).
|
|
32
|
+
*/
|
|
33
|
+
#closeOnce = null;
|
|
34
|
+
/** The SQLite database instance for querying archived data. */
|
|
35
|
+
#db;
|
|
36
|
+
/**
|
|
37
|
+
* Whether this accessor was opened in read-only mode. With HTML stored
|
|
38
|
+
* as a SQLite BLOB, this no longer toggles any code path — the SELECT
|
|
39
|
+
* is identical for writer- and reader-mode accessors. Kept on the
|
|
40
|
+
* accessor so callers like the viewer can still surface "this archive
|
|
41
|
+
* is being read read-only" UI hints without re-deriving it.
|
|
42
|
+
*/
|
|
43
|
+
#readOnly;
|
|
44
|
+
/** Absolute path to the temporary working directory containing the database and files. */
|
|
45
|
+
#tmpDir;
|
|
46
|
+
/**
|
|
47
|
+
* Whether this accessor was opened in read-only mode (no filesystem
|
|
48
|
+
* mutation on `tmpDir`).
|
|
49
|
+
*/
|
|
50
|
+
get readOnly() {
|
|
51
|
+
return this.#readOnly;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* The absolute path to the temporary working directory used by this accessor.
|
|
55
|
+
*/
|
|
56
|
+
get tmpDir() {
|
|
57
|
+
return this.#tmpDir;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Creates a new ArchiveAccessor instance.
|
|
61
|
+
* @param tmpDir - The path to the temporary directory containing the archive data.
|
|
62
|
+
* @param db - The Database instance for querying the SQLite database.
|
|
63
|
+
* @param options - Construction options.
|
|
64
|
+
* @param options.readOnly - When `true`, helpers must not mutate the
|
|
65
|
+
* filesystem under `tmpDir` (used for live-crawl / stub-mode opens
|
|
66
|
+
* where any write would race the crawler).
|
|
67
|
+
*/
|
|
68
|
+
constructor(tmpDir, db, options = {}) {
|
|
69
|
+
super();
|
|
70
|
+
this.#tmpDir = tmpDir;
|
|
71
|
+
this.#db = db;
|
|
72
|
+
this.#readOnly = options.readOnly ?? false;
|
|
73
|
+
this.#db.on('error', (e) => {
|
|
74
|
+
void this.emit('error', e);
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Enables `await using accessor = ...`. Delegates to {@link close} with
|
|
79
|
+
* the default timeout — callers that need a non-default `timeoutMs`
|
|
80
|
+
* must call `close` explicitly instead of relying on disposal.
|
|
81
|
+
*
|
|
82
|
+
* Dispatches through the instance's own `close`, so an `Archive`
|
|
83
|
+
* (which overrides `close`) gets its full teardown here too — this
|
|
84
|
+
* method does not need to be re-implemented on subclasses.
|
|
85
|
+
*/
|
|
86
|
+
async [Symbol.asyncDispose]() {
|
|
87
|
+
await this.close();
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Closes the underlying database connection.
|
|
91
|
+
*
|
|
92
|
+
* This is the **read-only** close path: it releases the SQLite handle and
|
|
93
|
+
* does nothing else. The temporary working directory is left untouched and
|
|
94
|
+
* no `.nitpicker` archive is produced. This makes it safe to call from
|
|
95
|
+
* read-only consumers (e.g. the viewer attached to an in-progress crawl's
|
|
96
|
+
* tmpDir), where touching the filesystem would race with — or destroy —
|
|
97
|
+
* the live crawler's working state.
|
|
98
|
+
*
|
|
99
|
+
* Subclasses that own the archive's lifecycle (notably `Archive`)
|
|
100
|
+
* override this to add write/cleanup steps.
|
|
101
|
+
*
|
|
102
|
+
* **Idempotent and concurrent-safe**: the first invocation captures the
|
|
103
|
+
* close promise; later invocations (from the same caller, a shutdown
|
|
104
|
+
* signal handler, or a parallel manager teardown) await the same
|
|
105
|
+
* promise and resolve together. If `db.destroy()` rejects, the
|
|
106
|
+
* rejection propagates to *all* awaiters and the accessor stays
|
|
107
|
+
* latched closed — a hung knex pool is not safe to "retry close".
|
|
108
|
+
*
|
|
109
|
+
* **Bounded**: when the optional `timeoutMs` (default {@link
|
|
110
|
+
* DEFAULT_CLOSE_TIMEOUT_MS}) elapses before `db.destroy()` settles, the
|
|
111
|
+
* call resolves with a warning. This prevents a viewer shutdown from
|
|
112
|
+
* being held for the underlying pool's 10-minute `acquireTimeoutMillis`
|
|
113
|
+
* when the live crawler holds the SQLite write lock.
|
|
114
|
+
* @param options - Close options.
|
|
115
|
+
* @param options.timeoutMs - Milliseconds to wait for `db.destroy()`
|
|
116
|
+
* before giving up. Use `Infinity` to wait indefinitely (only
|
|
117
|
+
* advisable in tests and batch jobs that own the DB exclusively).
|
|
118
|
+
*/
|
|
119
|
+
async close(options = {}) {
|
|
120
|
+
if (this.#closeOnce) {
|
|
121
|
+
return this.#closeOnce;
|
|
122
|
+
}
|
|
123
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_CLOSE_TIMEOUT_MS;
|
|
124
|
+
this.#closeOnce = this.#runClose(timeoutMs);
|
|
125
|
+
return this.#closeOnce;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Retrieves anchor (link) data for a specific page by its database ID.
|
|
129
|
+
* @param pageId - The database ID of the page whose anchors to retrieve.
|
|
130
|
+
* @returns An array of anchor records found on the page.
|
|
131
|
+
*/
|
|
132
|
+
async getAnchorsOnPage(pageId) {
|
|
133
|
+
const refs = await this.#db.getAnchorsOnPage(pageId);
|
|
134
|
+
return refs;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Retrieves the audios within the given page's detected main content
|
|
138
|
+
* region, from `page_main_content_audios`.
|
|
139
|
+
* @param pageId - The database id of the page.
|
|
140
|
+
* @returns Ordered audio rows.
|
|
141
|
+
*/
|
|
142
|
+
async getAudiosOfPage(pageId) {
|
|
143
|
+
return this.#db.getAudiosOfPage(pageId);
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Retrieves the button-like elements within the given page's detected
|
|
147
|
+
* main content region, from `page_main_content_buttons`.
|
|
148
|
+
* @param pageId - The database id of the page.
|
|
149
|
+
* @returns Ordered button rows.
|
|
150
|
+
*/
|
|
151
|
+
async getButtonsOfPage(pageId) {
|
|
152
|
+
return this.#db.getButtonsOfPage(pageId);
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Retrieves the canvases within the given page's detected main content
|
|
156
|
+
* region, from `page_main_content_canvases`.
|
|
157
|
+
* @param pageId - The database id of the page.
|
|
158
|
+
* @returns Ordered canvas rows.
|
|
159
|
+
*/
|
|
160
|
+
async getCanvasesOfPage(pageId) {
|
|
161
|
+
return this.#db.getCanvasesOfPage(pageId);
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Retrieves the crawl configuration stored in the archive database.
|
|
165
|
+
* @returns The parsed {@link Config} object.
|
|
166
|
+
*/
|
|
167
|
+
async getConfig() {
|
|
168
|
+
return this.#db.getConfig();
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Retrieves the Web Components (custom elements) within the given page's
|
|
172
|
+
* detected main content region, from `page_main_content_custom_elements`.
|
|
173
|
+
* @param pageId - The database id of the page.
|
|
174
|
+
* @returns Ordered custom-element rows.
|
|
175
|
+
*/
|
|
176
|
+
async getCustomElementsOfPage(pageId) {
|
|
177
|
+
return this.#db.getCustomElementsOfPage(pageId);
|
|
178
|
+
}
|
|
179
|
+
async getData(name, format = 'json') {
|
|
180
|
+
const filePath = safePath(this.#tmpDir, `${name}.${format}`);
|
|
181
|
+
if (format === 'json') {
|
|
182
|
+
return await readJSON(filePath);
|
|
183
|
+
}
|
|
184
|
+
return await readText(filePath);
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Retrieves the headings within the given page's detected main content
|
|
188
|
+
* region, from `page_main_content_headings`.
|
|
189
|
+
* @param pageId - The database id of the page.
|
|
190
|
+
* @returns Ordered heading rows.
|
|
191
|
+
*/
|
|
192
|
+
async getHeadingsOfPage(pageId) {
|
|
193
|
+
return this.#db.getHeadingsOfPage(pageId);
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Reads the HTML snapshot of a page from the archive.
|
|
197
|
+
*
|
|
198
|
+
* HTML is stored as zstd-compressed BLOBs in `page_html_blobs` (keyed by
|
|
199
|
+
* SHA-256 of the raw bytes) with `page_html_ref` linking `page_id → hash`.
|
|
200
|
+
* The read is a straight join + decompress; writer- and read-only (stub)
|
|
201
|
+
* accessors take the same code path because nothing here touches the
|
|
202
|
+
* filesystem.
|
|
203
|
+
*
|
|
204
|
+
* Returns `null` when the page row exists but has no stored body — for
|
|
205
|
+
* example a redirect source, a non-HTML resource (PDF), a page that
|
|
206
|
+
* failed to render, or an external page (whose row is metadata-only).
|
|
207
|
+
* Distinguishing "no body stored" from "empty body" is preserved: an
|
|
208
|
+
* empty HTML string returns `''`, not `null`.
|
|
209
|
+
*
|
|
210
|
+
* Throws if the page's referenced blob is missing, its codec marker is
|
|
211
|
+
* unrecognised, its zstd data is corrupt, or it decompresses past
|
|
212
|
+
* `MAX_DECODED_SNAPSHOT_BYTES` — each indicates an archive that was
|
|
213
|
+
* truncated, damaged or written by a future tool, none of which we
|
|
214
|
+
* silently paper over. A decode failure names the page and the blob hash
|
|
215
|
+
* (`HTML snapshot of page <id> (page_html_blobs hash <hex>) could not be decoded: …`).
|
|
216
|
+
* @param pageId - The database id of the page to read.
|
|
217
|
+
* @returns The HTML content as a UTF-8 string, or `null` when no body
|
|
218
|
+
* is stored for `pageId`.
|
|
219
|
+
* @example
|
|
220
|
+
* const html = await accessor.getHtmlOfPage(pageId);
|
|
221
|
+
* if (html === null) {
|
|
222
|
+
* // page has no stored body — redirect source / non-HTML / failed render
|
|
223
|
+
* } else {
|
|
224
|
+
* processHtml(html);
|
|
225
|
+
* }
|
|
226
|
+
*/
|
|
227
|
+
async getHtmlOfPage(pageId) {
|
|
228
|
+
const html = await this.#db.getHtmlOfPageById(pageId);
|
|
229
|
+
if (html === null) {
|
|
230
|
+
log('No HTML body stored for page id=%d', pageId);
|
|
231
|
+
}
|
|
232
|
+
return html;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Retrieves the iframes within the given page's detected main content
|
|
236
|
+
* region, from `page_main_content_iframes`.
|
|
237
|
+
* @param pageId - The database id of the page.
|
|
238
|
+
* @returns Ordered iframe rows.
|
|
239
|
+
*/
|
|
240
|
+
async getIframesOfPage(pageId) {
|
|
241
|
+
return this.#db.getIframesOfPage(pageId);
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Retrieves the JSON-LD / SpeculationRules entries for the given page,
|
|
245
|
+
* parsed back from the `page_jsonld` table.
|
|
246
|
+
* @param pageId - The database id of the page.
|
|
247
|
+
* @returns Ordered entries with `kind`, `type`, `raw`, `parsed`, `parseError`.
|
|
248
|
+
*/
|
|
249
|
+
async getJsonLdOfPage(pageId) {
|
|
250
|
+
return this.#db.getJsonLdOfPage(pageId);
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Returns the underlying Knex query builder instance for direct SQL access.
|
|
254
|
+
* Enables advanced queries (GROUP BY, HAVING, JOINs) at the database layer
|
|
255
|
+
* for performance-critical operations on large datasets.
|
|
256
|
+
* @returns The Knex instance connected to the SQLite database.
|
|
257
|
+
*/
|
|
258
|
+
getKnex() {
|
|
259
|
+
return this.#db.getKnex();
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Retrieves the images within the given page's detected main content
|
|
263
|
+
* region, from `page_main_content_images`.
|
|
264
|
+
* @param pageId - The database id of the page.
|
|
265
|
+
* @returns Ordered image rows.
|
|
266
|
+
*/
|
|
267
|
+
async getMainContentImagesOfPage(pageId) {
|
|
268
|
+
return this.#db.getMainContentImagesOfPage(pageId);
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Retrieves the tables within the given page's detected main content
|
|
272
|
+
* region, from `page_main_content_tables`.
|
|
273
|
+
* @param pageId - The database id of the page.
|
|
274
|
+
* @returns Ordered table rows.
|
|
275
|
+
*/
|
|
276
|
+
async getMainContentTablesOfPage(pageId) {
|
|
277
|
+
return this.#db.getMainContentTablesOfPage(pageId);
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Retrieves all pages from the archive, optionally filtered by type.
|
|
281
|
+
* Eagerly loads redirect relationships (`redirectFrom`) but does NOT load
|
|
282
|
+
* anchor or referrer relationships.
|
|
283
|
+
* Use {@link getPagesWithRefs} if you need those relationships.
|
|
284
|
+
* @param filter - An optional filter to narrow the results (e.g., `'internal-page'`, `'external-page'`).
|
|
285
|
+
* @returns An array of {@link Page} instances.
|
|
286
|
+
*/
|
|
287
|
+
async getPages(filter) {
|
|
288
|
+
const pages = await this.#db.getPages(filter);
|
|
289
|
+
if (pages.length === 0)
|
|
290
|
+
return [];
|
|
291
|
+
const pageIds = pages.map((p) => p.id);
|
|
292
|
+
const redirects = await this.#db.getRedirectsForPages(pageIds);
|
|
293
|
+
const redirectMap = new Map();
|
|
294
|
+
for (const redirect of redirects) {
|
|
295
|
+
const current = redirectMap.get(redirect.pageId);
|
|
296
|
+
if (current) {
|
|
297
|
+
current.push(redirect);
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
redirectMap.set(redirect.pageId, [redirect]);
|
|
301
|
+
}
|
|
302
|
+
return pages.map((page) => new Page(this, page, redirectMap.get(page.id) || []));
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* Retrieves pages with their related data (redirects, anchors, referrers) in batches.
|
|
306
|
+
* Processes pages in chunks of `limit` size, calling the callback for each batch.
|
|
307
|
+
* @param limit - The maximum number of pages to load per batch.
|
|
308
|
+
* @param callback - A function called for each batch of pages with the current offset and total count.
|
|
309
|
+
* @param options - Optional URL parsing options and whether to include referrer relationships.
|
|
310
|
+
*/
|
|
311
|
+
async getPagesWithRefs(limit, callback, options) {
|
|
312
|
+
const max = await this.#getPageCount();
|
|
313
|
+
let times = 0;
|
|
314
|
+
while (true) {
|
|
315
|
+
const offset = times * limit;
|
|
316
|
+
log('%d times loop: %o', times, {
|
|
317
|
+
offset,
|
|
318
|
+
limit,
|
|
319
|
+
max,
|
|
320
|
+
});
|
|
321
|
+
const pages = await this.#getPagesWithRels(offset, limit, options);
|
|
322
|
+
if (pages.length === 0) {
|
|
323
|
+
break;
|
|
324
|
+
}
|
|
325
|
+
await callback(pages, offset, max);
|
|
326
|
+
times++;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Retrieves the confidence-combined technology roll-up for the given
|
|
331
|
+
* page, from the `page_technologies` table.
|
|
332
|
+
* @param pageId - The database id of the page.
|
|
333
|
+
* @returns Technology rows with category, version, confidence, signalCount.
|
|
334
|
+
*/
|
|
335
|
+
async getPageTechnologiesOfPage(pageId) {
|
|
336
|
+
return this.#db.getPageTechnologiesOfPage(pageId);
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Retrieves pages that link to the specified page (incoming links).
|
|
340
|
+
* @param pageId - The database ID of the target page.
|
|
341
|
+
* @returns An array of referrer records.
|
|
342
|
+
*/
|
|
343
|
+
async getReferrersOfPage(pageId) {
|
|
344
|
+
const refs = await this.#db.getReferrersOfPage(pageId);
|
|
345
|
+
return refs;
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Retrieves page URLs that reference the specified resource.
|
|
349
|
+
* @param pageId - The database ID of the resource.
|
|
350
|
+
* @returns An array of page URL strings that reference this resource.
|
|
351
|
+
*/
|
|
352
|
+
async getReferrersOfResource(pageId) {
|
|
353
|
+
const refs = await this.#db.getReferrersOfResource(pageId);
|
|
354
|
+
return refs;
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Retrieves all sub-resources (CSS, JS, images, etc.) stored in the archive.
|
|
358
|
+
* @returns An array of {@link Resource} instances.
|
|
359
|
+
*/
|
|
360
|
+
async getResources() {
|
|
361
|
+
const resources = await this.#db.getResources();
|
|
362
|
+
return resources.map((r) => new Resource(this, r));
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Retrieves a flat list of all resource URLs stored in the archive.
|
|
366
|
+
* @param onProgress - Forwarded to {@link Database.getResourceUrlList} —
|
|
367
|
+
* see that method's docs.
|
|
368
|
+
* @returns An array of resource URL strings.
|
|
369
|
+
*/
|
|
370
|
+
async getResourceUrlList(onProgress) {
|
|
371
|
+
return this.#db.getResourceUrlList(onProgress);
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* Retrieves the raw technology-detection signals for the given page,
|
|
375
|
+
* from the `technology_signals` table.
|
|
376
|
+
* @param pageId - The database id of the page.
|
|
377
|
+
* @returns Ordered signal rows with technology, signalType, evidence, weight.
|
|
378
|
+
*/
|
|
379
|
+
async getTechnologySignalsOfPage(pageId) {
|
|
380
|
+
return this.#db.getTechnologySignalsOfPage(pageId);
|
|
381
|
+
}
|
|
382
|
+
/**
|
|
383
|
+
* Retrieves the videos within the given page's detected main content
|
|
384
|
+
* region, from `page_main_content_videos`.
|
|
385
|
+
* @param pageId - The database id of the page.
|
|
386
|
+
* @returns Ordered video rows.
|
|
387
|
+
*/
|
|
388
|
+
async getVideosOfPage(pageId) {
|
|
389
|
+
return this.#db.getVideosOfPage(pageId);
|
|
390
|
+
}
|
|
391
|
+
/**
|
|
392
|
+
* Returns the total number of internal pages in the archive.
|
|
393
|
+
*/
|
|
394
|
+
async #getPageCount() {
|
|
395
|
+
return this.#db.getPageCount();
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Loads a batch of pages with their related data (redirects, anchors, referrers).
|
|
399
|
+
* When `withRefs` is false, loads only pages without relationships for better performance.
|
|
400
|
+
* @param offset - The number of pages to skip
|
|
401
|
+
* @param limit - The maximum number of pages to return
|
|
402
|
+
* @param options - URL parsing and referrer loading options
|
|
403
|
+
*/
|
|
404
|
+
async #getPagesWithRels(offset, limit, options) {
|
|
405
|
+
if (options?.withRefs === false) {
|
|
406
|
+
const pages = await this.#db.getPages('internal-page', offset, limit);
|
|
407
|
+
return pages.map((page) => new Page(this, page));
|
|
408
|
+
}
|
|
409
|
+
const { pages, redirects, anchors, referrers } = await this.#db.getPagesWithRels(offset, limit);
|
|
410
|
+
const redirectMap = new Map();
|
|
411
|
+
const anchorMap = new Map();
|
|
412
|
+
const refersMap = new Map();
|
|
413
|
+
log('Mapping redirects');
|
|
414
|
+
for (const redirect of redirects) {
|
|
415
|
+
const current = redirectMap.get(redirect.pageId);
|
|
416
|
+
if (current) {
|
|
417
|
+
current.push(redirect);
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
redirectMap.set(redirect.pageId, [redirect]);
|
|
421
|
+
}
|
|
422
|
+
log('Mapping anchors');
|
|
423
|
+
for (const anchor of anchors) {
|
|
424
|
+
const current = anchorMap.get(anchor.pageId);
|
|
425
|
+
if (current) {
|
|
426
|
+
current.push(anchor);
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
anchorMap.set(anchor.pageId, [anchor]);
|
|
430
|
+
}
|
|
431
|
+
log('Mapping referrers');
|
|
432
|
+
for (const referrer of referrers) {
|
|
433
|
+
const current = refersMap.get(referrer.pageId);
|
|
434
|
+
if (current) {
|
|
435
|
+
current.push(referrer);
|
|
436
|
+
continue;
|
|
437
|
+
}
|
|
438
|
+
refersMap.set(referrer.pageId, [referrer]);
|
|
439
|
+
}
|
|
440
|
+
log('Create Page Data');
|
|
441
|
+
const pPages = [];
|
|
442
|
+
for (const page of pages) {
|
|
443
|
+
const pRedirects = redirectMap.get(page.id) || [];
|
|
444
|
+
const pAnchors = anchorMap.get(page.id) || [];
|
|
445
|
+
const pRefers = refersMap.get(page.id) || [];
|
|
446
|
+
pPages.push(new Page(this, page, pRedirects, pAnchors, pRefers, options?.disableQueries));
|
|
447
|
+
}
|
|
448
|
+
log('Create Page Data: Done');
|
|
449
|
+
return pPages;
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* Actual close worker — invoked exactly once per accessor via
|
|
453
|
+
* {@link close}'s shared promise. Races `db.destroy()` against the
|
|
454
|
+
* caller-supplied timeout; on timeout we log and resolve so the
|
|
455
|
+
* consumer (typically a process shutting down) is not blocked, even
|
|
456
|
+
* though the underlying knex pool may still be draining in the
|
|
457
|
+
* background.
|
|
458
|
+
* @param timeoutMs - Maximum time to wait for `db.destroy()`.
|
|
459
|
+
*/
|
|
460
|
+
async #runClose(timeoutMs) {
|
|
461
|
+
if (!Number.isFinite(timeoutMs)) {
|
|
462
|
+
await this.#db.destroy();
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
// Track destroy() so we can attach an error-suppressing handler if we
|
|
466
|
+
// give up waiting — otherwise a late rejection becomes an unhandled
|
|
467
|
+
// promise rejection on the process.
|
|
468
|
+
const destroy = this.#db.destroy().then(() => 'done');
|
|
469
|
+
const { timeout } = await raceWithTimeout(() => destroy, timeoutMs);
|
|
470
|
+
if (timeout) {
|
|
471
|
+
log('ArchiveAccessor.close: db.destroy() did not settle within %dms — giving up', timeoutMs);
|
|
472
|
+
destroy.catch((error) => {
|
|
473
|
+
log('ArchiveAccessor.close: late db.destroy() rejection (post-timeout): %O', error);
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lock failure surfaced when an archive's tmpDir is already in use by another
|
|
3
|
+
* live process.
|
|
4
|
+
*
|
|
5
|
+
* The lock path and PID of the holder are exposed so callers (CLI / orchestrator)
|
|
6
|
+
* can render a precise error message and let operators recover manually.
|
|
7
|
+
*/
|
|
8
|
+
export declare class ArchiveLockError extends Error {
|
|
9
|
+
/** PID of the live process currently holding the lock, or `null` if unknown. */
|
|
10
|
+
readonly holderPid: number | null;
|
|
11
|
+
/** The lock directory path that could not be acquired. */
|
|
12
|
+
readonly lockPath: string;
|
|
13
|
+
/**
|
|
14
|
+
* @param lockPath - Absolute path of the lock directory.
|
|
15
|
+
* @param holderPid - PID stored in the lock's `pid.txt`, or `null` if unreadable.
|
|
16
|
+
*/
|
|
17
|
+
constructor(lockPath: string, holderPid: number | null);
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Acquire an advisory lock on the given tmpDir.
|
|
21
|
+
*
|
|
22
|
+
* Uses `fs.mkdir(lockPath, { recursive: false })` as the atomic primitive so
|
|
23
|
+
* that two concurrent processes targeting the same archive cannot both believe
|
|
24
|
+
* they own it. The lock is published as a sibling directory named
|
|
25
|
+
* `{tmpDir}.lock` containing a `pid.txt` file with the holder's PID.
|
|
26
|
+
*
|
|
27
|
+
* On `EEXIST` the function tries once to detect a stale lock: if the recorded
|
|
28
|
+
* PID is not alive any more (`process.kill(pid, 0)` throws `ESRCH`), the stale
|
|
29
|
+
* directory is removed and a single retry is issued. If acquisition still
|
|
30
|
+
* fails — or the holder is alive — an {@link ArchiveLockError} is thrown.
|
|
31
|
+
*
|
|
32
|
+
* The returned function releases the lock; it is idempotent so callers can put
|
|
33
|
+
* it in a `finally` block without worrying about double-release.
|
|
34
|
+
* @param tmpDir - Absolute path to the archive's temporary working directory.
|
|
35
|
+
* @returns A release function to be called when the work is done.
|
|
36
|
+
* @throws {ArchiveLockError} When the lock cannot be acquired even after a stale-lock retry.
|
|
37
|
+
* @example
|
|
38
|
+
* const releaseLock = await acquireArchiveLock(tmpDir);
|
|
39
|
+
* try {
|
|
40
|
+
* // ... exclusive work against tmpDir ...
|
|
41
|
+
* } finally {
|
|
42
|
+
* await releaseLock();
|
|
43
|
+
* }
|
|
44
|
+
*/
|
|
45
|
+
export declare function acquireArchiveLock(tmpDir: string): Promise<() => Promise<void>>;
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
/**
|
|
4
|
+
* Lock failure surfaced when an archive's tmpDir is already in use by another
|
|
5
|
+
* live process.
|
|
6
|
+
*
|
|
7
|
+
* The lock path and PID of the holder are exposed so callers (CLI / orchestrator)
|
|
8
|
+
* can render a precise error message and let operators recover manually.
|
|
9
|
+
*/
|
|
10
|
+
export class ArchiveLockError extends Error {
|
|
11
|
+
/** PID of the live process currently holding the lock, or `null` if unknown. */
|
|
12
|
+
holderPid;
|
|
13
|
+
/** The lock directory path that could not be acquired. */
|
|
14
|
+
lockPath;
|
|
15
|
+
/**
|
|
16
|
+
* @param lockPath - Absolute path of the lock directory.
|
|
17
|
+
* @param holderPid - PID stored in the lock's `pid.txt`, or `null` if unreadable.
|
|
18
|
+
*/
|
|
19
|
+
constructor(lockPath, holderPid) {
|
|
20
|
+
const suffix = holderPid === null ? '' : ` (PID ${holderPid})`;
|
|
21
|
+
super(`Archive is being used by another process${suffix}: ${lockPath}`);
|
|
22
|
+
this.name = 'ArchiveLockError';
|
|
23
|
+
this.lockPath = lockPath;
|
|
24
|
+
this.holderPid = holderPid;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Acquire an advisory lock on the given tmpDir.
|
|
29
|
+
*
|
|
30
|
+
* Uses `fs.mkdir(lockPath, { recursive: false })` as the atomic primitive so
|
|
31
|
+
* that two concurrent processes targeting the same archive cannot both believe
|
|
32
|
+
* they own it. The lock is published as a sibling directory named
|
|
33
|
+
* `{tmpDir}.lock` containing a `pid.txt` file with the holder's PID.
|
|
34
|
+
*
|
|
35
|
+
* On `EEXIST` the function tries once to detect a stale lock: if the recorded
|
|
36
|
+
* PID is not alive any more (`process.kill(pid, 0)` throws `ESRCH`), the stale
|
|
37
|
+
* directory is removed and a single retry is issued. If acquisition still
|
|
38
|
+
* fails — or the holder is alive — an {@link ArchiveLockError} is thrown.
|
|
39
|
+
*
|
|
40
|
+
* The returned function releases the lock; it is idempotent so callers can put
|
|
41
|
+
* it in a `finally` block without worrying about double-release.
|
|
42
|
+
* @param tmpDir - Absolute path to the archive's temporary working directory.
|
|
43
|
+
* @returns A release function to be called when the work is done.
|
|
44
|
+
* @throws {ArchiveLockError} When the lock cannot be acquired even after a stale-lock retry.
|
|
45
|
+
* @example
|
|
46
|
+
* const releaseLock = await acquireArchiveLock(tmpDir);
|
|
47
|
+
* try {
|
|
48
|
+
* // ... exclusive work against tmpDir ...
|
|
49
|
+
* } finally {
|
|
50
|
+
* await releaseLock();
|
|
51
|
+
* }
|
|
52
|
+
*/
|
|
53
|
+
export async function acquireArchiveLock(tmpDir) {
|
|
54
|
+
const lockPath = `${tmpDir}.lock`;
|
|
55
|
+
await tryAcquire(lockPath);
|
|
56
|
+
let released = false;
|
|
57
|
+
return async () => {
|
|
58
|
+
if (released) {
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
released = true;
|
|
62
|
+
await releaseLock(lockPath);
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Attempt to create the lock directory and write the holder PID, handling one
|
|
67
|
+
* round of stale-lock recovery.
|
|
68
|
+
* @param lockPath - The absolute lock directory path.
|
|
69
|
+
*/
|
|
70
|
+
async function tryAcquire(lockPath) {
|
|
71
|
+
try {
|
|
72
|
+
await fs.mkdir(lockPath, { recursive: false });
|
|
73
|
+
await fs.writeFile(path.join(lockPath, 'pid.txt'), String(process.pid), 'utf8');
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
if (!isEexist(error)) {
|
|
78
|
+
throw error;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
const holderPid = await readHolderPid(lockPath);
|
|
82
|
+
if (holderPid !== null && isProcessAlive(holderPid)) {
|
|
83
|
+
throw new ArchiveLockError(lockPath, holderPid);
|
|
84
|
+
}
|
|
85
|
+
// Stale lock — clean up and retry once. A concurrent acquirer may have
|
|
86
|
+
// already reclaimed the directory, so a second EEXIST is treated as a real
|
|
87
|
+
// collision and surfaced to the caller.
|
|
88
|
+
await fs.rm(lockPath, { recursive: true, force: true });
|
|
89
|
+
try {
|
|
90
|
+
await fs.mkdir(lockPath, { recursive: false });
|
|
91
|
+
await fs.writeFile(path.join(lockPath, 'pid.txt'), String(process.pid), 'utf8');
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
if (isEexist(error)) {
|
|
95
|
+
const pid = await readHolderPid(lockPath);
|
|
96
|
+
throw new ArchiveLockError(lockPath, pid);
|
|
97
|
+
}
|
|
98
|
+
throw error;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Remove the lock directory. `ENOENT` is swallowed so `finally`-style callers
|
|
103
|
+
* tolerate an externally-cleaned lock, but any other failure (permissions,
|
|
104
|
+
* disk full, etc.) is propagated so the caller can react.
|
|
105
|
+
* @param lockPath - The absolute lock directory path.
|
|
106
|
+
*/
|
|
107
|
+
async function releaseLock(lockPath) {
|
|
108
|
+
try {
|
|
109
|
+
await fs.rm(lockPath, { recursive: true, force: true });
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
if (error.code !== 'ENOENT') {
|
|
113
|
+
throw error;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Read the PID stored in the lock directory's `pid.txt`.
|
|
119
|
+
* @param lockPath - The absolute lock directory path.
|
|
120
|
+
* @returns The parsed PID, or `null` if the file is missing or malformed.
|
|
121
|
+
*/
|
|
122
|
+
async function readHolderPid(lockPath) {
|
|
123
|
+
try {
|
|
124
|
+
const raw = await fs.readFile(path.join(lockPath, 'pid.txt'), 'utf8');
|
|
125
|
+
const pid = Number.parseInt(raw.trim(), 10);
|
|
126
|
+
return Number.isFinite(pid) && pid > 0 ? pid : null;
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Check whether the given PID is still alive using a signal-0 probe.
|
|
134
|
+
* @param pid - The process id to probe.
|
|
135
|
+
* @returns `true` if the process exists (regardless of permission).
|
|
136
|
+
*/
|
|
137
|
+
function isProcessAlive(pid) {
|
|
138
|
+
try {
|
|
139
|
+
process.kill(pid, 0);
|
|
140
|
+
return true;
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
// EPERM means the process exists but is owned by another user — still alive.
|
|
144
|
+
return error.code === 'EPERM';
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Type-guard for an `EEXIST` filesystem error.
|
|
149
|
+
* @param error - The error value to inspect.
|
|
150
|
+
* @returns `true` when the error is a Node `EEXIST` from a filesystem call.
|
|
151
|
+
*/
|
|
152
|
+
function isEexist(error) {
|
|
153
|
+
return error?.code === 'EEXIST';
|
|
154
|
+
}
|