@nitpicker/archive 0.0.0-stage → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +191 -0
- package/README.md +105 -2
- package/lib/apply-connection-pragmas.d.ts +18 -0
- package/lib/apply-connection-pragmas.js +40 -0
- package/lib/archive-accessor.d.ts +265 -0
- package/lib/archive-accessor.js +477 -0
- package/lib/archive-lock.d.ts +45 -0
- package/lib/archive-lock.js +154 -0
- package/lib/archive.d.ts +736 -0
- package/lib/archive.js +1056 -0
- package/lib/body-hash/compute-body-hash.d.ts +32 -0
- package/lib/body-hash/compute-body-hash.js +35 -0
- package/lib/cache/clear-archive-cache-entry.d.ts +18 -0
- package/lib/cache/clear-archive-cache-entry.js +24 -0
- package/lib/cache/clear-archive-cache-root.d.ts +26 -0
- package/lib/cache/clear-archive-cache-root.js +32 -0
- package/lib/cache/compute-archive-cache-key.d.ts +39 -0
- package/lib/cache/compute-archive-cache-key.js +95 -0
- package/lib/cache/extract-archive-to-cache.d.ts +57 -0
- package/lib/cache/extract-archive-to-cache.js +326 -0
- package/lib/cache/get-archive-cache-root.d.ts +20 -0
- package/lib/cache/get-archive-cache-root.js +53 -0
- package/lib/cache/is-archive-cache-disabled.d.ts +24 -0
- package/lib/cache/is-archive-cache-disabled.js +34 -0
- package/lib/cache/list-archive-cache-entries.d.ts +23 -0
- package/lib/cache/list-archive-cache-entries.js +130 -0
- package/lib/cache/path-exists.d.ts +9 -0
- package/lib/cache/path-exists.js +18 -0
- package/lib/cache/resolve-archive-cache-dir.d.ts +26 -0
- package/lib/cache/resolve-archive-cache-dir.js +32 -0
- package/lib/cache/types.d.ts +33 -0
- package/lib/cache/types.js +1 -0
- package/lib/content-type/is-html-content-type.d.ts +17 -0
- package/lib/content-type/is-html-content-type.js +19 -0
- package/lib/content-type/normalize-content-type.d.ts +14 -0
- package/lib/content-type/normalize-content-type.js +20 -0
- package/lib/create-adjunct-tables.d.ts +66 -0
- package/lib/create-adjunct-tables.js +510 -0
- package/lib/create-entity-tables.d.ts +191 -0
- package/lib/create-entity-tables.js +387 -0
- package/lib/create-progress-reporter.d.ts +30 -0
- package/lib/create-progress-reporter.js +38 -0
- package/lib/create-ref-tables.d.ts +35 -0
- package/lib/create-ref-tables.js +185 -0
- package/lib/create-template-tables.d.ts +28 -0
- package/lib/create-template-tables.js +92 -0
- package/lib/database.d.ts +549 -0
- package/lib/database.js +804 -0
- package/lib/db-ops/_shared/clear-write-ref-caches.d.ts +27 -0
- package/lib/db-ops/_shared/clear-write-ref-caches.js +35 -0
- package/lib/db-ops/_shared/compress-payload.d.ts +26 -0
- package/lib/db-ops/_shared/compress-payload.js +30 -0
- package/lib/db-ops/_shared/create-write-ref-caches.d.ts +17 -0
- package/lib/db-ops/_shared/create-write-ref-caches.js +27 -0
- package/lib/db-ops/_shared/decode-json-ref.d.ts +17 -0
- package/lib/db-ops/_shared/decode-json-ref.js +31 -0
- package/lib/db-ops/_shared/keyset-paginate-by-id.d.ts +49 -0
- package/lib/db-ops/_shared/keyset-paginate-by-id.js +69 -0
- package/lib/db-ops/_shared/load-response-headers-by-set-ids.d.ts +20 -0
- package/lib/db-ops/_shared/load-response-headers-by-set-ids.js +53 -0
- package/lib/db-ops/_shared/resolve-content-item-id.d.ts +62 -0
- package/lib/db-ops/_shared/resolve-content-item-id.js +143 -0
- package/lib/db-ops/_shared/resolve-url-or-blob.d.ts +23 -0
- package/lib/db-ops/_shared/resolve-url-or-blob.js +29 -0
- package/lib/db-ops/_shared/retry-setting.d.ts +16 -0
- package/lib/db-ops/_shared/retry-setting.js +18 -0
- package/lib/db-ops/_shared/safe-parse-json.d.ts +11 -0
- package/lib/db-ops/_shared/safe-parse-json.js +18 -0
- package/lib/db-ops/_shared/types.d.ts +94 -0
- package/lib/db-ops/_shared/types.js +1 -0
- package/lib/db-ops/_shared/upsert-blob-ref.d.ts +25 -0
- package/lib/db-ops/_shared/upsert-blob-ref.js +48 -0
- package/lib/db-ops/_shared/upsert-content-type-ref.d.ts +30 -0
- package/lib/db-ops/_shared/upsert-content-type-ref.js +45 -0
- package/lib/db-ops/_shared/upsert-json-ref.d.ts +22 -0
- package/lib/db-ops/_shared/upsert-json-ref.js +41 -0
- package/lib/db-ops/_shared/upsert-response-headers.d.ts +35 -0
- package/lib/db-ops/_shared/upsert-response-headers.js +49 -0
- package/lib/db-ops/_shared/upsert-url-ref.d.ts +39 -0
- package/lib/db-ops/_shared/upsert-url-ref.js +62 -0
- package/lib/db-ops/anchors/get-anchors-on-page.d.ts +10 -0
- package/lib/db-ops/anchors/get-anchors-on-page.js +21 -0
- package/lib/db-ops/config/get-base-url.d.ts +8 -0
- package/lib/db-ops/config/get-base-url.js +14 -0
- package/lib/db-ops/config/get-config.d.ts +10 -0
- package/lib/db-ops/config/get-config.js +29 -0
- package/lib/db-ops/config/get-created-cwd.d.ts +11 -0
- package/lib/db-ops/config/get-created-cwd.js +13 -0
- package/lib/db-ops/config/get-name.d.ts +8 -0
- package/lib/db-ops/config/get-name.js +14 -0
- package/lib/db-ops/config/info-column-allowlist.d.ts +7 -0
- package/lib/db-ops/config/info-column-allowlist.js +29 -0
- package/lib/db-ops/config/info-json-columns.d.ts +5 -0
- package/lib/db-ops/config/info-json-columns.js +11 -0
- package/lib/db-ops/config/set-config.d.ts +12 -0
- package/lib/db-ops/config/set-config.js +21 -0
- package/lib/db-ops/config/update-config.d.ts +17 -0
- package/lib/db-ops/config/update-config.js +36 -0
- package/lib/db-ops/console-logs/compute-console-log-hash.d.ts +19 -0
- package/lib/db-ops/console-logs/compute-console-log-hash.js +30 -0
- package/lib/db-ops/console-logs/replace-console-logs.d.ts +34 -0
- package/lib/db-ops/console-logs/replace-console-logs.js +123 -0
- package/lib/db-ops/console-logs/stringify-console-log-args.d.ts +16 -0
- package/lib/db-ops/console-logs/stringify-console-log-args.js +26 -0
- package/lib/db-ops/console-logs/types.d.ts +38 -0
- package/lib/db-ops/console-logs/types.js +1 -0
- package/lib/db-ops/console-logs/upsert-console-log-item.d.ts +29 -0
- package/lib/db-ops/console-logs/upsert-console-log-item.js +52 -0
- package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.d.ts +18 -0
- package/lib/db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js +23 -0
- package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.d.ts +12 -0
- package/lib/db-ops/dedupe-cap/finalize-dedupe-cap-event.js +15 -0
- package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.d.ts +14 -0
- package/lib/db-ops/dedupe-cap/insert-dedupe-cap-event.js +30 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.d.ts +57 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-observations.js +87 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.d.ts +21 -0
- package/lib/db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js +27 -0
- package/lib/db-ops/errors/insert-crawl-error.d.ts +15 -0
- package/lib/db-ops/errors/insert-crawl-error.js +21 -0
- package/lib/db-ops/errors/insert-page-error.d.ts +21 -0
- package/lib/db-ops/errors/insert-page-error.js +30 -0
- package/lib/db-ops/errors/list-dns-burned-host-candidates.d.ts +33 -0
- package/lib/db-ops/errors/list-dns-burned-host-candidates.js +158 -0
- package/lib/db-ops/html/read-page-html-blob.d.ts +17 -0
- package/lib/db-ops/html/read-page-html-blob.js +20 -0
- package/lib/db-ops/lifecycle/checkpoint.d.ts +8 -0
- package/lib/db-ops/lifecycle/checkpoint.js +9 -0
- package/lib/db-ops/lifecycle/destroy.d.ts +6 -0
- package/lib/db-ops/lifecycle/destroy.js +7 -0
- package/lib/db-ops/lifecycle/init.d.ts +58 -0
- package/lib/db-ops/lifecycle/init.js +122 -0
- package/lib/db-ops/list-reconcile/record-list-reconcile-run.d.ts +22 -0
- package/lib/db-ops/list-reconcile/record-list-reconcile-run.js +41 -0
- package/lib/db-ops/meta/get-audios-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-audios-of-page.js +15 -0
- package/lib/db-ops/meta/get-buttons-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-buttons-of-page.js +15 -0
- package/lib/db-ops/meta/get-canvases-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-canvases-of-page.js +15 -0
- package/lib/db-ops/meta/get-custom-elements-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-custom-elements-of-page.js +15 -0
- package/lib/db-ops/meta/get-headings-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-headings-of-page.js +15 -0
- package/lib/db-ops/meta/get-iframes-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-iframes-of-page.js +15 -0
- package/lib/db-ops/meta/get-jsonld-of-page.d.ts +13 -0
- package/lib/db-ops/meta/get-jsonld-of-page.js +27 -0
- package/lib/db-ops/meta/get-main-content-images-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-main-content-images-of-page.js +15 -0
- package/lib/db-ops/meta/get-main-content-tables-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-main-content-tables-of-page.js +15 -0
- package/lib/db-ops/meta/get-page-technologies-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-page-technologies-of-page.js +15 -0
- package/lib/db-ops/meta/get-technology-signals-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-technology-signals-of-page.js +15 -0
- package/lib/db-ops/meta/get-videos-of-page.d.ts +11 -0
- package/lib/db-ops/meta/get-videos-of-page.js +15 -0
- package/lib/db-ops/outages/close-network-outage.d.ts +12 -0
- package/lib/db-ops/outages/close-network-outage.js +15 -0
- package/lib/db-ops/outages/close-stale-open-network-outages.d.ts +26 -0
- package/lib/db-ops/outages/close-stale-open-network-outages.js +42 -0
- package/lib/db-ops/outages/compute-outage-clamp-timestamp.d.ts +26 -0
- package/lib/db-ops/outages/compute-outage-clamp-timestamp.js +31 -0
- package/lib/db-ops/outages/insert-network-outage.d.ts +14 -0
- package/lib/db-ops/outages/insert-network-outage.js +29 -0
- package/lib/db-ops/outages/list-network-outages.d.ts +28 -0
- package/lib/db-ops/outages/list-network-outages.js +42 -0
- package/lib/db-ops/pages/order/set-url-order.d.ts +13 -0
- package/lib/db-ops/pages/order/set-url-order.js +38 -0
- package/lib/db-ops/pages/read/build-page-query.d.ts +18 -0
- package/lib/db-ops/pages/read/build-page-query.js +40 -0
- package/lib/db-ops/pages/read/get-crawling-state.d.ts +117 -0
- package/lib/db-ops/pages/read/get-crawling-state.js +161 -0
- package/lib/db-ops/pages/read/get-existing-page-urls.d.ts +15 -0
- package/lib/db-ops/pages/read/get-existing-page-urls.js +30 -0
- package/lib/db-ops/pages/read/get-page-count.d.ts +12 -0
- package/lib/db-ops/pages/read/get-page-count.js +21 -0
- package/lib/db-ops/pages/read/get-page-source-by-url.d.ts +24 -0
- package/lib/db-ops/pages/read/get-page-source-by-url.js +28 -0
- package/lib/db-ops/pages/read/get-pages-with-rels.d.ts +38 -0
- package/lib/db-ops/pages/read/get-pages-with-rels.js +107 -0
- package/lib/db-ops/pages/read/get-pages.d.ts +11 -0
- package/lib/db-ops/pages/read/get-pages.js +51 -0
- package/lib/db-ops/pages/read/get-scraped-html-page-count.d.ts +18 -0
- package/lib/db-ops/pages/read/get-scraped-html-page-count.js +25 -0
- package/lib/db-ops/pages/read/reconstruct-page-rows.d.ts +31 -0
- package/lib/db-ops/pages/read/reconstruct-page-rows.js +32 -0
- package/lib/db-ops/pages/reset/clear-page-derived-rows.d.ts +20 -0
- package/lib/db-ops/pages/reset/clear-page-derived-rows.js +37 -0
- package/lib/db-ops/pages/reset/repromote-external-pages.d.ts +30 -0
- package/lib/db-ops/pages/reset/repromote-external-pages.js +91 -0
- package/lib/db-ops/pages/reset/reset-failed-pages.d.ts +88 -0
- package/lib/db-ops/pages/reset/reset-failed-pages.js +216 -0
- package/lib/db-ops/pages/reset/reset-pages-by-urls.d.ts +80 -0
- package/lib/db-ops/pages/reset/reset-pages-by-urls.js +151 -0
- package/lib/db-ops/pages/reset/retryable-image-scan-codes.d.ts +24 -0
- package/lib/db-ops/pages/reset/retryable-image-scan-codes.js +24 -0
- package/lib/db-ops/pages/write/insert-audios.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-audios.js +25 -0
- package/lib/db-ops/pages/write/insert-buttons.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-buttons.js +29 -0
- package/lib/db-ops/pages/write/insert-canvases.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-canvases.js +26 -0
- package/lib/db-ops/pages/write/insert-custom-elements.d.ts +27 -0
- package/lib/db-ops/pages/write/insert-custom-elements.js +40 -0
- package/lib/db-ops/pages/write/insert-headings.d.ts +15 -0
- package/lib/db-ops/pages/write/insert-headings.js +27 -0
- package/lib/db-ops/pages/write/insert-iframes.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-iframes.js +28 -0
- package/lib/db-ops/pages/write/insert-inventory-content-items.d.ts +38 -0
- package/lib/db-ops/pages/write/insert-inventory-content-items.js +60 -0
- package/lib/db-ops/pages/write/insert-inventory-seeds.d.ts +36 -0
- package/lib/db-ops/pages/write/insert-inventory-seeds.js +48 -0
- package/lib/db-ops/pages/write/insert-inventory-skipped-pages.d.ts +42 -0
- package/lib/db-ops/pages/write/insert-inventory-skipped-pages.js +56 -0
- package/lib/db-ops/pages/write/insert-jsonld.d.ts +17 -0
- package/lib/db-ops/pages/write/insert-jsonld.js +49 -0
- package/lib/db-ops/pages/write/insert-main-content-images.d.ts +20 -0
- package/lib/db-ops/pages/write/insert-main-content-images.js +32 -0
- package/lib/db-ops/pages/write/insert-main-content-tables.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-main-content-tables.js +29 -0
- package/lib/db-ops/pages/write/insert-page.d.ts +56 -0
- package/lib/db-ops/pages/write/insert-page.js +259 -0
- package/lib/db-ops/pages/write/insert-technologies.d.ts +25 -0
- package/lib/db-ops/pages/write/insert-technologies.js +45 -0
- package/lib/db-ops/pages/write/insert-videos.d.ts +14 -0
- package/lib/db-ops/pages/write/insert-videos.js +28 -0
- package/lib/db-ops/pages/write/link-redirect-sources.d.ts +42 -0
- package/lib/db-ops/pages/write/link-redirect-sources.js +101 -0
- package/lib/db-ops/pages/write/record-redirect.d.ts +35 -0
- package/lib/db-ops/pages/write/record-redirect.js +100 -0
- package/lib/db-ops/pages/write/set-skipped-page.d.ts +13 -0
- package/lib/db-ops/pages/write/set-skipped-page.js +24 -0
- package/lib/db-ops/pages/write/update-page.d.ts +40 -0
- package/lib/db-ops/pages/write/update-page.js +419 -0
- package/lib/db-ops/pages/write/write-page-html-blob.d.ts +19 -0
- package/lib/db-ops/pages/write/write-page-html-blob.js +41 -0
- package/lib/db-ops/referrers/get-redirects-for-pages.d.ts +9 -0
- package/lib/db-ops/referrers/get-redirects-for-pages.js +15 -0
- package/lib/db-ops/referrers/get-referrers-of-page.d.ts +17 -0
- package/lib/db-ops/referrers/get-referrers-of-page.js +32 -0
- package/lib/db-ops/referrers/get-referrers-of-resource.d.ts +8 -0
- package/lib/db-ops/referrers/get-referrers-of-resource.js +15 -0
- package/lib/db-ops/resources/build-resource-query.d.ts +25 -0
- package/lib/db-ops/resources/build-resource-query.js +29 -0
- package/lib/db-ops/resources/get-existing-resource-urls.d.ts +9 -0
- package/lib/db-ops/resources/get-existing-resource-urls.js +24 -0
- package/lib/db-ops/resources/get-resource-by-url.d.ts +13 -0
- package/lib/db-ops/resources/get-resource-by-url.js +22 -0
- package/lib/db-ops/resources/get-resource-url-list.d.ts +17 -0
- package/lib/db-ops/resources/get-resource-url-list.js +26 -0
- package/lib/db-ops/resources/get-resources.d.ts +8 -0
- package/lib/db-ops/resources/get-resources.js +11 -0
- package/lib/db-ops/resources/insert-inventory-resources.d.ts +24 -0
- package/lib/db-ops/resources/insert-inventory-resources.js +64 -0
- package/lib/db-ops/resources/insert-resource-referrers.d.ts +15 -0
- package/lib/db-ops/resources/insert-resource-referrers.js +54 -0
- package/lib/db-ops/resources/insert-resource.d.ts +34 -0
- package/lib/db-ops/resources/insert-resource.js +73 -0
- package/lib/db-ops/resources/reconstruct-resource-rows.d.ts +26 -0
- package/lib/db-ops/resources/reconstruct-resource-rows.js +30 -0
- package/lib/db-ops/templates/assign-template-labels.d.ts +60 -0
- package/lib/db-ops/templates/assign-template-labels.js +187 -0
- package/lib/db-ops/templates/replace-page-templates.d.ts +44 -0
- package/lib/db-ops/templates/replace-page-templates.js +174 -0
- package/lib/db-ops/templates/serialize-template-cluster-reason.d.ts +16 -0
- package/lib/db-ops/templates/serialize-template-cluster-reason.js +17 -0
- package/lib/db-ops/templates/types.d.ts +103 -0
- package/lib/db-ops/templates/types.js +1 -0
- package/lib/debug.d.ts +8 -0
- package/lib/debug.js +9 -0
- package/lib/decode-html-blob.d.ts +20 -0
- package/lib/decode-html-blob.js +23 -0
- package/lib/decode-page-html-blob-bytes.d.ts +25 -0
- package/lib/decode-page-html-blob-bytes.js +29 -0
- package/lib/decode-stored-blob-bytes.d.ts +25 -0
- package/lib/decode-stored-blob-bytes.js +47 -0
- package/lib/derive-lineage-from-parent.d.ts +37 -0
- package/lib/derive-lineage-from-parent.js +42 -0
- package/lib/drop-legacy-tables.d.ts +45 -0
- package/lib/drop-legacy-tables.js +56 -0
- package/lib/error-kind/classify-error-kind.d.ts +20 -0
- package/lib/error-kind/classify-error-kind.js +136 -0
- package/lib/error-kind/network-related-error-kinds.d.ts +23 -0
- package/lib/error-kind/network-related-error-kinds.js +28 -0
- package/lib/error-kind/permanent-error-kinds.d.ts +48 -0
- package/lib/error-kind/permanent-error-kinds.js +54 -0
- package/lib/error-kind/types.d.ts +54 -0
- package/lib/error-kind/types.js +1 -0
- package/lib/filesystem/append-text.d.ts +9 -0
- package/lib/filesystem/append-text.js +14 -0
- package/lib/filesystem/copy-dir-sync.d.ts +6 -0
- package/lib/filesystem/copy-dir-sync.js +9 -0
- package/lib/filesystem/copy-dir.d.ts +7 -0
- package/lib/filesystem/copy-dir.js +13 -0
- package/lib/filesystem/copy-file-with-progress.d.ts +20 -0
- package/lib/filesystem/copy-file-with-progress.js +34 -0
- package/lib/filesystem/exists.d.ts +6 -0
- package/lib/filesystem/exists.js +9 -0
- package/lib/filesystem/get-file-list.d.ts +8 -0
- package/lib/filesystem/get-file-list.js +12 -0
- package/lib/filesystem/is-dir.d.ts +6 -0
- package/lib/filesystem/is-dir.js +10 -0
- package/lib/filesystem/mkdir.d.ts +8 -0
- package/lib/filesystem/mkdir.js +15 -0
- package/lib/filesystem/output-binary.d.ts +12 -0
- package/lib/filesystem/output-binary.js +17 -0
- package/lib/filesystem/output-json.d.ts +9 -0
- package/lib/filesystem/output-json.js +14 -0
- package/lib/filesystem/output-text.d.ts +11 -0
- package/lib/filesystem/output-text.js +32 -0
- package/lib/filesystem/parse-pax-path.d.ts +18 -0
- package/lib/filesystem/parse-pax-path.js +47 -0
- package/lib/filesystem/parse-tar-size-field.d.ts +13 -0
- package/lib/filesystem/parse-tar-size-field.js +40 -0
- package/lib/filesystem/peek-tar-top-dir.d.ts +35 -0
- package/lib/filesystem/peek-tar-top-dir.js +215 -0
- package/lib/filesystem/read-json.d.ts +7 -0
- package/lib/filesystem/read-json.js +11 -0
- package/lib/filesystem/read-text.d.ts +6 -0
- package/lib/filesystem/read-text.js +10 -0
- package/lib/filesystem/readline.d.ts +11 -0
- package/lib/filesystem/readline.js +26 -0
- package/lib/filesystem/remove.d.ts +5 -0
- package/lib/filesystem/remove.js +10 -0
- package/lib/filesystem/rename.d.ts +16 -0
- package/lib/filesystem/rename.js +48 -0
- package/lib/filesystem/tar.d.ts +23 -0
- package/lib/filesystem/tar.js +78 -0
- package/lib/filesystem/untar.d.ts +33 -0
- package/lib/filesystem/untar.js +84 -0
- package/lib/get-failed-page-messages.d.ts +56 -0
- package/lib/get-failed-page-messages.js +146 -0
- package/lib/get-json.d.ts +10 -0
- package/lib/get-json.js +25 -0
- package/lib/html-snapshot/match-selector-html.d.ts +25 -0
- package/lib/html-snapshot/match-selector-html.js +28 -0
- package/lib/html-snapshot/match-selector-snapshots.d.ts +33 -0
- package/lib/html-snapshot/match-selector-snapshots.js +33 -0
- package/lib/html-snapshot/search-html-document.d.ts +29 -0
- package/lib/html-snapshot/search-html-document.js +27 -0
- package/lib/html-snapshot/search-html-snapshots.d.ts +36 -0
- package/lib/html-snapshot/search-html-snapshots.js +35 -0
- package/lib/html-snapshot/types.d.ts +111 -0
- package/lib/html-snapshot/types.js +7 -0
- package/lib/html-snapshot/validate-search-pattern.d.ts +15 -0
- package/lib/html-snapshot/validate-search-pattern.js +17 -0
- package/lib/init-schema.d.ts +48 -0
- package/lib/init-schema.js +124 -0
- package/lib/is-inventory-source.d.ts +21 -0
- package/lib/is-inventory-source.js +22 -0
- package/lib/is-within-outage-window.d.ts +49 -0
- package/lib/is-within-outage-window.js +33 -0
- package/lib/isolated-cluster/create-isolated-cluster-run.d.ts +24 -0
- package/lib/isolated-cluster/create-isolated-cluster-run.js +26 -0
- package/lib/libsql-dialect.d.ts +115 -0
- package/lib/libsql-dialect.js +116 -0
- package/lib/limited-page-ids.d.ts +9 -0
- package/lib/limited-page-ids.js +18 -0
- package/lib/meta/assert-compatible-version.d.ts +60 -0
- package/lib/meta/assert-compatible-version.js +93 -0
- package/lib/meta/classify-jsonld-type.d.ts +23 -0
- package/lib/meta/classify-jsonld-type.js +43 -0
- package/lib/meta/compare-semver.d.ts +23 -0
- package/lib/meta/compare-semver.js +51 -0
- package/lib/meta/compute-main-contents-denormalized.d.ts +41 -0
- package/lib/meta/compute-main-contents-denormalized.js +88 -0
- package/lib/meta/compute-page-denormalized.d.ts +20 -0
- package/lib/meta/compute-page-denormalized.js +35 -0
- package/lib/meta/derive-flat-from-meta.d.ts +34 -0
- package/lib/meta/derive-flat-from-meta.js +158 -0
- package/lib/meta/derive-meta-extras.d.ts +20 -0
- package/lib/meta/derive-meta-extras.js +23 -0
- package/lib/meta/extract-tags-for-archive.d.ts +17 -0
- package/lib/meta/extract-tags-for-archive.js +36 -0
- package/lib/meta/parse-main-content-class-list.d.ts +19 -0
- package/lib/meta/parse-main-content-class-list.js +34 -0
- package/lib/meta/summarize-jsonld.d.ts +17 -0
- package/lib/meta/summarize-jsonld.js +29 -0
- package/lib/meta/technologies/combine-technology-confidence.d.ts +46 -0
- package/lib/meta/technologies/combine-technology-confidence.js +93 -0
- package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.d.ts +45 -0
- package/lib/meta/technologies/convert-legacy-page-tags-to-inserts.js +53 -0
- package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.d.ts +33 -0
- package/lib/meta/technologies/convert-tag-row-to-wappalyzer-signal.js +46 -0
- package/lib/meta/technologies/extract-technologies-for-archive.d.ts +33 -0
- package/lib/meta/technologies/extract-technologies-for-archive.js +30 -0
- package/lib/meta/technologies/match-js-license-comment-signal.d.ts +21 -0
- package/lib/meta/technologies/match-js-license-comment-signal.js +23 -0
- package/lib/meta/technologies/match-structural-technology-signals.d.ts +26 -0
- package/lib/meta/technologies/match-structural-technology-signals.js +48 -0
- package/lib/meta/technologies/normalize-wappalyzer-entries.d.ts +25 -0
- package/lib/meta/technologies/normalize-wappalyzer-entries.js +35 -0
- package/lib/meta/technologies/technology-signal-definitions.d.ts +44 -0
- package/lib/meta/technologies/technology-signal-definitions.js +60 -0
- package/lib/meta/technologies/types.d.ts +70 -0
- package/lib/meta/technologies/types.js +13 -0
- package/lib/meta/types.d.ts +1490 -0
- package/lib/meta/types.js +65 -0
- package/lib/migrate-content-items-alias-of-id.d.ts +52 -0
- package/lib/migrate-content-items-alias-of-id.js +69 -0
- package/lib/migrate-content-items-dedupe-cap-event-id.d.ts +46 -0
- package/lib/migrate-content-items-dedupe-cap-event-id.js +62 -0
- package/lib/migrate-content-items-is-metadata-only.d.ts +44 -0
- package/lib/migrate-content-items-is-metadata-only.js +60 -0
- package/lib/migrate-entity-tables.d.ts +55 -0
- package/lib/migrate-entity-tables.js +66 -0
- package/lib/migrate-info-created-cwd.d.ts +17 -0
- package/lib/migrate-info-created-cwd.js +36 -0
- package/lib/migrate-info-main-content-selector.d.ts +16 -0
- package/lib/migrate-info-main-content-selector.js +35 -0
- package/lib/migrate-info-request-header-names.d.ts +16 -0
- package/lib/migrate-info-request-header-names.js +35 -0
- package/lib/migrate-info-roots.d.ts +20 -0
- package/lib/migrate-info-roots.js +52 -0
- package/lib/migrate-inventory-runs-to-list-reconcile-runs.d.ts +34 -0
- package/lib/migrate-inventory-runs-to-list-reconcile-runs.js +51 -0
- package/lib/migrate-list-reconcile-runs-exclude-skipped.d.ts +25 -0
- package/lib/migrate-list-reconcile-runs-exclude-skipped.js +44 -0
- package/lib/migrate-list-reconcile-runs-invalid-skipped.d.ts +16 -0
- package/lib/migrate-list-reconcile-runs-invalid-skipped.js +35 -0
- package/lib/migrate-main-contents-columns.d.ts +26 -0
- package/lib/migrate-main-contents-columns.js +61 -0
- package/lib/migrate-page-meta-body-hash.d.ts +44 -0
- package/lib/migrate-page-meta-body-hash.js +63 -0
- package/lib/migrate-page-meta-console-error-count.d.ts +27 -0
- package/lib/migrate-page-meta-console-error-count.js +45 -0
- package/lib/migrate-page-meta-custom-element-count.d.ts +33 -0
- package/lib/migrate-page-meta-custom-element-count.js +51 -0
- package/lib/migrate-page-meta-image-scan.d.ts +27 -0
- package/lib/migrate-page-meta-image-scan.js +52 -0
- package/lib/migrate-page-tags-to-page-technologies.d.ts +39 -0
- package/lib/migrate-page-tags-to-page-technologies.js +79 -0
- package/lib/migrate-ref-tables.d.ts +25 -0
- package/lib/migrate-ref-tables.js +38 -0
- package/lib/native-executor/check-native-executor-config.d.ts +17 -0
- package/lib/native-executor/check-native-executor-config.js +20 -0
- package/lib/page-meta-column-maps.d.ts +32 -0
- package/lib/page-meta-column-maps.js +43 -0
- package/lib/page.d.ts +546 -0
- package/lib/page.js +764 -0
- package/lib/peek-archive-lock.d.ts +40 -0
- package/lib/peek-archive-lock.js +62 -0
- package/lib/populate-entity-tables/collapse-anchor-rows.d.ts +41 -0
- package/lib/populate-entity-tables/collapse-anchor-rows.js +87 -0
- package/lib/populate-entity-tables/derive-dom-path.d.ts +35 -0
- package/lib/populate-entity-tables/derive-dom-path.js +72 -0
- package/lib/populate-entity-tables/is-blob-ref-value.d.ts +16 -0
- package/lib/populate-entity-tables/is-blob-ref-value.js +19 -0
- package/lib/populate-entity-tables/match-images-to-dom-paths.d.ts +66 -0
- package/lib/populate-entity-tables/match-images-to-dom-paths.js +96 -0
- package/lib/populate-entity-tables/populate-anchor-edges.d.ts +33 -0
- package/lib/populate-entity-tables/populate-anchor-edges.js +153 -0
- package/lib/populate-entity-tables/populate-content-items.d.ts +40 -0
- package/lib/populate-entity-tables/populate-content-items.js +141 -0
- package/lib/populate-entity-tables/populate-entities.d.ts +81 -0
- package/lib/populate-entity-tables/populate-entities.js +110 -0
- package/lib/populate-entity-tables/populate-image-items.d.ts +91 -0
- package/lib/populate-entity-tables/populate-image-items.js +226 -0
- package/lib/populate-entity-tables/populate-page-meta.d.ts +33 -0
- package/lib/populate-entity-tables/populate-page-meta.js +267 -0
- package/lib/populate-entity-tables/populate-resource-items.d.ts +22 -0
- package/lib/populate-entity-tables/populate-resource-items.js +114 -0
- package/lib/populate-entity-tables/populate-resource-ref-edges.d.ts +31 -0
- package/lib/populate-entity-tables/populate-resource-ref-edges.js +33 -0
- package/lib/populate-entity-tables/resolve-blob-refs.d.ts +31 -0
- package/lib/populate-entity-tables/resolve-blob-refs.js +100 -0
- package/lib/populate-entity-tables/resolve-content-type-refs.d.ts +22 -0
- package/lib/populate-entity-tables/resolve-content-type-refs.js +27 -0
- package/lib/populate-entity-tables/resolve-header-sets.d.ts +49 -0
- package/lib/populate-entity-tables/resolve-header-sets.js +122 -0
- package/lib/populate-entity-tables/resolve-json-refs.d.ts +25 -0
- package/lib/populate-entity-tables/resolve-json-refs.js +67 -0
- package/lib/populate-entity-tables/resolve-text-refs.d.ts +30 -0
- package/lib/populate-entity-tables/resolve-text-refs.js +61 -0
- package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.d.ts +21 -0
- package/lib/populate-entity-tables/resolve-url-or-blob-from-maps.js +27 -0
- package/lib/populate-entity-tables/resolve-url-refs.d.ts +33 -0
- package/lib/populate-entity-tables/resolve-url-refs.js +60 -0
- package/lib/populate-entity-tables/test-utils/count-rows.d.ts +17 -0
- package/lib/populate-entity-tables/test-utils/count-rows.js +20 -0
- package/lib/populate-entity-tables/test-utils/seed-content-items.d.ts +25 -0
- package/lib/populate-entity-tables/test-utils/seed-content-items.js +42 -0
- package/lib/populate-entity-tables/test-utils/setup-entities-db.d.ts +30 -0
- package/lib/populate-entity-tables/test-utils/setup-entities-db.js +175 -0
- package/lib/populate-entity-tables/types.d.ts +157 -0
- package/lib/populate-entity-tables/types.js +12 -0
- package/lib/populate-entity-tables/upsert-text-refs.d.ts +38 -0
- package/lib/populate-entity-tables/upsert-text-refs.js +78 -0
- package/lib/populate-ref-tables/classify-content-type.d.ts +16 -0
- package/lib/populate-ref-tables/classify-content-type.js +52 -0
- package/lib/populate-ref-tables/compute-content-hash.d.ts +22 -0
- package/lib/populate-ref-tables/compute-content-hash.js +26 -0
- package/lib/populate-ref-tables/compute-header-flags.d.ts +16 -0
- package/lib/populate-ref-tables/compute-header-flags.js +70 -0
- package/lib/populate-ref-tables/content-type-rules.d.ts +38 -0
- package/lib/populate-ref-tables/content-type-rules.js +133 -0
- package/lib/populate-ref-tables/create-header-table-caches.d.ts +25 -0
- package/lib/populate-ref-tables/create-header-table-caches.js +49 -0
- package/lib/populate-ref-tables/data-uri-url-refs-limit.d.ts +15 -0
- package/lib/populate-ref-tables/data-uri-url-refs-limit.js +15 -0
- package/lib/populate-ref-tables/decode-data-uri.d.ts +21 -0
- package/lib/populate-ref-tables/decode-data-uri.js +126 -0
- package/lib/populate-ref-tables/decompose-header-set.d.ts +29 -0
- package/lib/populate-ref-tables/decompose-header-set.js +157 -0
- package/lib/populate-ref-tables/decompose-url.d.ts +25 -0
- package/lib/populate-ref-tables/decompose-url.js +70 -0
- package/lib/populate-ref-tables/header-stability.d.ts +19 -0
- package/lib/populate-ref-tables/header-stability.js +22 -0
- package/lib/populate-ref-tables/header-value-cache-key.d.ts +17 -0
- package/lib/populate-ref-tables/header-value-cache-key.js +19 -0
- package/lib/populate-ref-tables/normalize-mime.d.ts +24 -0
- package/lib/populate-ref-tables/normalize-mime.js +36 -0
- package/lib/populate-ref-tables/populate-blob-refs.d.ts +38 -0
- package/lib/populate-ref-tables/populate-blob-refs.js +134 -0
- package/lib/populate-ref-tables/populate-content-type-refs.d.ts +27 -0
- package/lib/populate-ref-tables/populate-content-type-refs.js +70 -0
- package/lib/populate-ref-tables/populate-header-tables.d.ts +35 -0
- package/lib/populate-ref-tables/populate-header-tables.js +80 -0
- package/lib/populate-ref-tables/populate-json-refs.d.ts +29 -0
- package/lib/populate-ref-tables/populate-json-refs.js +101 -0
- package/lib/populate-ref-tables/populate-refs.d.ts +51 -0
- package/lib/populate-ref-tables/populate-refs.js +62 -0
- package/lib/populate-ref-tables/populate-text-refs.d.ts +32 -0
- package/lib/populate-ref-tables/populate-text-refs.js +133 -0
- package/lib/populate-ref-tables/populate-url-refs.d.ts +28 -0
- package/lib/populate-ref-tables/populate-url-refs.js +148 -0
- package/lib/populate-ref-tables/test-utils/count-rows.d.ts +15 -0
- package/lib/populate-ref-tables/test-utils/count-rows.js +17 -0
- package/lib/populate-ref-tables/types.d.ts +197 -0
- package/lib/populate-ref-tables/types.js +7 -0
- package/lib/populate-ref-tables/upsert-one-header-set.d.ts +34 -0
- package/lib/populate-ref-tables/upsert-one-header-set.js +208 -0
- package/lib/populate-ref-tables/volatile-header-names.d.ts +20 -0
- package/lib/populate-ref-tables/volatile-header-names.js +33 -0
- package/lib/redirect-table.d.ts +9 -0
- package/lib/redirect-table.js +28 -0
- package/lib/release-finished-statements.d.ts +42 -0
- package/lib/release-finished-statements.js +45 -0
- package/lib/resolve-redirect-chain.d.ts +33 -0
- package/lib/resolve-redirect-chain.js +27 -0
- package/lib/resource.d.ts +46 -0
- package/lib/resource.js +62 -0
- package/lib/retarget-legacy-fk-tables.d.ts +61 -0
- package/lib/retarget-legacy-fk-tables.js +137 -0
- package/lib/safe-path.d.ts +9 -0
- package/lib/safe-path.js +17 -0
- package/lib/scope/build-scope-map.d.ts +21 -0
- package/lib/scope/build-scope-map.js +32 -0
- package/lib/scope/find-scope-entry.d.ts +25 -0
- package/lib/scope/find-scope-entry.js +45 -0
- package/lib/template-classification/classify-archive-page-templates.d.ts +42 -0
- package/lib/template-classification/classify-archive-page-templates.js +70 -0
- package/lib/template-classification/classify-page-templates.d.ts +33 -0
- package/lib/template-classification/classify-page-templates.js +60 -0
- package/lib/template-classification/collect-page-ids-with-html.d.ts +24 -0
- package/lib/template-classification/collect-page-ids-with-html.js +30 -0
- package/lib/template-classification/collect-page-stylesheet-urls.d.ts +32 -0
- package/lib/template-classification/collect-page-stylesheet-urls.js +130 -0
- package/lib/template-classification/create-content-root-hint.d.ts +41 -0
- package/lib/template-classification/create-content-root-hint.js +51 -0
- package/lib/template-classification/create-page-cluster-factory.d.ts +36 -0
- package/lib/template-classification/create-page-cluster-factory.js +52 -0
- package/lib/template-classification/load-page-cluster-candidates.d.ts +32 -0
- package/lib/template-classification/load-page-cluster-candidates.js +93 -0
- package/lib/template-classification/page-cluster/resolve-page-cluster-keys.d.ts +34 -0
- package/lib/template-classification/page-cluster/resolve-page-cluster-keys.js +236 -0
- package/lib/template-classification/page-cluster/types.d.ts +123 -0
- package/lib/template-classification/page-cluster/types.js +1 -0
- package/lib/template-classification/types.d.ts +234 -0
- package/lib/template-classification/types.js +1 -0
- package/lib/test-utils/build-full-schema-test-db.d.ts +34 -0
- package/lib/test-utils/build-full-schema-test-db.js +74 -0
- package/lib/test-utils/create-legacy-analysis-tables.d.ts +13 -0
- package/lib/test-utils/create-legacy-analysis-tables.js +37 -0
- package/lib/test-utils/fk-parent-tables.d.ts +15 -0
- package/lib/test-utils/fk-parent-tables.js +19 -0
- package/lib/test-utils/seed-content-item.d.ts +35 -0
- package/lib/test-utils/seed-content-item.js +42 -0
- package/lib/test-utils/setup-legacy-fk-db.d.ts +34 -0
- package/lib/test-utils/setup-legacy-fk-db.js +271 -0
- package/lib/transfer/attach-source-database.d.ts +31 -0
- package/lib/transfer/attach-source-database.js +40 -0
- package/lib/transfer/build-split-config.d.ts +30 -0
- package/lib/transfer/build-split-config.js +42 -0
- package/lib/transfer/clear-replaced-page-rows.d.ts +23 -0
- package/lib/transfer/clear-replaced-page-rows.js +61 -0
- package/lib/transfer/concat-archives.d.ts +55 -0
- package/lib/transfer/concat-archives.js +95 -0
- package/lib/transfer/concat-source-transfer-phases.d.ts +10 -0
- package/lib/transfer/concat-source-transfer-phases.js +17 -0
- package/lib/transfer/content-item-rank-sql.d.ts +21 -0
- package/lib/transfer/content-item-rank-sql.js +23 -0
- package/lib/transfer/copy-anchor-edges.d.ts +28 -0
- package/lib/transfer/copy-anchor-edges.js +42 -0
- package/lib/transfer/copy-dictionaries-for-concat.d.ts +39 -0
- package/lib/transfer/copy-dictionaries-for-concat.js +81 -0
- package/lib/transfer/copy-dictionaries-for-split.d.ts +29 -0
- package/lib/transfer/copy-dictionaries-for-split.js +224 -0
- package/lib/transfer/copy-header-set-children.d.ts +27 -0
- package/lib/transfer/copy-header-set-children.js +51 -0
- package/lib/transfer/copy-image-items.d.ts +21 -0
- package/lib/transfer/copy-image-items.js +62 -0
- package/lib/transfer/copy-inventory-source-lists.d.ts +30 -0
- package/lib/transfer/copy-inventory-source-lists.js +61 -0
- package/lib/transfer/copy-journal-tables.d.ts +30 -0
- package/lib/transfer/copy-journal-tables.js +56 -0
- package/lib/transfer/copy-page-console-logs.d.ts +16 -0
- package/lib/transfer/copy-page-console-logs.js +27 -0
- package/lib/transfer/copy-page-html-blobs.d.ts +31 -0
- package/lib/transfer/copy-page-html-blobs.js +43 -0
- package/lib/transfer/copy-page-meta.d.ts +27 -0
- package/lib/transfer/copy-page-meta.js +68 -0
- package/lib/transfer/copy-page-template-clusters.d.ts +31 -0
- package/lib/transfer/copy-page-template-clusters.js +56 -0
- package/lib/transfer/copy-page-template-labels.d.ts +34 -0
- package/lib/transfer/copy-page-template-labels.js +88 -0
- package/lib/transfer/copy-resource-items.d.ts +29 -0
- package/lib/transfer/copy-resource-items.js +89 -0
- package/lib/transfer/copy-resource-ref-edges.d.ts +20 -0
- package/lib/transfer/copy-resource-ref-edges.js +33 -0
- package/lib/transfer/copy-simple-page-scoped-tables.d.ts +15 -0
- package/lib/transfer/copy-simple-page-scoped-tables.js +34 -0
- package/lib/transfer/count-external-pages-in-scope.d.ts +26 -0
- package/lib/transfer/count-external-pages-in-scope.js +40 -0
- package/lib/transfer/create-transfer-temp-tables.d.ts +30 -0
- package/lib/transfer/create-transfer-temp-tables.js +59 -0
- package/lib/transfer/dictionary-copy-specs.d.ts +32 -0
- package/lib/transfer/dictionary-copy-specs.js +28 -0
- package/lib/transfer/drop-transfer-temp-tables.d.ts +9 -0
- package/lib/transfer/drop-transfer-temp-tables.js +18 -0
- package/lib/transfer/flatten-redirect-chains.d.ts +44 -0
- package/lib/transfer/flatten-redirect-chains.js +74 -0
- package/lib/transfer/insert-content-items.d.ts +35 -0
- package/lib/transfer/insert-content-items.js +116 -0
- package/lib/transfer/list-transfer-columns.d.ts +35 -0
- package/lib/transfer/list-transfer-columns.js +45 -0
- package/lib/transfer/mark-in-scope-content-items.d.ts +31 -0
- package/lib/transfer/mark-in-scope-content-items.js +54 -0
- package/lib/transfer/merge-archive-configs.d.ts +51 -0
- package/lib/transfer/merge-archive-configs.js +85 -0
- package/lib/transfer/plan-content-items-for-concat.d.ts +33 -0
- package/lib/transfer/plan-content-items-for-concat.js +51 -0
- package/lib/transfer/plan-content-items-for-split.d.ts +42 -0
- package/lib/transfer/plan-content-items-for-split.js +83 -0
- package/lib/transfer/plan-resource-items.d.ts +40 -0
- package/lib/transfer/plan-resource-items.js +69 -0
- package/lib/transfer/quote-transfer-identifier.d.ts +18 -0
- package/lib/transfer/quote-transfer-identifier.js +20 -0
- package/lib/transfer/reclassify-resource-externality.d.ts +37 -0
- package/lib/transfer/reclassify-resource-externality.js +66 -0
- package/lib/transfer/remap-redirect-dest-ids.d.ts +34 -0
- package/lib/transfer/remap-redirect-dest-ids.js +51 -0
- package/lib/transfer/replace-content-items.d.ts +39 -0
- package/lib/transfer/replace-content-items.js +83 -0
- package/lib/transfer/simple-page-scoped-tables.d.ts +25 -0
- package/lib/transfer/simple-page-scoped-tables.js +38 -0
- package/lib/transfer/split-archive.d.ts +55 -0
- package/lib/transfer/split-archive.js +98 -0
- package/lib/transfer/split-source-transfer-phases.d.ts +11 -0
- package/lib/transfer/split-source-transfer-phases.js +18 -0
- package/lib/transfer/transfer-action.d.ts +34 -0
- package/lib/transfer/transfer-action.js +32 -0
- package/lib/transfer/transfer-archive-rows.d.ts +65 -0
- package/lib/transfer/transfer-archive-rows.js +156 -0
- package/lib/transfer/transfer-dictionary-tables.d.ts +12 -0
- package/lib/transfer/transfer-dictionary-tables.js +22 -0
- package/lib/transfer/transfer-post-phases.d.ts +10 -0
- package/lib/transfer/transfer-post-phases.js +13 -0
- package/lib/transfer/transfer-source-alias.d.ts +9 -0
- package/lib/transfer/transfer-source-alias.js +9 -0
- package/lib/transfer/types.d.ts +111 -0
- package/lib/transfer/types.js +35 -0
- package/lib/types.d.ts +772 -0
- package/lib/types.js +1 -0
- package/lib/url-alias/compute-tier-a-alias-key.d.ts +44 -0
- package/lib/url-alias/compute-tier-a-alias-key.js +47 -0
- package/lib/url-alias/compute-tier-b-alias-key.d.ts +23 -0
- package/lib/url-alias/compute-tier-b-alias-key.js +26 -0
- package/lib/url-alias/group-alias-candidates.d.ts +27 -0
- package/lib/url-alias/group-alias-candidates.js +29 -0
- package/lib/url-alias/types.d.ts +12 -0
- package/lib/url-alias/types.js +1 -0
- package/lib/url-pattern/compute-shape-key.d.ts +58 -0
- package/lib/url-pattern/compute-shape-key.js +129 -0
- package/lib/url-pattern/decompose-url.d.ts +33 -0
- package/lib/url-pattern/decompose-url.js +79 -0
- package/lib/url-pattern/reconstruct-url.d.ts +15 -0
- package/lib/url-pattern/reconstruct-url.js +33 -0
- package/lib/url-sort/compute-url-sort-ranks.d.ts +30 -0
- package/lib/url-sort/compute-url-sort-ranks.js +33 -0
- package/lib/utils/array/each-splitted.d.ts +10 -0
- package/lib/utils/array/each-splitted.js +14 -0
- package/lib/utils/debug.d.ts +8 -0
- package/lib/utils/debug.js +8 -0
- package/lib/utils/error/emit-error-with-retry.d.ts +40 -0
- package/lib/utils/error/emit-error-with-retry.js +44 -0
- package/lib/utils/error/emit-error.d.ts +39 -0
- package/lib/utils/error/emit-error.js +41 -0
- package/lib/utils/object/parse-response-headers.d.ts +12 -0
- package/lib/utils/object/parse-response-headers.js +26 -0
- package/lib/utils/object/redact-request-headers.d.ts +15 -0
- package/lib/utils/object/redact-request-headers.js +27 -0
- package/lib/utils/types/types.d.ts +383 -0
- package/lib/utils/types/types.js +1 -0
- package/lib/verify-migration/capture-rejection.d.ts +24 -0
- package/lib/verify-migration/capture-rejection.js +31 -0
- package/lib/verify-migration/check-anchor-edges-count.d.ts +34 -0
- package/lib/verify-migration/check-anchor-edges-count.js +72 -0
- package/lib/verify-migration/check-anchor-edges-sum.d.ts +13 -0
- package/lib/verify-migration/check-anchor-edges-sum.js +27 -0
- package/lib/verify-migration/check-content-items-count.d.ts +16 -0
- package/lib/verify-migration/check-content-items-count.js +30 -0
- package/lib/verify-migration/check-content-type-preservation.d.ts +22 -0
- package/lib/verify-migration/check-content-type-preservation.js +40 -0
- package/lib/verify-migration/check-foreign-key-integrity.d.ts +31 -0
- package/lib/verify-migration/check-foreign-key-integrity.js +47 -0
- package/lib/verify-migration/check-image-items-count.d.ts +12 -0
- package/lib/verify-migration/check-image-items-count.js +26 -0
- package/lib/verify-migration/check-page-meta-count.d.ts +15 -0
- package/lib/verify-migration/check-page-meta-count.js +31 -0
- package/lib/verify-migration/check-reader-parity.d.ts +23 -0
- package/lib/verify-migration/check-reader-parity.js +182 -0
- package/lib/verify-migration/check-resource-items-count.d.ts +17 -0
- package/lib/verify-migration/check-resource-items-count.js +33 -0
- package/lib/verify-migration/check-url-round-trip.d.ts +43 -0
- package/lib/verify-migration/check-url-round-trip.js +112 -0
- package/lib/verify-migration/types.d.ts +70 -0
- package/lib/verify-migration/types.js +63 -0
- package/lib/verify-migration/verify-migration.d.ts +41 -0
- package/lib/verify-migration/verify-migration.js +120 -0
- package/lib/with-bulk-write-pragmas.d.ts +51 -0
- package/lib/with-bulk-write-pragmas.js +132 -0
- package/lib/write-queue.d.ts +32 -0
- package/lib/write-queue.js +55 -0
- package/package.json +360 -5
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { eachSplitted } from '../../utils/array/each-splitted.js';
|
|
2
|
+
/**
|
|
3
|
+
* Return the subset of `urls` that already exist as `resource_items` rows.
|
|
4
|
+
* See `getExistingPageUrls` — same chunking strategy.
|
|
5
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
6
|
+
* @param urls - URL strings to probe.
|
|
7
|
+
* @returns URLs found among `resource_items`.
|
|
8
|
+
*/
|
|
9
|
+
export async function getExistingResourceUrls(knex, urls) {
|
|
10
|
+
if (urls.length === 0) {
|
|
11
|
+
return [];
|
|
12
|
+
}
|
|
13
|
+
const found = [];
|
|
14
|
+
await eachSplitted([...urls], 500, async (chunk) => {
|
|
15
|
+
const rows = await knex('resource_items')
|
|
16
|
+
.join('url_refs', 'url_refs.id', 'resource_items.url_id')
|
|
17
|
+
.select('url_refs.url as url')
|
|
18
|
+
.whereIn('url_refs.url', chunk);
|
|
19
|
+
for (const row of rows) {
|
|
20
|
+
found.push(row.url);
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
return found;
|
|
24
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { DB_Resource } from '../../types.js';
|
|
2
|
+
import type { Knex } from 'knex';
|
|
3
|
+
/**
|
|
4
|
+
* Retrieves a single sub-resource by its URL.
|
|
5
|
+
*
|
|
6
|
+
* Accepts multiple URL candidates because the stored key is the resource's
|
|
7
|
+
* `href` while callers may only know the hash-stripped form; the first match
|
|
8
|
+
* wins.
|
|
9
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
10
|
+
* @param urls - URL candidates to match against `url_refs.url`.
|
|
11
|
+
* @returns The reconstructed {@link DB_Resource} row, or `null` if none match.
|
|
12
|
+
*/
|
|
13
|
+
export declare function getResourceByUrl(knex: Knex, urls: readonly string[]): Promise<DB_Resource | null>;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { buildResourceQuery } from './build-resource-query.js';
|
|
2
|
+
import { reconstructResourceRows } from './reconstruct-resource-rows.js';
|
|
3
|
+
/**
|
|
4
|
+
* Retrieves a single sub-resource by its URL.
|
|
5
|
+
*
|
|
6
|
+
* Accepts multiple URL candidates because the stored key is the resource's
|
|
7
|
+
* `href` while callers may only know the hash-stripped form; the first match
|
|
8
|
+
* wins.
|
|
9
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
10
|
+
* @param urls - URL candidates to match against `url_refs.url`.
|
|
11
|
+
* @returns The reconstructed {@link DB_Resource} row, or `null` if none match.
|
|
12
|
+
*/
|
|
13
|
+
export async function getResourceByUrl(knex, urls) {
|
|
14
|
+
const row = await buildResourceQuery(knex)
|
|
15
|
+
.whereIn('ur.url', [...urls])
|
|
16
|
+
.first();
|
|
17
|
+
if (!row) {
|
|
18
|
+
return null;
|
|
19
|
+
}
|
|
20
|
+
const [reconstructed] = await reconstructResourceRows(knex, [row]);
|
|
21
|
+
return reconstructed ?? null;
|
|
22
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { Knex } from 'knex';
|
|
2
|
+
/**
|
|
3
|
+
* Retrieves a flat list of all resource URLs from the `resource_items`
|
|
4
|
+
* table. URL text is normalised into `url_refs`, so the read joins the
|
|
5
|
+
* two tables and returns the resolved strings.
|
|
6
|
+
*
|
|
7
|
+
* Read in `resource_items.id`-keyset chunks rather than a single SELECT
|
|
8
|
+
* (issue #294) via `keysetPaginateById` — see that function's JSDoc for
|
|
9
|
+
* why (progress observability on a resource-heavy archive, not memory
|
|
10
|
+
* bounding).
|
|
11
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
12
|
+
* @param onProgress - Called after each chunk with the `resource_items.id`
|
|
13
|
+
* scanned up to so far and the max id. Omit for no reporting (the
|
|
14
|
+
* default; e.g. tests).
|
|
15
|
+
* @returns An array of resource URL strings.
|
|
16
|
+
*/
|
|
17
|
+
export declare function getResourceUrlList(knex: Knex, onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<string[]>;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { keysetPaginateById } from '../_shared/keyset-paginate-by-id.js';
|
|
2
|
+
/** Rows read per `resource_items.id` keyset chunk. */
|
|
3
|
+
const READ_CHUNK_SIZE = 2000;
|
|
4
|
+
/**
|
|
5
|
+
* Retrieves a flat list of all resource URLs from the `resource_items`
|
|
6
|
+
* table. URL text is normalised into `url_refs`, so the read joins the
|
|
7
|
+
* two tables and returns the resolved strings.
|
|
8
|
+
*
|
|
9
|
+
* Read in `resource_items.id`-keyset chunks rather than a single SELECT
|
|
10
|
+
* (issue #294) via `keysetPaginateById` — see that function's JSDoc for
|
|
11
|
+
* why (progress observability on a resource-heavy archive, not memory
|
|
12
|
+
* bounding).
|
|
13
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
14
|
+
* @param onProgress - Called after each chunk with the `resource_items.id`
|
|
15
|
+
* scanned up to so far and the max id. Omit for no reporting (the
|
|
16
|
+
* default; e.g. tests).
|
|
17
|
+
* @returns An array of resource URL strings.
|
|
18
|
+
*/
|
|
19
|
+
export async function getResourceUrlList(knex, onProgress) {
|
|
20
|
+
return keysetPaginateById(knex, 'resource_items', (lastId) => knex('resource_items')
|
|
21
|
+
.join('url_refs', 'url_refs.id', 'resource_items.url_id')
|
|
22
|
+
.where('resource_items.id', '>', lastId)
|
|
23
|
+
.orderBy('resource_items.id', 'asc')
|
|
24
|
+
.limit(READ_CHUNK_SIZE)
|
|
25
|
+
.select('resource_items.id as id', 'url_refs.url as url'), (row) => row.url, onProgress);
|
|
26
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { DB_Resource } from '../../types.js';
|
|
2
|
+
import type { Knex } from 'knex';
|
|
3
|
+
/**
|
|
4
|
+
* Retrieves all sub-resources.
|
|
5
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
6
|
+
* @returns An array of reconstructed {@link DB_Resource} rows.
|
|
7
|
+
*/
|
|
8
|
+
export declare function getResources(knex: Knex): Promise<DB_Resource[]>;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { buildResourceQuery } from './build-resource-query.js';
|
|
2
|
+
import { reconstructResourceRows } from './reconstruct-resource-rows.js';
|
|
3
|
+
/**
|
|
4
|
+
* Retrieves all sub-resources.
|
|
5
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
6
|
+
* @returns An array of reconstructed {@link DB_Resource} rows.
|
|
7
|
+
*/
|
|
8
|
+
export async function getResources(knex) {
|
|
9
|
+
const rows = await buildResourceQuery(knex);
|
|
10
|
+
return reconstructResourceRows(knex, rows);
|
|
11
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { WriteRefCaches } from '../_shared/types.js';
|
|
2
|
+
import type { Knex } from 'knex';
|
|
3
|
+
/**
|
|
4
|
+
* Pre-insert inventory non-HTML URLs into `resource_items` as placeholder
|
|
5
|
+
* rows with `source = 'inventory-seed'` and all metadata columns NULL —
|
|
6
|
+
* the non-HTML counterpart of `insertInventorySeeds`. Used by
|
|
7
|
+
* `CrawlerOrchestrator.inventory` so the ingestion phase commits all of
|
|
8
|
+
* its non-HTML URLs in one chunked round-trip per 500 instead of N
|
|
9
|
+
* sequential `insertResource` awaits — a per-URL loop would spend
|
|
10
|
+
* minutes inside the `.bak`-protected window on a 50k-URL inventory
|
|
11
|
+
* list, where the bulk path finishes in seconds.
|
|
12
|
+
*
|
|
13
|
+
* Idempotent: both the `url_refs` and `resource_items` inserts are
|
|
14
|
+
* `ON CONFLICT ... IGNORE`, leaving existing rows untouched (the
|
|
15
|
+
* orchestrator's `getExistingResourceUrls` filter is what keeps a
|
|
16
|
+
* crawled-lineage `resource_items` row from being downgraded to the
|
|
17
|
+
* inventory label here).
|
|
18
|
+
*
|
|
19
|
+
* Chunked at 500 to stay well under SQLite's `SQLITE_MAX_VARIABLE_NUMBER`.
|
|
20
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
21
|
+
* @param caches
|
|
22
|
+
* @param urls - URL strings (already in `withoutHashAndAuth` form).
|
|
23
|
+
*/
|
|
24
|
+
export declare function insertInventoryResources(knex: Knex, caches: WriteRefCaches, urls: readonly string[]): Promise<void>;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { resolveUrlRefs } from '../../populate-entity-tables/resolve-url-refs.js';
|
|
2
|
+
import { decomposeUrl } from '../../populate-ref-tables/decompose-url.js';
|
|
3
|
+
import { eachSplitted } from '../../utils/array/each-splitted.js';
|
|
4
|
+
/**
|
|
5
|
+
* Pre-insert inventory non-HTML URLs into `resource_items` as placeholder
|
|
6
|
+
* rows with `source = 'inventory-seed'` and all metadata columns NULL —
|
|
7
|
+
* the non-HTML counterpart of `insertInventorySeeds`. Used by
|
|
8
|
+
* `CrawlerOrchestrator.inventory` so the ingestion phase commits all of
|
|
9
|
+
* its non-HTML URLs in one chunked round-trip per 500 instead of N
|
|
10
|
+
* sequential `insertResource` awaits — a per-URL loop would spend
|
|
11
|
+
* minutes inside the `.bak`-protected window on a 50k-URL inventory
|
|
12
|
+
* list, where the bulk path finishes in seconds.
|
|
13
|
+
*
|
|
14
|
+
* Idempotent: both the `url_refs` and `resource_items` inserts are
|
|
15
|
+
* `ON CONFLICT ... IGNORE`, leaving existing rows untouched (the
|
|
16
|
+
* orchestrator's `getExistingResourceUrls` filter is what keeps a
|
|
17
|
+
* crawled-lineage `resource_items` row from being downgraded to the
|
|
18
|
+
* inventory label here).
|
|
19
|
+
*
|
|
20
|
+
* Chunked at 500 to stay well under SQLite's `SQLITE_MAX_VARIABLE_NUMBER`.
|
|
21
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
22
|
+
* @param caches
|
|
23
|
+
* @param urls - URL strings (already in `withoutHashAndAuth` form).
|
|
24
|
+
*/
|
|
25
|
+
export async function insertInventoryResources(knex, caches, urls) {
|
|
26
|
+
if (urls.length === 0) {
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
await eachSplitted([...urls], 500, async (chunk) => {
|
|
30
|
+
await knex('url_refs')
|
|
31
|
+
.insert(chunk.map((url) => ({ url, ...decomposeUrl(url) })))
|
|
32
|
+
.onConflict('url')
|
|
33
|
+
.ignore();
|
|
34
|
+
const urlIds = await resolveUrlRefs(knex, chunk);
|
|
35
|
+
const rows = chunk.map((url) => {
|
|
36
|
+
const urlId = urlIds.get(url);
|
|
37
|
+
if (urlId === undefined) {
|
|
38
|
+
throw new Error(`insertInventoryResources: url_refs.id not resolved for ${url}`);
|
|
39
|
+
}
|
|
40
|
+
caches.urlIds.set(url, urlId);
|
|
41
|
+
return {
|
|
42
|
+
url_id: urlId,
|
|
43
|
+
is_external: 0,
|
|
44
|
+
status: null,
|
|
45
|
+
status_text: null,
|
|
46
|
+
content_type_id: null,
|
|
47
|
+
content_length: null,
|
|
48
|
+
header_set_id: null,
|
|
49
|
+
compress: 0,
|
|
50
|
+
cdn: 0,
|
|
51
|
+
source: 'inventory-seed',
|
|
52
|
+
};
|
|
53
|
+
});
|
|
54
|
+
await knex('resource_items').insert(rows).onConflict('url_id').ignore();
|
|
55
|
+
const inserted = (await knex
|
|
56
|
+
.select('ri.id', 'ur.url')
|
|
57
|
+
.from('resource_items as ri')
|
|
58
|
+
.join('url_refs as ur', 'ur.id', 'ri.url_id')
|
|
59
|
+
.whereIn('ur.url', chunk));
|
|
60
|
+
for (const row of inserted) {
|
|
61
|
+
caches.resourceIds.set(row.url, row.id);
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { WriteRefCaches } from '../_shared/types.js';
|
|
2
|
+
import type { Knex } from 'knex';
|
|
3
|
+
/**
|
|
4
|
+
* Inserts a referrer relationship between a resource and a page into the
|
|
5
|
+
* `resource_ref_edges` table. Silently skips if the resource is not found.
|
|
6
|
+
*
|
|
7
|
+
* A repeat observation of the same `(resource, page)` pair is ignored
|
|
8
|
+
* (`ON CONFLICT ... IGNORE`), leaving `count` at its DEFAULT `1` — one
|
|
9
|
+
* row per pair, matching the legacy uniqueness contract.
|
|
10
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
11
|
+
* @param caches - The connection's write-side id caches.
|
|
12
|
+
* @param src - The URL of the resource.
|
|
13
|
+
* @param pageUrl - The URL of the page that references the resource.
|
|
14
|
+
*/
|
|
15
|
+
export declare function insertResourceReferrers(knex: Knex, caches: WriteRefCaches, src: string, pageUrl: string): Promise<void>;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { resolveContentItemId } from '../_shared/resolve-content-item-id.js';
|
|
2
|
+
/**
|
|
3
|
+
* Inserts a referrer relationship between a resource and a page into the
|
|
4
|
+
* `resource_ref_edges` table. Silently skips if the resource is not found.
|
|
5
|
+
*
|
|
6
|
+
* A repeat observation of the same `(resource, page)` pair is ignored
|
|
7
|
+
* (`ON CONFLICT ... IGNORE`), leaving `count` at its DEFAULT `1` — one
|
|
8
|
+
* row per pair, matching the legacy uniqueness contract.
|
|
9
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
10
|
+
* @param caches - The connection's write-side id caches.
|
|
11
|
+
* @param src - The URL of the resource.
|
|
12
|
+
* @param pageUrl - The URL of the page that references the resource.
|
|
13
|
+
*/
|
|
14
|
+
export async function insertResourceReferrers(knex, caches, src, pageUrl) {
|
|
15
|
+
const resourceId = await findResourceIdByUrl(knex, caches, src);
|
|
16
|
+
if (resourceId === undefined) {
|
|
17
|
+
// Ignore when the resource is not found
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
const pageId = await resolveContentItemId(knex, caches, pageUrl);
|
|
21
|
+
await knex('resource_ref_edges')
|
|
22
|
+
.insert({
|
|
23
|
+
resource_id: resourceId,
|
|
24
|
+
page_id: pageId,
|
|
25
|
+
})
|
|
26
|
+
.onConflict(['resource_id', 'page_id'])
|
|
27
|
+
.ignore();
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Cache-first lookup of `resource_items.id` by the resource URL. The
|
|
31
|
+
* cache only ever gains entries for URLs that exist in the table
|
|
32
|
+
* (resource identities are never deleted during a crawl), so a hit is
|
|
33
|
+
* always valid; a miss falls through to a `url_refs` join.
|
|
34
|
+
* @param knex - Knex query builder.
|
|
35
|
+
* @param caches - The connection's write-side id caches.
|
|
36
|
+
* @param url - The resource URL, verbatim as stored.
|
|
37
|
+
* @returns The resource id, or `undefined` when the URL is unknown.
|
|
38
|
+
*/
|
|
39
|
+
async function findResourceIdByUrl(knex, caches, url) {
|
|
40
|
+
const cached = caches.resourceIds.get(url);
|
|
41
|
+
if (cached !== undefined) {
|
|
42
|
+
return cached;
|
|
43
|
+
}
|
|
44
|
+
const [row] = (await knex
|
|
45
|
+
.select('ri.id')
|
|
46
|
+
.from('resource_items as ri')
|
|
47
|
+
.join('url_refs as ur', 'ur.id', 'ri.url_id')
|
|
48
|
+
.where('ur.url', url));
|
|
49
|
+
if (row === undefined) {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
caches.resourceIds.set(url, row.id);
|
|
53
|
+
return row.id;
|
|
54
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { PageSource } from '../../types.js';
|
|
2
|
+
import type { Resource } from '../../utils/types/types.js';
|
|
3
|
+
import type { WriteRefCaches } from '../_shared/types.js';
|
|
4
|
+
import type { Knex } from 'knex';
|
|
5
|
+
/**
|
|
6
|
+
* Inserts a sub-resource into the `resource_items` table, interning its
|
|
7
|
+
* URL / content type / response headers into the ref tables first. A
|
|
8
|
+
* resource's identity URL routes through {@link resolveUrlOrBlob} like an
|
|
9
|
+
* image `src` — a large inline `data:` URI (e.g. a CSS
|
|
10
|
+
* `background-image` sub-resource) lands in `blob_refs` instead of
|
|
11
|
+
* `url_refs`.
|
|
12
|
+
*
|
|
13
|
+
* A conflict on the identity column (`url_id` or `url_blob_id`) is an
|
|
14
|
+
* **upgrade-only merge**, not first-write-wins: the existing row's
|
|
15
|
+
* response fields (`status` / `status_text` / `content_type_id` /
|
|
16
|
+
* `content_length` / `header_set_id` / `compress` / `cdn`) are replaced
|
|
17
|
+
* only when the existing row currently has `status IS NULL` (beholder's
|
|
18
|
+
* signal for "the request never got a response" — see the `Resource`
|
|
19
|
+
* type) AND the new observation has a non-null `status`. A resource that
|
|
20
|
+
* failed on first sight and later succeeds (the exact case
|
|
21
|
+
* `--retry-failed` re-renders a page for) is therefore healed instead of
|
|
22
|
+
* staying frozen at its failure; a resource that already has a real
|
|
23
|
+
* response never gets overwritten by a later transient failure, and two
|
|
24
|
+
* failures in a row are a no-op — neither carries new information.
|
|
25
|
+
*
|
|
26
|
+
* The `source` provenance label is written ONLY on insert; a conflict
|
|
27
|
+
* (upgrade or no-op alike) leaves an existing row's source untouched
|
|
28
|
+
* (this is what makes a second `crawl --inventory` non-destructive).
|
|
29
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
30
|
+
* @param caches - The connection's write-side id caches.
|
|
31
|
+
* @param resource - The resource data to insert.
|
|
32
|
+
* @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
|
|
33
|
+
*/
|
|
34
|
+
export declare function insertResource(knex: Knex, caches: WriteRefCaches, resource: Resource, source?: PageSource): Promise<void>;
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { normalizeContentType } from '../../content-type/normalize-content-type.js';
|
|
2
|
+
import { resolveUrlOrBlob } from '../_shared/resolve-url-or-blob.js';
|
|
3
|
+
import { upsertContentTypeRef } from '../_shared/upsert-content-type-ref.js';
|
|
4
|
+
import { upsertResponseHeaders } from '../_shared/upsert-response-headers.js';
|
|
5
|
+
/**
|
|
6
|
+
* Inserts a sub-resource into the `resource_items` table, interning its
|
|
7
|
+
* URL / content type / response headers into the ref tables first. A
|
|
8
|
+
* resource's identity URL routes through {@link resolveUrlOrBlob} like an
|
|
9
|
+
* image `src` — a large inline `data:` URI (e.g. a CSS
|
|
10
|
+
* `background-image` sub-resource) lands in `blob_refs` instead of
|
|
11
|
+
* `url_refs`.
|
|
12
|
+
*
|
|
13
|
+
* A conflict on the identity column (`url_id` or `url_blob_id`) is an
|
|
14
|
+
* **upgrade-only merge**, not first-write-wins: the existing row's
|
|
15
|
+
* response fields (`status` / `status_text` / `content_type_id` /
|
|
16
|
+
* `content_length` / `header_set_id` / `compress` / `cdn`) are replaced
|
|
17
|
+
* only when the existing row currently has `status IS NULL` (beholder's
|
|
18
|
+
* signal for "the request never got a response" — see the `Resource`
|
|
19
|
+
* type) AND the new observation has a non-null `status`. A resource that
|
|
20
|
+
* failed on first sight and later succeeds (the exact case
|
|
21
|
+
* `--retry-failed` re-renders a page for) is therefore healed instead of
|
|
22
|
+
* staying frozen at its failure; a resource that already has a real
|
|
23
|
+
* response never gets overwritten by a later transient failure, and two
|
|
24
|
+
* failures in a row are a no-op — neither carries new information.
|
|
25
|
+
*
|
|
26
|
+
* The `source` provenance label is written ONLY on insert; a conflict
|
|
27
|
+
* (upgrade or no-op alike) leaves an existing row's source untouched
|
|
28
|
+
* (this is what makes a second `crawl --inventory` non-destructive).
|
|
29
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
30
|
+
* @param caches - The connection's write-side id caches.
|
|
31
|
+
* @param resource - The resource data to insert.
|
|
32
|
+
* @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
|
|
33
|
+
*/
|
|
34
|
+
export async function insertResource(knex, caches, resource, source) {
|
|
35
|
+
const slot = await resolveUrlOrBlob(knex, caches, resource.url.href);
|
|
36
|
+
// Canonicalize like the page path (see `insertPage`) so resource
|
|
37
|
+
// content-type filters / dedupe keys are case- and whitespace-stable.
|
|
38
|
+
const contentType = normalizeContentType(resource.contentType);
|
|
39
|
+
const contentTypeId = contentType == null || contentType === ''
|
|
40
|
+
? null
|
|
41
|
+
: await upsertContentTypeRef(knex, caches, contentType);
|
|
42
|
+
const headerSetId = await upsertResponseHeaders(knex, caches, resource.headers);
|
|
43
|
+
const identityColumn = slot.blob === null ? 'url_id' : 'url_blob_id';
|
|
44
|
+
const identityValue = slot.blob === null ? slot.url : slot.blob;
|
|
45
|
+
const responseFields = {
|
|
46
|
+
is_external: resource.isExternal ? 1 : 0,
|
|
47
|
+
status: resource.status,
|
|
48
|
+
status_text: resource.statusText,
|
|
49
|
+
content_type_id: contentTypeId,
|
|
50
|
+
content_length: resource.contentLength,
|
|
51
|
+
header_set_id: headerSetId,
|
|
52
|
+
compress: resource.compress || 0,
|
|
53
|
+
cdn: resource.cdn || 0,
|
|
54
|
+
};
|
|
55
|
+
const existing = await knex('resource_items')
|
|
56
|
+
.select('id', 'status')
|
|
57
|
+
.where(identityColumn, identityValue)
|
|
58
|
+
.first();
|
|
59
|
+
if (existing === undefined) {
|
|
60
|
+
await knex('resource_items').insert({
|
|
61
|
+
url_id: slot.url,
|
|
62
|
+
url_blob_id: slot.blob,
|
|
63
|
+
...responseFields,
|
|
64
|
+
...(source === undefined ? {} : { source }),
|
|
65
|
+
});
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (existing.status === null && resource.status !== null) {
|
|
69
|
+
await knex('resource_items')
|
|
70
|
+
.where('id', existing.id)
|
|
71
|
+
.update(responseFields);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { DB_Resource } from '../../types.js';
|
|
2
|
+
import type { Knex } from 'knex';
|
|
3
|
+
/**
|
|
4
|
+
* Raw row shape produced by {@link ../resources/build-resource-query.js}
|
|
5
|
+
* before `responseHeaders` reconstruction.
|
|
6
|
+
*/
|
|
7
|
+
interface RawResourceRow extends Omit<DB_Resource, 'responseHeaders'> {
|
|
8
|
+
/** `resource_items.header_set_id`, or null when no headers were recorded. */
|
|
9
|
+
headerSetId: number | null;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Reconstructs `responseHeaders` (JSON string) for a batch of raw resource
|
|
13
|
+
* rows. Headers load through
|
|
14
|
+
* {@link ../_shared/load-response-headers-by-set-ids.js} (chunked batch
|
|
15
|
+
* lookup, never N+1) — the same reconstruction the page read path uses,
|
|
16
|
+
* so pages and resources always agree on how a given `header_set_id`
|
|
17
|
+
* reads back.
|
|
18
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
19
|
+
* @param rows - Raw rows from {@link ../resources/build-resource-query.js}.
|
|
20
|
+
* @returns Fully reconstructed `DB_Resource` rows, in the same order as `rows`.
|
|
21
|
+
* @example
|
|
22
|
+
* const raw = await buildResourceQuery(knex).whereIn('ur.url', urls);
|
|
23
|
+
* const resources = await reconstructResourceRows(knex, raw);
|
|
24
|
+
*/
|
|
25
|
+
export declare function reconstructResourceRows(knex: Knex, rows: readonly RawResourceRow[]): Promise<DB_Resource[]>;
|
|
26
|
+
export {};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { loadResponseHeadersBySetIds } from '../_shared/load-response-headers-by-set-ids.js';
|
|
2
|
+
/**
|
|
3
|
+
* Reconstructs `responseHeaders` (JSON string) for a batch of raw resource
|
|
4
|
+
* rows. Headers load through
|
|
5
|
+
* {@link ../_shared/load-response-headers-by-set-ids.js} (chunked batch
|
|
6
|
+
* lookup, never N+1) — the same reconstruction the page read path uses,
|
|
7
|
+
* so pages and resources always agree on how a given `header_set_id`
|
|
8
|
+
* reads back.
|
|
9
|
+
* @param knex - Knex query builder connected to the archive DB.
|
|
10
|
+
* @param rows - Raw rows from {@link ../resources/build-resource-query.js}.
|
|
11
|
+
* @returns Fully reconstructed `DB_Resource` rows, in the same order as `rows`.
|
|
12
|
+
* @example
|
|
13
|
+
* const raw = await buildResourceQuery(knex).whereIn('ur.url', urls);
|
|
14
|
+
* const resources = await reconstructResourceRows(knex, raw);
|
|
15
|
+
*/
|
|
16
|
+
export async function reconstructResourceRows(knex, rows) {
|
|
17
|
+
const headerSetIds = [
|
|
18
|
+
...new Set(rows.map((r) => r.headerSetId).filter((id) => id != null)),
|
|
19
|
+
];
|
|
20
|
+
const headersBySetId = await loadResponseHeadersBySetIds(knex, headerSetIds);
|
|
21
|
+
return rows.map((row) => {
|
|
22
|
+
const { headerSetId, ...rest } = row;
|
|
23
|
+
return {
|
|
24
|
+
...rest,
|
|
25
|
+
responseHeaders: headerSetId == null
|
|
26
|
+
? null
|
|
27
|
+
: JSON.stringify(headersBySetId.get(headerSetId) ?? {}),
|
|
28
|
+
};
|
|
29
|
+
});
|
|
30
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { AssignTemplateLabelsParams, TemplateLabel } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Assigns every cluster of a fresh template classification its
|
|
4
|
+
* {@link TemplateLabel}, carrying labels forward from the previous
|
|
5
|
+
* classification wherever the same template survived the re-run.
|
|
6
|
+
*
|
|
7
|
+
* ## Inheritance
|
|
8
|
+
*
|
|
9
|
+
* A new cluster inherits an old cluster's label when the two share a
|
|
10
|
+
* **mutual majority** of pages: the overlap is more than half of the new
|
|
11
|
+
* cluster *and* more than half of the old one. Mutual majority means at
|
|
12
|
+
* most one new cluster can claim a given old label, and a cluster that
|
|
13
|
+
* merely absorbed a few pages of a much larger old template does not steal
|
|
14
|
+
* its name. A split keeps the old label on the half that holds most of the
|
|
15
|
+
* old pages; the other half is named afresh.
|
|
16
|
+
*
|
|
17
|
+
* The old cluster's size counts only pages that are classified in the new
|
|
18
|
+
* run too. Pages the crawl has since dropped (`--recrawl` turned them into
|
|
19
|
+
* redirects, errors, …) are not evidence that the template changed, so
|
|
20
|
+
* they must not dilute the old half of the majority — otherwise an old
|
|
21
|
+
* 7-page cluster whose 4 missing pages left 3 unchanged ones would fail
|
|
22
|
+
* `3 > 7/2` and be renamed for no reason.
|
|
23
|
+
*
|
|
24
|
+
* Page ids are the identity used for overlap (not URLs) because they are
|
|
25
|
+
* what `page_templates` stores and they survive `--append` / `--recrawl`
|
|
26
|
+
* without renumbering.
|
|
27
|
+
*
|
|
28
|
+
* ## Fresh labels
|
|
29
|
+
*
|
|
30
|
+
* Clusters left without an inherited label are numbered in page-count
|
|
31
|
+
* order (largest first, template key as a tie-break) so a first run reads
|
|
32
|
+
* naturally: `A` is the section's biggest template. Each takes the next
|
|
33
|
+
* ordinal **after every ordinal in `previousLabels` for its section**, so a
|
|
34
|
+
* letter is never re-issued to a different template — which would silently
|
|
35
|
+
* change what "events template B" means between two people's archives.
|
|
36
|
+
* That guarantee is only as good as `previousLabels`: `replacePageTemplates`
|
|
37
|
+
* keeps the rows of clusters that disappeared (retired labels) precisely so
|
|
38
|
+
* they stay in this set across any number of later runs.
|
|
39
|
+
*
|
|
40
|
+
* ## Section
|
|
41
|
+
*
|
|
42
|
+
* A cluster's section is the first path segment every member URL shares
|
|
43
|
+
* (`events` for `/events/...`). Members spread over several segments, or
|
|
44
|
+
* all at the site root, yield `null`: such a cluster is numbered in the
|
|
45
|
+
* site-wide sequence. An inherited label keeps its old section even if the
|
|
46
|
+
* cluster's members have since drifted — stability of the name wins over
|
|
47
|
+
* keeping the section literally accurate.
|
|
48
|
+
* @param params - See {@link AssignTemplateLabelsParams}.
|
|
49
|
+
* @returns Template key → label, one entry per cluster in `params.clusters`.
|
|
50
|
+
* @example
|
|
51
|
+
* ```ts
|
|
52
|
+
* const labels = assignTemplateLabels({
|
|
53
|
+
* clusters: new Map([['["path:events","cluster:0"]', { pageIds: [1, 2], urls: ['https://example.com/events/a', 'https://example.com/events/b'] }]]),
|
|
54
|
+
* previousMembership: new Map(),
|
|
55
|
+
* previousLabels: new Map(),
|
|
56
|
+
* });
|
|
57
|
+
* labels.get('["path:events","cluster:0"]'); // { section: 'events', ordinal: 1 }
|
|
58
|
+
* ```
|
|
59
|
+
*/
|
|
60
|
+
export declare function assignTemplateLabels(params: AssignTemplateLabelsParams): Map<string, TemplateLabel>;
|