@nitpicker/crawler 0.9.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/archive/archive-accessor.d.ts +87 -14
- package/lib/archive/archive-accessor.js +162 -36
- package/lib/archive/archive.d.ts +262 -24
- package/lib/archive/archive.js +397 -86
- package/lib/archive/cache/compute-archive-cache-key.d.ts +39 -0
- package/lib/archive/cache/compute-archive-cache-key.js +95 -0
- package/lib/archive/cache/extract-archive-to-cache.d.ts +43 -0
- package/lib/archive/cache/extract-archive-to-cache.js +309 -0
- package/lib/archive/cache/get-archive-cache-root.d.ts +20 -0
- package/lib/archive/cache/get-archive-cache-root.js +53 -0
- package/lib/archive/cache/is-archive-cache-disabled.d.ts +24 -0
- package/lib/archive/cache/is-archive-cache-disabled.js +34 -0
- package/lib/archive/cache/resolve-archive-cache-dir.d.ts +26 -0
- package/lib/archive/cache/resolve-archive-cache-dir.js +32 -0
- package/lib/archive/database.d.ts +411 -26
- package/lib/archive/database.js +1881 -540
- package/lib/archive/derive-lineage-from-parent.d.ts +37 -0
- package/lib/archive/derive-lineage-from-parent.js +42 -0
- package/lib/archive/filesystem/peek-tar-top-dir.d.ts +28 -0
- package/lib/archive/filesystem/peek-tar-top-dir.js +65 -0
- package/lib/archive/get-failed-page-messages.d.ts +43 -0
- package/lib/archive/get-failed-page-messages.js +131 -0
- package/lib/archive/init-schema.d.ts +53 -2
- package/lib/archive/init-schema.js +399 -15
- package/lib/archive/is-inventory-source.d.ts +21 -0
- package/lib/archive/is-inventory-source.js +22 -0
- package/lib/archive/meta/assert-compatible-version.d.ts +39 -0
- package/lib/archive/meta/assert-compatible-version.js +72 -0
- package/lib/archive/meta/classify-jsonld-type.d.ts +23 -0
- package/lib/archive/meta/classify-jsonld-type.js +43 -0
- package/lib/archive/meta/compare-semver.d.ts +23 -0
- package/lib/archive/meta/compare-semver.js +51 -0
- package/lib/archive/meta/compute-page-denormalized.d.ts +21 -0
- package/lib/archive/meta/compute-page-denormalized.js +35 -0
- package/lib/archive/meta/derive-flat-from-meta.d.ts +35 -0
- package/lib/archive/meta/derive-flat-from-meta.js +158 -0
- package/lib/archive/meta/derive-meta-extras.d.ts +20 -0
- package/lib/archive/meta/derive-meta-extras.js +23 -0
- package/lib/archive/meta/extract-tags-for-archive.d.ts +18 -0
- package/lib/archive/meta/extract-tags-for-archive.js +36 -0
- package/lib/archive/meta/summarize-jsonld.d.ts +17 -0
- package/lib/archive/meta/summarize-jsonld.js +29 -0
- package/lib/archive/meta/summarize-tags.d.ts +16 -0
- package/lib/archive/meta/summarize-tags.js +33 -0
- package/lib/archive/meta/types.d.ts +207 -0
- package/lib/archive/meta/types.js +33 -0
- package/lib/archive/migrate-crawl-errors.d.ts +20 -0
- package/lib/archive/migrate-crawl-errors.js +38 -0
- package/lib/archive/migrate-html-blob-tables.d.ts +24 -0
- package/lib/archive/migrate-html-blob-tables.js +53 -0
- package/lib/archive/migrate-inventory-runs.d.ts +29 -0
- package/lib/archive/migrate-inventory-runs.js +52 -0
- package/lib/archive/migrate-page-errors.d.ts +16 -0
- package/lib/archive/migrate-page-errors.js +35 -0
- package/lib/archive/migrate-pages-resources-source.d.ts +16 -0
- package/lib/archive/migrate-pages-resources-source.js +46 -0
- package/lib/archive/page.d.ts +187 -49
- package/lib/archive/page.js +258 -63
- package/lib/archive/peek-archive-lock.d.ts +40 -0
- package/lib/archive/peek-archive-lock.js +62 -0
- package/lib/archive/resolve-redirect-chain.d.ts +33 -0
- package/lib/archive/resolve-redirect-chain.js +27 -0
- package/lib/archive/types.d.ts +168 -26
- package/lib/classify-error-kind.d.ts +19 -0
- package/lib/classify-error-kind.js +122 -0
- package/lib/crawler/build-js-redirect-edge.d.ts +68 -0
- package/lib/crawler/build-js-redirect-edge.js +57 -0
- package/lib/crawler/build-redirect-event.d.ts +24 -0
- package/lib/crawler/build-redirect-event.js +28 -0
- package/lib/crawler/clear-dns-burned-host-cache.d.ts +6 -0
- package/lib/crawler/clear-dns-burned-host-cache.js +11 -0
- package/lib/crawler/close-browser-safely.d.ts +64 -0
- package/lib/crawler/close-browser-safely.js +73 -0
- package/lib/crawler/crawler.d.ts +7 -2
- package/lib/crawler/crawler.js +919 -113
- package/lib/crawler/create-change-phase-handler.d.ts +54 -0
- package/lib/crawler/create-change-phase-handler.js +44 -0
- package/lib/crawler/derive-js-redirect-target.d.ts +68 -0
- package/lib/crawler/derive-js-redirect-target.js +129 -0
- package/lib/crawler/derive-page-source.d.ts +23 -0
- package/lib/crawler/derive-page-source.js +28 -0
- package/lib/crawler/derive-resource-source.d.ts +33 -0
- package/lib/crawler/derive-resource-source.js +37 -0
- package/lib/crawler/dns-burned-host-cache.d.ts +26 -0
- package/lib/crawler/dns-burned-host-cache.js +25 -0
- package/lib/crawler/dns-burned-host-short-circuit-counter.d.ts +13 -0
- package/lib/crawler/dns-burned-host-short-circuit-counter.js +11 -0
- package/lib/crawler/drain-phase-errors.d.ts +48 -0
- package/lib/crawler/drain-phase-errors.js +35 -0
- package/lib/crawler/fetch-destination.d.ts +12 -4
- package/lib/crawler/fetch-destination.js +132 -18
- package/lib/crawler/format-crawl-progress.d.ts +12 -3
- package/lib/crawler/format-crawl-progress.js +14 -6
- package/lib/crawler/handle-browser-close.d.ts +29 -0
- package/lib/crawler/handle-browser-close.js +28 -0
- package/lib/crawler/is-html-content-type.d.ts +17 -0
- package/lib/crawler/is-html-content-type.js +19 -0
- package/lib/crawler/is-js-redirect-error-shape.d.ts +40 -0
- package/lib/crawler/is-js-redirect-error-shape.js +53 -0
- package/lib/crawler/is-likely-html-url.d.ts +22 -0
- package/lib/crawler/is-likely-html-url.js +65 -0
- package/lib/crawler/is-puppeteer-fallback-candidate.d.ts +16 -0
- package/lib/crawler/is-puppeteer-fallback-candidate.js +63 -0
- package/lib/crawler/kill-process-tree.d.ts +94 -0
- package/lib/crawler/kill-process-tree.js +178 -0
- package/lib/crawler/link-list.d.ts +21 -1
- package/lib/crawler/link-list.js +25 -4
- package/lib/crawler/link-to-page-data.d.ts +13 -5
- package/lib/crawler/link-to-page-data.js +26 -5
- package/lib/crawler/log-undrained-phase-errors.d.ts +37 -0
- package/lib/crawler/log-undrained-phase-errors.js +34 -0
- package/lib/crawler/normalize-content-type.d.ts +14 -0
- package/lib/crawler/normalize-content-type.js +20 -0
- package/lib/crawler/partition-urls-by-html.d.ts +16 -0
- package/lib/crawler/partition-urls-by-html.js +23 -0
- package/lib/crawler/plan-sub-resource-emits.d.ts +63 -0
- package/lib/crawler/plan-sub-resource-emits.js +44 -0
- package/lib/crawler/preload-short-circuit-error.d.ts +22 -0
- package/lib/crawler/preload-short-circuit-error.js +25 -0
- package/lib/crawler/redirect-dest-key.d.ts +19 -0
- package/lib/crawler/redirect-dest-key.js +27 -0
- package/lib/crawler/resource-to-page-data.d.ts +28 -0
- package/lib/crawler/resource-to-page-data.js +59 -0
- package/lib/crawler/should-burn-host.d.ts +78 -0
- package/lib/crawler/should-burn-host.js +61 -0
- package/lib/crawler/should-get-fallback-on-head-failure.d.ts +38 -0
- package/lib/crawler/should-get-fallback-on-head-failure.js +46 -0
- package/lib/crawler/types.d.ts +229 -1
- package/lib/crawler-orchestrator.d.ts +105 -3
- package/lib/crawler-orchestrator.js +620 -20
- package/lib/crawler.d.ts +8 -2
- package/lib/crawler.js +6 -1
- package/lib/permanent-error-kinds.d.ts +43 -0
- package/lib/permanent-error-kinds.js +48 -0
- package/lib/resource-row-to-lookup-result.d.ts +13 -0
- package/lib/resource-row-to-lookup-result.js +20 -0
- package/lib/types.d.ts +95 -1
- package/lib/utils/compute-file-sha256.d.ts +23 -0
- package/lib/utils/compute-file-sha256.js +55 -0
- package/lib/utils/error/emit-error-with-retry.d.ts +40 -0
- package/lib/utils/error/emit-error-with-retry.js +44 -0
- package/lib/utils/error/emit-error.d.ts +39 -0
- package/lib/utils/error/emit-error.js +41 -0
- package/lib/utils/object/parse-response-headers.d.ts +12 -0
- package/lib/utils/object/parse-response-headers.js +26 -0
- package/package.json +11 -11
- package/lib/utils/error/error-emitter.d.ts +0 -18
- package/lib/utils/error/error-emitter.js +0 -29
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import { log } from '../debug.js';
|
|
2
|
-
const errorLog = log.extend('ErrorEmitter');
|
|
3
|
-
/**
|
|
4
|
-
* A class method decorator factory that wraps the decorated method with error handling.
|
|
5
|
-
* When the method throws an `Error`, it emits an `'error'` event on the class instance
|
|
6
|
-
* (which must extend {@link EventEmitter}) with the caught error, then re-throws the error.
|
|
7
|
-
* @template C - The class type, which must be an EventEmitter capable of emitting error events.
|
|
8
|
-
* @template E - The error event type, defaults to {@link ErrorEvent}.
|
|
9
|
-
* @returns A decorator function that wraps the target method with error-emitting behavior.
|
|
10
|
-
*/
|
|
11
|
-
export function ErrorEmitter() {
|
|
12
|
-
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
|
|
13
|
-
return (method, context) => {
|
|
14
|
-
return async function (...args) {
|
|
15
|
-
const constructorName = String(this.constructor?.name || this.constructor || this);
|
|
16
|
-
const methodName = `${constructorName}.${String(context.name)}`;
|
|
17
|
-
try {
|
|
18
|
-
return await method.apply(this, args);
|
|
19
|
-
}
|
|
20
|
-
catch (error) {
|
|
21
|
-
if (error instanceof Error) {
|
|
22
|
-
errorLog('%s: %O', methodName, error);
|
|
23
|
-
void this.emit('error', error);
|
|
24
|
-
}
|
|
25
|
-
throw error;
|
|
26
|
-
}
|
|
27
|
-
};
|
|
28
|
-
};
|
|
29
|
-
}
|