@nitpicker/crawler 0.9.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/lib/archive/archive-accessor.d.ts +87 -14
  2. package/lib/archive/archive-accessor.js +162 -36
  3. package/lib/archive/archive.d.ts +262 -24
  4. package/lib/archive/archive.js +397 -86
  5. package/lib/archive/cache/compute-archive-cache-key.d.ts +39 -0
  6. package/lib/archive/cache/compute-archive-cache-key.js +95 -0
  7. package/lib/archive/cache/extract-archive-to-cache.d.ts +43 -0
  8. package/lib/archive/cache/extract-archive-to-cache.js +309 -0
  9. package/lib/archive/cache/get-archive-cache-root.d.ts +20 -0
  10. package/lib/archive/cache/get-archive-cache-root.js +53 -0
  11. package/lib/archive/cache/is-archive-cache-disabled.d.ts +24 -0
  12. package/lib/archive/cache/is-archive-cache-disabled.js +34 -0
  13. package/lib/archive/cache/resolve-archive-cache-dir.d.ts +26 -0
  14. package/lib/archive/cache/resolve-archive-cache-dir.js +32 -0
  15. package/lib/archive/database.d.ts +411 -26
  16. package/lib/archive/database.js +1881 -540
  17. package/lib/archive/derive-lineage-from-parent.d.ts +37 -0
  18. package/lib/archive/derive-lineage-from-parent.js +42 -0
  19. package/lib/archive/filesystem/peek-tar-top-dir.d.ts +28 -0
  20. package/lib/archive/filesystem/peek-tar-top-dir.js +65 -0
  21. package/lib/archive/get-failed-page-messages.d.ts +43 -0
  22. package/lib/archive/get-failed-page-messages.js +131 -0
  23. package/lib/archive/init-schema.d.ts +53 -2
  24. package/lib/archive/init-schema.js +399 -15
  25. package/lib/archive/is-inventory-source.d.ts +21 -0
  26. package/lib/archive/is-inventory-source.js +22 -0
  27. package/lib/archive/meta/assert-compatible-version.d.ts +39 -0
  28. package/lib/archive/meta/assert-compatible-version.js +72 -0
  29. package/lib/archive/meta/classify-jsonld-type.d.ts +23 -0
  30. package/lib/archive/meta/classify-jsonld-type.js +43 -0
  31. package/lib/archive/meta/compare-semver.d.ts +23 -0
  32. package/lib/archive/meta/compare-semver.js +51 -0
  33. package/lib/archive/meta/compute-page-denormalized.d.ts +21 -0
  34. package/lib/archive/meta/compute-page-denormalized.js +35 -0
  35. package/lib/archive/meta/derive-flat-from-meta.d.ts +35 -0
  36. package/lib/archive/meta/derive-flat-from-meta.js +158 -0
  37. package/lib/archive/meta/derive-meta-extras.d.ts +20 -0
  38. package/lib/archive/meta/derive-meta-extras.js +23 -0
  39. package/lib/archive/meta/extract-tags-for-archive.d.ts +18 -0
  40. package/lib/archive/meta/extract-tags-for-archive.js +36 -0
  41. package/lib/archive/meta/summarize-jsonld.d.ts +17 -0
  42. package/lib/archive/meta/summarize-jsonld.js +29 -0
  43. package/lib/archive/meta/summarize-tags.d.ts +16 -0
  44. package/lib/archive/meta/summarize-tags.js +33 -0
  45. package/lib/archive/meta/types.d.ts +207 -0
  46. package/lib/archive/meta/types.js +33 -0
  47. package/lib/archive/migrate-crawl-errors.d.ts +20 -0
  48. package/lib/archive/migrate-crawl-errors.js +38 -0
  49. package/lib/archive/migrate-html-blob-tables.d.ts +24 -0
  50. package/lib/archive/migrate-html-blob-tables.js +53 -0
  51. package/lib/archive/migrate-inventory-runs.d.ts +29 -0
  52. package/lib/archive/migrate-inventory-runs.js +52 -0
  53. package/lib/archive/migrate-page-errors.d.ts +16 -0
  54. package/lib/archive/migrate-page-errors.js +35 -0
  55. package/lib/archive/migrate-pages-resources-source.d.ts +16 -0
  56. package/lib/archive/migrate-pages-resources-source.js +46 -0
  57. package/lib/archive/page.d.ts +187 -49
  58. package/lib/archive/page.js +258 -63
  59. package/lib/archive/peek-archive-lock.d.ts +40 -0
  60. package/lib/archive/peek-archive-lock.js +62 -0
  61. package/lib/archive/resolve-redirect-chain.d.ts +33 -0
  62. package/lib/archive/resolve-redirect-chain.js +27 -0
  63. package/lib/archive/types.d.ts +168 -26
  64. package/lib/classify-error-kind.d.ts +19 -0
  65. package/lib/classify-error-kind.js +122 -0
  66. package/lib/crawler/build-js-redirect-edge.d.ts +68 -0
  67. package/lib/crawler/build-js-redirect-edge.js +57 -0
  68. package/lib/crawler/build-redirect-event.d.ts +24 -0
  69. package/lib/crawler/build-redirect-event.js +28 -0
  70. package/lib/crawler/clear-dns-burned-host-cache.d.ts +6 -0
  71. package/lib/crawler/clear-dns-burned-host-cache.js +11 -0
  72. package/lib/crawler/close-browser-safely.d.ts +64 -0
  73. package/lib/crawler/close-browser-safely.js +73 -0
  74. package/lib/crawler/crawler.d.ts +7 -2
  75. package/lib/crawler/crawler.js +919 -113
  76. package/lib/crawler/create-change-phase-handler.d.ts +54 -0
  77. package/lib/crawler/create-change-phase-handler.js +44 -0
  78. package/lib/crawler/derive-js-redirect-target.d.ts +68 -0
  79. package/lib/crawler/derive-js-redirect-target.js +129 -0
  80. package/lib/crawler/derive-page-source.d.ts +23 -0
  81. package/lib/crawler/derive-page-source.js +28 -0
  82. package/lib/crawler/derive-resource-source.d.ts +33 -0
  83. package/lib/crawler/derive-resource-source.js +37 -0
  84. package/lib/crawler/dns-burned-host-cache.d.ts +26 -0
  85. package/lib/crawler/dns-burned-host-cache.js +25 -0
  86. package/lib/crawler/dns-burned-host-short-circuit-counter.d.ts +13 -0
  87. package/lib/crawler/dns-burned-host-short-circuit-counter.js +11 -0
  88. package/lib/crawler/drain-phase-errors.d.ts +48 -0
  89. package/lib/crawler/drain-phase-errors.js +35 -0
  90. package/lib/crawler/fetch-destination.d.ts +12 -4
  91. package/lib/crawler/fetch-destination.js +132 -18
  92. package/lib/crawler/format-crawl-progress.d.ts +12 -3
  93. package/lib/crawler/format-crawl-progress.js +14 -6
  94. package/lib/crawler/handle-browser-close.d.ts +29 -0
  95. package/lib/crawler/handle-browser-close.js +28 -0
  96. package/lib/crawler/is-html-content-type.d.ts +17 -0
  97. package/lib/crawler/is-html-content-type.js +19 -0
  98. package/lib/crawler/is-js-redirect-error-shape.d.ts +40 -0
  99. package/lib/crawler/is-js-redirect-error-shape.js +53 -0
  100. package/lib/crawler/is-likely-html-url.d.ts +22 -0
  101. package/lib/crawler/is-likely-html-url.js +65 -0
  102. package/lib/crawler/is-puppeteer-fallback-candidate.d.ts +16 -0
  103. package/lib/crawler/is-puppeteer-fallback-candidate.js +63 -0
  104. package/lib/crawler/kill-process-tree.d.ts +94 -0
  105. package/lib/crawler/kill-process-tree.js +178 -0
  106. package/lib/crawler/link-list.d.ts +21 -1
  107. package/lib/crawler/link-list.js +25 -4
  108. package/lib/crawler/link-to-page-data.d.ts +13 -5
  109. package/lib/crawler/link-to-page-data.js +26 -5
  110. package/lib/crawler/log-undrained-phase-errors.d.ts +37 -0
  111. package/lib/crawler/log-undrained-phase-errors.js +34 -0
  112. package/lib/crawler/normalize-content-type.d.ts +14 -0
  113. package/lib/crawler/normalize-content-type.js +20 -0
  114. package/lib/crawler/partition-urls-by-html.d.ts +16 -0
  115. package/lib/crawler/partition-urls-by-html.js +23 -0
  116. package/lib/crawler/plan-sub-resource-emits.d.ts +63 -0
  117. package/lib/crawler/plan-sub-resource-emits.js +44 -0
  118. package/lib/crawler/preload-short-circuit-error.d.ts +22 -0
  119. package/lib/crawler/preload-short-circuit-error.js +25 -0
  120. package/lib/crawler/redirect-dest-key.d.ts +19 -0
  121. package/lib/crawler/redirect-dest-key.js +27 -0
  122. package/lib/crawler/resource-to-page-data.d.ts +28 -0
  123. package/lib/crawler/resource-to-page-data.js +59 -0
  124. package/lib/crawler/should-burn-host.d.ts +78 -0
  125. package/lib/crawler/should-burn-host.js +61 -0
  126. package/lib/crawler/should-get-fallback-on-head-failure.d.ts +38 -0
  127. package/lib/crawler/should-get-fallback-on-head-failure.js +46 -0
  128. package/lib/crawler/types.d.ts +229 -1
  129. package/lib/crawler-orchestrator.d.ts +105 -3
  130. package/lib/crawler-orchestrator.js +620 -20
  131. package/lib/crawler.d.ts +8 -2
  132. package/lib/crawler.js +6 -1
  133. package/lib/permanent-error-kinds.d.ts +43 -0
  134. package/lib/permanent-error-kinds.js +48 -0
  135. package/lib/resource-row-to-lookup-result.d.ts +13 -0
  136. package/lib/resource-row-to-lookup-result.js +20 -0
  137. package/lib/types.d.ts +95 -1
  138. package/lib/utils/compute-file-sha256.d.ts +23 -0
  139. package/lib/utils/compute-file-sha256.js +55 -0
  140. package/lib/utils/error/emit-error-with-retry.d.ts +40 -0
  141. package/lib/utils/error/emit-error-with-retry.js +44 -0
  142. package/lib/utils/error/emit-error.d.ts +39 -0
  143. package/lib/utils/error/emit-error.js +41 -0
  144. package/lib/utils/object/parse-response-headers.d.ts +12 -0
  145. package/lib/utils/object/parse-response-headers.js +26 -0
  146. package/package.json +11 -11
  147. package/lib/utils/error/error-emitter.d.ts +0 -18
  148. package/lib/utils/error/error-emitter.js +0 -29
@@ -1,29 +0,0 @@
1
- import { log } from '../debug.js';
2
- const errorLog = log.extend('ErrorEmitter');
3
- /**
4
- * A class method decorator factory that wraps the decorated method with error handling.
5
- * When the method throws an `Error`, it emits an `'error'` event on the class instance
6
- * (which must extend {@link EventEmitter}) with the caught error, then re-throws the error.
7
- * @template C - The class type, which must be an EventEmitter capable of emitting error events.
8
- * @template E - The error event type, defaults to {@link ErrorEvent}.
9
- * @returns A decorator function that wraps the target method with error-emitting behavior.
10
- */
11
- export function ErrorEmitter() {
12
- // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
13
- return (method, context) => {
14
- return async function (...args) {
15
- const constructorName = String(this.constructor?.name || this.constructor || this);
16
- const methodName = `${constructorName}.${String(context.name)}`;
17
- try {
18
- return await method.apply(this, args);
19
- }
20
- catch (error) {
21
- if (error instanceof Error) {
22
- errorLog('%s: %O', methodName, error);
23
- void this.emit('error', error);
24
- }
25
- throw error;
26
- }
27
- };
28
- };
29
- }