@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +5 -1
  2. package/autoscaling/snapshotter.d.ts.map +1 -1
  3. package/autoscaling/snapshotter.js +4 -11
  4. package/autoscaling/snapshotter.js.map +1 -1
  5. package/configuration.d.ts +0 -7
  6. package/configuration.d.ts.map +1 -1
  7. package/configuration.js +0 -4
  8. package/configuration.js.map +1 -1
  9. package/cookie_utils.d.ts +1 -1
  10. package/cookie_utils.d.ts.map +1 -1
  11. package/cookie_utils.js +4 -6
  12. package/cookie_utils.js.map +1 -1
  13. package/crawlers/context_pipeline.d.ts +61 -0
  14. package/crawlers/context_pipeline.d.ts.map +1 -0
  15. package/crawlers/context_pipeline.js +99 -0
  16. package/crawlers/context_pipeline.js.map +1 -0
  17. package/crawlers/crawler_commons.d.ts +12 -16
  18. package/crawlers/crawler_commons.d.ts.map +1 -1
  19. package/crawlers/crawler_commons.js +0 -8
  20. package/crawlers/crawler_commons.js.map +1 -1
  21. package/crawlers/error_snapshotter.d.ts +3 -2
  22. package/crawlers/error_snapshotter.d.ts.map +1 -1
  23. package/crawlers/error_snapshotter.js +2 -2
  24. package/crawlers/error_snapshotter.js.map +1 -1
  25. package/crawlers/error_tracker.d.ts +2 -1
  26. package/crawlers/error_tracker.d.ts.map +1 -1
  27. package/crawlers/error_tracker.js.map +1 -1
  28. package/crawlers/index.d.ts +1 -1
  29. package/crawlers/index.d.ts.map +1 -1
  30. package/crawlers/index.js +1 -1
  31. package/crawlers/index.js.map +1 -1
  32. package/crawlers/internals/types.d.ts +8 -0
  33. package/crawlers/internals/types.d.ts.map +1 -0
  34. package/crawlers/internals/types.js +2 -0
  35. package/crawlers/internals/types.js.map +1 -0
  36. package/enqueue_links/enqueue_links.d.ts +22 -15
  37. package/enqueue_links/enqueue_links.d.ts.map +1 -1
  38. package/enqueue_links/enqueue_links.js +33 -14
  39. package/enqueue_links/enqueue_links.js.map +1 -1
  40. package/enqueue_links/shared.d.ts +8 -2
  41. package/enqueue_links/shared.d.ts.map +1 -1
  42. package/enqueue_links/shared.js +40 -18
  43. package/enqueue_links/shared.js.map +1 -1
  44. package/errors.d.ts +12 -0
  45. package/errors.d.ts.map +1 -1
  46. package/errors.js +20 -0
  47. package/errors.js.map +1 -1
  48. package/events/local_event_manager.d.ts +0 -2
  49. package/events/local_event_manager.d.ts.map +1 -1
  50. package/events/local_event_manager.js +3 -32
  51. package/events/local_event_manager.js.map +1 -1
  52. package/http_clients/base-http-client.d.ts +26 -69
  53. package/http_clients/base-http-client.d.ts.map +1 -1
  54. package/http_clients/base-http-client.js +20 -9
  55. package/http_clients/base-http-client.js.map +1 -1
  56. package/http_clients/got-scraping-http-client.d.ts +10 -3
  57. package/http_clients/got-scraping-http-client.d.ts.map +1 -1
  58. package/http_clients/got-scraping-http-client.js +86 -39
  59. package/http_clients/got-scraping-http-client.js.map +1 -1
  60. package/index.d.ts +1 -0
  61. package/index.d.ts.map +1 -1
  62. package/index.js +1 -0
  63. package/index.js.map +1 -1
  64. package/log.d.ts +4 -2
  65. package/log.d.ts.map +1 -1
  66. package/log.js.map +1 -1
  67. package/package.json +6 -6
  68. package/proxy_configuration.d.ts +21 -40
  69. package/proxy_configuration.d.ts.map +1 -1
  70. package/proxy_configuration.js +15 -51
  71. package/proxy_configuration.js.map +1 -1
  72. package/recoverable_state.d.ts +123 -0
  73. package/recoverable_state.d.ts.map +1 -0
  74. package/recoverable_state.js +140 -0
  75. package/recoverable_state.js.map +1 -0
  76. package/request.d.ts +31 -5
  77. package/request.d.ts.map +1 -1
  78. package/request.js +35 -12
  79. package/request.js.map +1 -1
  80. package/session_pool/session.d.ts +6 -2
  81. package/session_pool/session.d.ts.map +1 -1
  82. package/session_pool/session.js +9 -2
  83. package/session_pool/session.js.map +1 -1
  84. package/session_pool/session_pool.d.ts +9 -2
  85. package/session_pool/session_pool.d.ts.map +1 -1
  86. package/session_pool/session_pool.js +13 -1
  87. package/session_pool/session_pool.js.map +1 -1
  88. package/storages/dataset.d.ts +7 -0
  89. package/storages/dataset.d.ts.map +1 -1
  90. package/storages/dataset.js +10 -1
  91. package/storages/dataset.js.map +1 -1
  92. package/storages/index.d.ts +2 -0
  93. package/storages/index.d.ts.map +1 -1
  94. package/storages/index.js +2 -0
  95. package/storages/index.js.map +1 -1
  96. package/storages/key_value_store.d.ts +13 -1
  97. package/storages/key_value_store.d.ts.map +1 -1
  98. package/storages/key_value_store.js +11 -6
  99. package/storages/key_value_store.js.map +1 -1
  100. package/storages/request_list.d.ts +0 -6
  101. package/storages/request_list.d.ts.map +1 -1
  102. package/storages/request_list.js.map +1 -1
  103. package/storages/request_list_adapter.d.ts +58 -0
  104. package/storages/request_list_adapter.d.ts.map +1 -0
  105. package/storages/request_list_adapter.js +81 -0
  106. package/storages/request_list_adapter.js.map +1 -0
  107. package/storages/request_manager_tandem.d.ts +68 -0
  108. package/storages/request_manager_tandem.d.ts.map +1 -0
  109. package/storages/request_manager_tandem.js +124 -0
  110. package/storages/request_manager_tandem.js.map +1 -0
  111. package/storages/request_provider.d.ts +68 -4
  112. package/storages/request_provider.d.ts.map +1 -1
  113. package/storages/request_provider.js +74 -44
  114. package/storages/request_provider.js.map +1 -1
  115. package/storages/request_queue_v2.d.ts +2 -2
  116. package/storages/request_queue_v2.d.ts.map +1 -1
  117. package/storages/request_queue_v2.js.map +1 -1
  118. package/storages/sitemap_request_list.d.ts.map +1 -1
  119. package/storages/sitemap_request_list.js.map +1 -1
  120. package/storages/storage_manager.d.ts.map +1 -1
  121. package/storages/storage_manager.js +1 -1
  122. package/storages/storage_manager.js.map +1 -1
  123. package/typedefs.d.ts +1 -1
  124. package/typedefs.d.ts.map +1 -1
  125. package/crawlers/crawler_extension.d.ts +0 -12
  126. package/crawlers/crawler_extension.d.ts.map +0 -1
  127. package/crawlers/crawler_extension.js +0 -14
  128. package/crawlers/crawler_extension.js.map +0 -1
  129. package/tsconfig.build.tsbuildinfo +0 -1
@@ -1,6 +1,7 @@
1
1
  import type { CrawlingContext } from '../crawlers/crawler_commons.js';
2
2
  import type { KeyValueStore } from '../storages/key_value_store.js';
3
3
  import type { ErrnoException } from './error_tracker.js';
4
+ import type { SnapshottableProperties } from './internals/types.js';
4
5
  interface BrowserCrawlingContext {
5
6
  saveSnapshot: (options: {
6
7
  key: string;
@@ -39,7 +40,7 @@ export declare class ErrorSnapshotter {
39
40
  /**
40
41
  * Capture a snapshot of the error context.
41
42
  */
42
- captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise<ErrorSnapshot>;
43
+ captureSnapshot(error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<ErrorSnapshot>;
43
44
  /**
44
45
  * Captures a snapshot of the current page using the context.saveSnapshot function.
45
46
  * This function is applicable for browser contexts only.
@@ -49,7 +50,7 @@ export declare class ErrorSnapshotter {
49
50
  /**
50
51
  * Save the HTML snapshot of the page, and return the fileName with the extension.
51
52
  */
52
- saveHTMLSnapshot(html: string, keyValueStore: KeyValueStore, fileName: string): Promise<string | undefined>;
53
+ saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
53
54
  /**
54
55
  * Generate a unique fileName for each error snapshot.
55
56
  */
@@ -1 +1 @@
1
- {"version":3,"file":"error_snapshotter.d.ts","sourceRoot":"","sources":["../../src/crawlers/error_snapshotter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGzD,UAAU,sBAAsB;IAC5B,YAAY,EAAE,CAAC,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7D;AAMD,MAAM,WAAW,cAAc;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,UAAU,aAAa;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,gBAAgB;IACzB,MAAM,CAAC,QAAQ,CAAC,oBAAoB,MAAM;IAC1C,MAAM,CAAC,QAAQ,CAAC,eAAe,MAAM;IACrC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,OAAO;IAC1C,MAAM,CAAC,QAAQ,CAAC,YAAY,uBAAuB;IACnD,MAAM,CAAC,QAAQ,CAAC,eAAe,oBAAoB;IAEnD;;OAEG;IACG,eAAe,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC;IAgD9F;;;;OAIG;IACG,sBAAsB,CACxB,OAAO,EAAE,sBAAsB,EAC/B,QAAQ,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;IAYtC;;OAEG;IACG,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IASjH;;OAEG;IACH,gBAAgB,CAAC,KAAK,EAAE,cAAc,GAAG,MAAM;CAyBlD"}
1
+ {"version":3,"file":"error_snapshotter.d.ts","sourceRoot":"","sources":["../../src/crawlers/error_snapshotter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAGpE,UAAU,sBAAsB;IAC5B,YAAY,EAAE,CAAC,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7D;AAED,MAAM,WAAW,cAAc;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,UAAU,aAAa;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,gBAAgB;IACzB,MAAM,CAAC,QAAQ,CAAC,oBAAoB,MAAM;IAC1C,MAAM,CAAC,QAAQ,CAAC,eAAe,MAAM;IACrC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,OAAO;IAC1C,MAAM,CAAC,QAAQ,CAAC,YAAY,uBAAuB;IACnD,MAAM,CAAC,QAAQ,CAAC,eAAe,oBAAoB;IAEnD;;OAEG;IACG,eAAe,CACjB,KAAK,EAAE,cAAc,EACrB,OAAO,EAAE,eAAe,GAAG,uBAAuB,GACnD,OAAO,CAAC,aAAa,CAAC;IAgDzB;;;;OAIG;IACG,sBAAsB,CACxB,OAAO,EAAE,sBAAsB,EAC/B,QAAQ,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;IAYtC;;OAEG;IACG,gBAAgB,CAClB,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,UAAU,CAAC,EAC9C,QAAQ,EAAE,MAAM,GACjB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAS9B;;OAEG;IACH,gBAAgB,CAAC,KAAK,EAAE,cAAc,GAAG,MAAM;CAyBlD"}
@@ -52,9 +52,9 @@ export class ErrorSnapshotter {
52
52
  }
53
53
  return {
54
54
  screenshotFileName,
55
- screenshotFileUrl: screenshotFileName && keyValueStore.getPublicUrl(screenshotFileName),
55
+ screenshotFileUrl: screenshotFileName && (await keyValueStore.getPublicUrl(screenshotFileName)),
56
56
  htmlFileName,
57
- htmlFileUrl: htmlFileName && keyValueStore.getPublicUrl(htmlFileName),
57
+ htmlFileUrl: htmlFileName && (await keyValueStore.getPublicUrl(htmlFileName)),
58
58
  };
59
59
  }
60
60
  catch {
@@ -1 +1 @@
1
- {"version":3,"file":"error_snapshotter.js","sourceRoot":"","sources":["../../src/crawlers/error_snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AA2BjC;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,gBAAgB;IACzB,MAAM,CAAU,oBAAoB,GAAG,EAAE,CAAC;IAC1C,MAAM,CAAU,eAAe,GAAG,EAAE,CAAC;IACrC,MAAM,CAAU,mBAAmB,GAAG,GAAG,CAAC;IAC1C,MAAM,CAAU,YAAY,GAAG,mBAAmB,CAAC;IACnD,MAAM,CAAU,eAAe,GAAG,gBAAgB,CAAC;IAEnD;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,KAAqB,EAAE,OAAwB;QACjE,IAAI,CAAC;YACD,MAAM,IAAI,GAAG,OAAO,EAAE,IAA+B,CAAC;YACtD,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,CAAC;YAE3B,MAAM,aAAa,GAAG,MAAM,OAAO,EAAE,gBAAgB,EAAE,CAAC;YACxD,0GAA0G;YAC1G,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrC,OAAO,EAAE,CAAC;YACd,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAE9C,IAAI,kBAAsC,CAAC;YAC3C,IAAI,YAAgC,CAAC;YAErC,IAAI,IAAI,EAAE,CAAC;gBACP,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,sBAAsB,CACnD,OAA4C,EAC5C,QAAQ,CACX,CAAC;gBAEF,IAAI,aAAa,EAAE,CAAC;oBAChB,kBAAkB,GAAG,aAAa,CAAC,kBAAkB,CAAC;oBACtD,YAAY,GAAG,aAAa,CAAC,YAAY,CAAC;gBAC9C,CAAC;gBAED,mGAAmG;gBACnG,IAAI,CAAC,YAAY,EAAE,CAAC;oBAChB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAClC,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACjG,CAAC;YACL,CAAC;iBAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAClC,2BAA2B;gBAC3B,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC9E,CAAC;YAED,OAAO;gBACH,kBAAkB;gBAClB,iBAAiB,EAAE,kBAAkB,IAAI,aAAa,CAAC,YAAY,CAAC,kBAAkB,CAAC;gBACvF,YAAY;gBACZ,WAAW,EAAE,YAAY,IAAI,aAAa,CAAC,YAAY,CAAC,YAAY,CAAC;aACxE,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,sBAAsB,CACxB,OAA+B,EAC/B,QAAgB;QAEhB,IAAI,CAAC;YACD,MAAM,OAAO,CAAC,YAAY,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC9C,OAAO;gBACH,kBAAkB,EAAE,GAAG,QAAQ,MAAM;gBACrC,YAAY,EAAE,GAAG,QAAQ,OAAO;aACnC,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,IAAY,EAAE,aAA4B,EAAE,QAAgB;QAC/E,IAAI,CAAC;YACD,MAAM,aAAa,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;YAC3E,OAAO,GAAG,QAAQ,OAAO,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,KAAqB;QAClC,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,GAC/F,gBAAgB,CAAC;QACrB,yCAAyC;QACzC,MAAM,cAAc,GAAG,MAAM;aACxB,UAAU,CAAC,MAAM,CAAC;aAClB,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;aAC1C,MAAM,CAAC,KAAK,CAAC;aACb,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;QAC/B,MAAM,kBAAkB,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC,IAAI,EAAE,CAAC;QAEjG;;WAEG;QACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAU,EAAE;YAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QACzC,CAAC,CAAC;QAEF,qDAAqD;QACrD,MAAM,QAAQ,GAAG,GAAG,eAAe,IAAI,cAAc,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,kBAAkB,CAAC,EAAE;aACxG,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,0CAA0C;aAC/D,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAEnC,OAAO,QAAQ,CAAC;IACpB,CAAC"}
1
+ {"version":3,"file":"error_snapshotter.js","sourceRoot":"","sources":["../../src/crawlers/error_snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAwBjC;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,gBAAgB;IACzB,MAAM,CAAU,oBAAoB,GAAG,EAAE,CAAC;IAC1C,MAAM,CAAU,eAAe,GAAG,EAAE,CAAC;IACrC,MAAM,CAAU,mBAAmB,GAAG,GAAG,CAAC;IAC1C,MAAM,CAAU,YAAY,GAAG,mBAAmB,CAAC;IACnD,MAAM,CAAU,eAAe,GAAG,gBAAgB,CAAC;IAEnD;;OAEG;IACH,KAAK,CAAC,eAAe,CACjB,KAAqB,EACrB,OAAkD;QAElD,IAAI,CAAC;YACD,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,CAAC;YAC3B,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,CAAC;YAE3B,MAAM,aAAa,GAAG,MAAM,OAAO,EAAE,gBAAgB,EAAE,CAAC;YACxD,0GAA0G;YAC1G,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrC,OAAO,EAAE,CAAC;YACd,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAE9C,IAAI,kBAAsC,CAAC;YAC3C,IAAI,YAAgC,CAAC;YAErC,IAAI,IAAI,EAAE,CAAC;gBACP,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,sBAAsB,CACnD,OAA4C,EAC5C,QAAQ,CACX,CAAC;gBAEF,IAAI,aAAa,EAAE,CAAC;oBAChB,kBAAkB,GAAG,aAAa,CAAC,kBAAkB,CAAC;oBACtD,YAAY,GAAG,aAAa,CAAC,YAAY,CAAC;gBAC9C,CAAC;gBAED,mGAAmG;gBACnG,IAAI,CAAC,YAAY,EAAE,CAAC;oBAChB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAClC,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACjG,CAAC;YACL,CAAC;iBAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAClC,2BAA2B;gBAC3B,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC9E,CAAC;YAED,OAAO;gBACH,kBAAkB;gBAClB,iBAAiB,EAAE,kBAAkB,IAAI,CAAC,MAAM,aAAa,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;gBAC/F,YAAY;gBACZ,WAAW,EAAE,YAAY,IAAI,CAAC,MAAM,aAAa,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;aAChF,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,sBAAsB,CACxB,OAA+B,EAC/B,QAAgB;QAEhB,IAAI,CAAC;YACD,MAAM,OAAO,CAAC,YAAY,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC9C,OAAO;gBACH,kBAAkB,EAAE,GAAG,QAAQ,MAAM;gBACrC,YAAY,EAAE,GAAG,QAAQ,OAAO;aACnC,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAClB,IAAY,EACZ,aAA8C,EAC9C,QAAgB;QAEhB,IAAI,CAAC;YACD,MAAM,aAAa,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;YAC3E,OAAO,GAAG,QAAQ,OAAO,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,KAAqB;QAClC,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,GAC/F,gBAAgB,CAAC;QACrB,yCAAyC;QACzC,MAAM,cAAc,GAAG,MAAM;aACxB,UAAU,CAAC,MAAM,CAAC;aAClB,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;aAC1C,MAAM,CAAC,KAAK,CAAC;aACb,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;QAC/B,MAAM,kBAAkB,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC,IAAI,EAAE,CAAC;QAEjG;;WAEG;QACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAU,EAAE;YAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QACzC,CAAC,CAAC;QAEF,qDAAqD;QACrD,MAAM,QAAQ,GAAG,GAAG,eAAe,IAAI,cAAc,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,kBAAkB,CAAC,EAAE;aACxG,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,0CAA0C;aAC/D,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAEnC,OAAO,QAAQ,CAAC;IACpB,CAAC"}
@@ -1,5 +1,6 @@
1
1
  import type { CrawlingContext } from '../crawlers/crawler_commons.js';
2
2
  import { ErrorSnapshotter } from './error_snapshotter.js';
3
+ import type { SnapshottableProperties } from './internals/types.js';
3
4
  /**
4
5
  * Node.js Error interface
5
6
  */
@@ -48,7 +49,7 @@ export declare class ErrorTracker {
48
49
  addAsync(error: ErrnoException, context?: CrawlingContext): Promise<void>;
49
50
  getUniqueErrorCount(): number;
50
51
  getMostPopularErrors(count: number): [number, string[]][];
51
- captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext): Promise<void>;
52
+ captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<void>;
52
53
  reset(): void;
53
54
  }
54
55
  //# sourceMappingURL=error_tracker.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"error_tracker.d.ts","sourceRoot":"","sources":["../../src/crawlers/error_tracker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,cAAe,SAAQ,KAAK;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,GAAG,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAChC,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,eAAe,EAAE,OAAO,CAAC;IACzB,kBAAkB,EAAE,OAAO,CAAC;CAC/B;AAuPD;;;;;;;;;;;;;GAaG;AACH,qBAAa,YAAY;;IAGrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC,KAAK,EAAE,MAAM,CAAC;IAEd,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;gBAExB,OAAO,GAAE,OAAO,CAAC,mBAAmB,CAAM;IAoBtD,OAAO,CAAC,WAAW;IAwBnB,GAAG,CAAC,KAAK,EAAE,cAAc;IAUzB;;;OAGG;IACG,QAAQ,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,eAAe;IAe/D,mBAAmB;IAoBnB,oBAAoB,CAAC,KAAK,EAAE,MAAM;IAoB5B,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,eAAe;IAWvG,KAAK;CAOR"}
1
+ {"version":3,"file":"error_tracker.d.ts","sourceRoot":"","sources":["../../src/crawlers/error_tracker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAEpE;;GAEG;AACH,MAAM,WAAW,cAAe,SAAQ,KAAK;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,GAAG,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAChC,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,eAAe,EAAE,OAAO,CAAC;IACzB,kBAAkB,EAAE,OAAO,CAAC;CAC/B;AAuPD;;;;;;;;;;;;;GAaG;AACH,qBAAa,YAAY;;IAGrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC,KAAK,EAAE,MAAM,CAAC;IAEd,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;gBAExB,OAAO,GAAE,OAAO,CAAC,mBAAmB,CAAM;IAoBtD,OAAO,CAAC,WAAW;IAwBnB,GAAG,CAAC,KAAK,EAAE,cAAc;IAUzB;;;OAGG;IACG,QAAQ,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,eAAe;IAe/D,mBAAmB;IAoBnB,oBAAoB,CAAC,KAAK,EAAE,MAAM;IAoB5B,eAAe,CACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,KAAK,EAAE,cAAc,EACrB,OAAO,EAAE,eAAe,GAAG,uBAAuB;IAYtD,KAAK;CAOR"}
@@ -1 +1 @@
1
- {"version":3,"file":"error_tracker.js","sourceRoot":"","sources":["../../src/crawlers/error_tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAuB1D,MAAM,6BAA6B,GAAG,CAAC,IAAY,EAAE,EAAE;IACnD,MAAM,6BAA6B,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAE5D,IAAI,6BAA6B,KAAK,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,6BAA6B,CAAC,CAAC;QAElF,IAAI,wBAAwB,KAAK,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,KAAK,CAAC,6BAA6B,GAAG,CAAC,EAAE,wBAAwB,CAAC,CAAC;QACnF,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,qEAAqE;AACrE,MAAM,qBAAqB,GAAG,CAAC,KAAe,EAAE,EAAE;IAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACnG,SAAS;QACb,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,aAAsB,EAAE,EAAE;IAC3G,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAElE,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IAEjB,IAAI,KAAK,EAAE,CAAC;QACR,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBAChE,OAAO,GAAG,CAAC,CAAC;gBACZ,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,oBAAoB,GAAG,IAAI,CAAC;IAChC,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,oBAAoB,GAAG,aAAa;YAChC,CAAC,CAAC,KAAM;iBACD,KAAK,CAAC,OAAO,CAAC;iBACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpB,IAAI,CAAC,IAAI,CAAC;YACjB,CAAC,CAAC,qBAAqB,CAAC,KAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACxB,oBAAoB,GAAG,qBAAqB,CAAC;IACjD,CAAC;IAED,IAAI,CAAC,CAAC,oBAAoB,IAAI,OAAO,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,oBAAoB,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,OAAO,OAAO,CAAC,oBAAoB,CAA4B,CAAC;AACpE,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,EAAE;IAClF,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAErB,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACrB,IAAI,GAAG,oBAAoB,CAAC;IAChC,CAAC;IAED,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAA4B,CAAC;AAC5D,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,EAAE;IAClF,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAEvB,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,CAA4B,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,2BAA2B,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IAC7D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QAC/C,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC;QAEhB,GAAG,CAAC;YACA,IAAI,UAAU,GAAG,MAAM,CAAC;YAExB,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;YAE1C,IAAI,UAAU,GAAG,MAAM,CAAC;YAExB,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,OAAO,UAAU,GAAG,CAAC,CAAC,MAAM,IAAI,UAAU,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;gBAC3F,MAAM,EAAE,CAAC;YACb,CAAC;YAED,IAAI,MAAM,GAAG,SAAS,EAAE,CAAC;gBACrB,SAAS,GAAG,MAAM,CAAC;gBACnB,YAAY,GAAG,MAAM,CAAC;gBACtB,YAAY,GAAG,MAAM,CAAC;YAC1B,CAAC;QACL,CAAC,QAAQ,MAAM,KAAK,CAAC,CAAC,EAAE;IAC5B,CAAC;IAED,OAAO;QACH,SAAS;QACT,YAAY;QACZ,YAAY;KACf,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,KAAgB,EAAE,MAAe,EAAE,EAAE;IACrD,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YAClB,MAAM,EAAE,CAAC;QACb,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IACtD,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,2BAA2B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEpF,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;IAEjD,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,YAAY,GAAG,SAAS,CAAC,CAAC,CAAC;IAEhE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,8BAA8B,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IAChE,MAAM,MAAM,GAAG,oBAAoB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE1C,0BAA0B;IAC1B,IAAI,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,aAAa,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,OAAgC,EAAE,EAAE;IAC7E,MAAM,WAAW,GAAG,8BAA8B,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEzF,IAAI,WAAW,KAAK,GAAG,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACrB,CAAC;IAMD,MAAM,KAAK,GAAI,OAAO,CAAC,CAAC,CAAc,CAAC,KAAK,GAAI,OAAO,CAAC,CAAC,CAAc,CAAC,KAAK,CAAC;IAE9E,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAClB,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAElB,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACtD,KAAK;KACR,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACvB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,eAAwB,EAAE,EAAE;IAC/G,IAAI,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAExB,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,IAAI,CAAC;YACD,OAAO;gBACH,OAAO,KAAK,KAAK,QAAQ;oBACrB,CAAC,CAAC,KAAK;oBACP,CAAC,CAAC,qDAAqD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3F,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,GAAG,iFAAiF,OAAO,CAAC,KAAK,EAAE;gBACtG,KAAK,EAAE,CAAC;aACX,CAAC,EAAE,CAAC;QACT,CAAC;IACL,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACnB,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YAClD,KAAK,EAAE,CAAC;SACX,CAAC,CAAC;QAEH,yFAAyF;QACzF,8DAA8D;QAC9D,KAAK,MAAM,eAAe,IAAI,OAAO,EAAE,CAAC;YACpC,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YACpE,IAAI,UAAU,EAAE,CAAC;gBACb,OAAO,GAAG,UAAU,CAAC;gBACrB,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,CAA4B,CAAC;AACvD,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,KAAyB,EAAE,EAAE;IAChD,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,EAAE,CAAC;QACtB,oDAAoD;QACpD,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,KAAM,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,YAAY;IACrB,QAAQ,CAAsB;IAE9B,MAAM,CAA0B;IAEhC,KAAK,CAAS;IAEd,gBAAgB,CAAoB;IAEpC,YAAY,UAAwC,EAAE;QAClD,IAAI,CAAC,QAAQ,GAAG;YACZ,aAAa,EAAE,IAAI;YACnB,aAAa,EAAE,IAAI;YACnB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,KAAK;YACpB,gBAAgB,EAAE,IAAI;YACtB,eAAe,EAAE,KAAK;YACtB,kBAAkB,EAAE,KAAK;YACzB,GAAG,OAAO;SACb,CAAC;QAEF,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,EAAE,CAAC;YACnC,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACnD,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IACnB,CAAC;IAEO,WAAW,CAAC,KAAqB;QACrC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC/B,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC9B,KAAK,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC9B,KAAK,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YACjC,KAAK,GAAG,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAC9E,CAAC;QAED,aAAa,CAAC,KAA0B,CAAC,CAAC;QAE1C,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,GAAG,CAAC,KAAqB;QACrB,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAExB,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAqB,EAAE,OAAyB;QAC3D,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAEtC,+EAA+E;QAC/E,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1D,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;IACL,CAAC;IAED,mBAAmB;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,MAAM,QAAQ,GAAG,CAAC,KAA8B,EAAQ,EAAE;YACtD,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;gBACnB,KAAK,EAAE,CAAC;gBACR,OAAO;YACX,CAAC;YAED,8DAA8D;YAC9D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC,CAAC;YACpD,CAAC;QACL,CAAC,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEtB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,oBAAoB,CAAC,KAAa;QAC9B,MAAM,MAAM,GAAyB,EAAE,CAAC;QAExC,MAAM,QAAQ,GAAG,CAAC,KAA8B,EAAE,IAAc,EAAQ,EAAE;YACtE,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,CAAE,KAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC1C,OAAO;YACX,CAAC;YAED,8DAA8D;YAC9D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAA4B,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;YACpE,CAAC;QACL,CAAC,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAE1B,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAgC,EAAE,KAAqB,EAAE,OAAwB;QACnG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACzB,OAAO;QACX,CAAC;QAED,MAAM,EAAE,iBAAiB,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAEvG,OAAO,CAAC,uBAAuB,GAAG,iBAAiB,CAAC;QACpD,OAAO,CAAC,iBAAiB,GAAG,WAAW,CAAC;IAC5C,CAAC;IAED,KAAK;QACD,yFAAyF;QACzF,8DAA8D;QAC9D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;CACJ"}
1
+ {"version":3,"file":"error_tracker.js","sourceRoot":"","sources":["../../src/crawlers/error_tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAwB1D,MAAM,6BAA6B,GAAG,CAAC,IAAY,EAAE,EAAE;IACnD,MAAM,6BAA6B,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAE5D,IAAI,6BAA6B,KAAK,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,6BAA6B,CAAC,CAAC;QAElF,IAAI,wBAAwB,KAAK,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,KAAK,CAAC,6BAA6B,GAAG,CAAC,EAAE,wBAAwB,CAAC,CAAC;QACnF,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,qEAAqE;AACrE,MAAM,qBAAqB,GAAG,CAAC,KAAe,EAAE,EAAE;IAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACnG,SAAS;QACb,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,aAAsB,EAAE,EAAE;IAC3G,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAElE,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IAEjB,IAAI,KAAK,EAAE,CAAC;QACR,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBAChE,OAAO,GAAG,CAAC,CAAC;gBACZ,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,oBAAoB,GAAG,IAAI,CAAC;IAChC,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,oBAAoB,GAAG,aAAa;YAChC,CAAC,CAAC,KAAM;iBACD,KAAK,CAAC,OAAO,CAAC;iBACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpB,IAAI,CAAC,IAAI,CAAC;YACjB,CAAC,CAAC,qBAAqB,CAAC,KAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACxB,oBAAoB,GAAG,qBAAqB,CAAC;IACjD,CAAC;IAED,IAAI,CAAC,CAAC,oBAAoB,IAAI,OAAO,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,oBAAoB,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,OAAO,OAAO,CAAC,oBAAoB,CAA4B,CAAC;AACpE,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,EAAE;IAClF,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAErB,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACrB,IAAI,GAAG,oBAAoB,CAAC;IAChC,CAAC;IAED,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAA4B,CAAC;AAC5D,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,EAAE;IAClF,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAEvB,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,CAA4B,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,2BAA2B,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IAC7D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QAC/C,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC;QAEhB,GAAG,CAAC;YACA,IAAI,UAAU,GAAG,MAAM,CAAC;YAExB,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;YAE1C,IAAI,UAAU,GAAG,MAAM,CAAC;YAExB,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,OAAO,UAAU,GAAG,CAAC,CAAC,MAAM,IAAI,UAAU,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;gBAC3F,MAAM,EAAE,CAAC;YACb,CAAC;YAED,IAAI,MAAM,GAAG,SAAS,EAAE,CAAC;gBACrB,SAAS,GAAG,MAAM,CAAC;gBACnB,YAAY,GAAG,MAAM,CAAC;gBACtB,YAAY,GAAG,MAAM,CAAC;YAC1B,CAAC;QACL,CAAC,QAAQ,MAAM,KAAK,CAAC,CAAC,EAAE;IAC5B,CAAC;IAED,OAAO;QACH,SAAS;QACT,YAAY;QACZ,YAAY;KACf,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,KAAgB,EAAE,MAAe,EAAE,EAAE;IACrD,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YAClB,MAAM,EAAE,CAAC;QACb,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IACtD,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,2BAA2B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEpF,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;IAEjD,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,YAAY,GAAG,SAAS,CAAC,CAAC,CAAC;IAEhE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,8BAA8B,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IAChE,MAAM,MAAM,GAAG,oBAAoB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE1C,0BAA0B;IAC1B,IAAI,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,aAAa,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,OAAgC,EAAE,EAAE;IAC7E,MAAM,WAAW,GAAG,8BAA8B,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEzF,IAAI,WAAW,KAAK,GAAG,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACrB,CAAC;IAMD,MAAM,KAAK,GAAI,OAAO,CAAC,CAAC,CAAc,CAAC,KAAK,GAAI,OAAO,CAAC,CAAC,CAAc,CAAC,KAAK,CAAC;IAE9E,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAClB,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAElB,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACtD,KAAK;KACR,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACvB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,eAAwB,EAAE,EAAE;IAC/G,IAAI,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAExB,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,IAAI,CAAC;YACD,OAAO;gBACH,OAAO,KAAK,KAAK,QAAQ;oBACrB,CAAC,CAAC,KAAK;oBACP,CAAC,CAAC,qDAAqD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3F,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,GAAG,iFAAiF,OAAO,CAAC,KAAK,EAAE;gBACtG,KAAK,EAAE,CAAC;aACX,CAAC,EAAE,CAAC;QACT,CAAC;IACL,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACnB,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YAClD,KAAK,EAAE,CAAC;SACX,CAAC,CAAC;QAEH,yFAAyF;QACzF,8DAA8D;QAC9D,KAAK,MAAM,eAAe,IAAI,OAAO,EAAE,CAAC;YACpC,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YACpE,IAAI,UAAU,EAAE,CAAC;gBACb,OAAO,GAAG,UAAU,CAAC;gBACrB,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,CAA4B,CAAC;AACvD,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,KAAyB,EAAE,EAAE;IAChD,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,EAAE,CAAC;QACtB,oDAAoD;QACpD,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,KAAM,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,YAAY;IACrB,QAAQ,CAAsB;IAE9B,MAAM,CAA0B;IAEhC,KAAK,CAAS;IAEd,gBAAgB,CAAoB;IAEpC,YAAY,UAAwC,EAAE;QAClD,IAAI,CAAC,QAAQ,GAAG;YACZ,aAAa,EAAE,IAAI;YACnB,aAAa,EAAE,IAAI;YACnB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,KAAK;YACpB,gBAAgB,EAAE,IAAI;YACtB,eAAe,EAAE,KAAK;YACtB,kBAAkB,EAAE,KAAK;YACzB,GAAG,OAAO;SACb,CAAC;QAEF,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,EAAE,CAAC;YACnC,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACnD,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IACnB,CAAC;IAEO,WAAW,CAAC,KAAqB;QACrC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC/B,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC9B,KAAK,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC9B,KAAK,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YACjC,KAAK,GAAG,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAC9E,CAAC;QAED,aAAa,CAAC,KAA0B,CAAC,CAAC;QAE1C,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,GAAG,CAAC,KAAqB;QACrB,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAExB,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAqB,EAAE,OAAyB;QAC3D,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAEtC,+EAA+E;QAC/E,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1D,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;IACL,CAAC;IAED,mBAAmB;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,MAAM,QAAQ,GAAG,CAAC,KAA8B,EAAQ,EAAE;YACtD,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;gBACnB,KAAK,EAAE,CAAC;gBACR,OAAO;YACX,CAAC;YAED,8DAA8D;YAC9D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC,CAAC;YACpD,CAAC;QACL,CAAC,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEtB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,oBAAoB,CAAC,KAAa;QAC9B,MAAM,MAAM,GAAyB,EAAE,CAAC;QAExC,MAAM,QAAQ,GAAG,CAAC,KAA8B,EAAE,IAAc,EAAQ,EAAE;YACtE,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,CAAE,KAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC1C,OAAO;YACX,CAAC;YAED,8DAA8D;YAC9D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAA4B,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;YACpE,CAAC;QACL,CAAC,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAE1B,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,eAAe,CACjB,OAAgC,EAChC,KAAqB,EACrB,OAAkD;QAElD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACzB,OAAO;QACX,CAAC;QAED,MAAM,EAAE,iBAAiB,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAEvG,OAAO,CAAC,uBAAuB,GAAG,iBAAiB,CAAC;QACpD,OAAO,CAAC,iBAAiB,GAAG,WAAW,CAAC;IAC5C,CAAC;IAED,KAAK;QACD,yFAAyF;QACzF,8DAA8D;QAC9D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;CACJ"}
@@ -1,5 +1,5 @@
1
+ export * from './context_pipeline.js';
1
2
  export * from './crawler_commons.js';
2
- export * from './crawler_extension.js';
3
3
  export * from './crawler_utils.js';
4
4
  export * from './statistics.js';
5
5
  export * from './error_tracker.js';
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,wBAAwB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,wBAAwB,CAAC"}
package/crawlers/index.js CHANGED
@@ -1,5 +1,5 @@
1
+ export * from './context_pipeline.js';
1
2
  export * from './crawler_commons.js';
2
- export * from './crawler_extension.js';
3
3
  export * from './crawler_utils.js';
4
4
  export * from './statistics.js';
5
5
  export * from './error_tracker.js';
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,wBAAwB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,wBAAwB,CAAC"}
@@ -0,0 +1,8 @@
1
+ export interface BrowserPage {
2
+ content: () => Promise<string>;
3
+ }
4
+ export interface SnapshottableProperties {
5
+ body?: unknown;
6
+ page?: BrowserPage;
7
+ }
8
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/crawlers/internals/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IACxB,OAAO,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,uBAAuB;IACpC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,WAAW,CAAC;CACtB"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/crawlers/internals/types.ts"],"names":[],"mappings":""}
@@ -1,12 +1,9 @@
1
- import type { Awaitable, BatchAddRequestsResult, Dictionary } from '@crawlee/types';
1
+ import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
2
2
  import { type RobotsTxtFile } from '@crawlee/utils';
3
3
  import type { SetRequired } from 'type-fest';
4
- import type { RequestProvider, RequestQueueOperationOptions } from '../storages/request_provider.js';
5
- import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform } from './shared.js';
6
- export type SkippedRequestCallback = (args: {
7
- url: string;
8
- reason: 'robotsTxt';
9
- }) => Awaitable<void>;
4
+ import type { Request } from '../request.js';
5
+ import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestProvider, RequestQueueOperationOptions } from '../storages/request_provider.js';
6
+ import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from './shared.js';
10
7
  export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
11
8
  /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
12
9
  limit?: number;
@@ -18,7 +15,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
18
15
  selector?: string;
19
16
  /** Sets {@link Request.userData} for newly enqueued requests. */
20
17
  userData?: Dictionary;
21
- /** Sets {@link Request.label} for newly enqueued requests. */
18
+ /**
19
+ * Sets {@link Request.label} for newly enqueued requests.
20
+ *
21
+ * Note that the request options specified in `globs`, `regexps`, or `pseudoUrls` objects
22
+ * have priority over this option.
23
+ */
22
24
  label?: string;
23
25
  /**
24
26
  * If set to `true`, tells the crawler to skip navigation and process the request directly.
@@ -49,7 +51,6 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
49
51
  * containing patterns matching URLs that will **never** be enqueued.
50
52
  *
51
53
  * The plain objects must include either the `glob` property or the `regexp` property.
52
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
53
54
  *
54
55
  * Glob matching is always case-insensitive.
55
56
  * If you need case-sensitive matching, provide a regexp.
@@ -105,9 +106,8 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
105
106
  * }
106
107
  * ```
107
108
  *
108
- * Note that `transformRequestFunction` has a priority over request options
109
- * specified in `globs`, `regexps`, or `pseudoUrls` objects,
110
- * and thus some options could be over-written by `transformRequestFunction`.
109
+ * Note that the request options specified in `globs`, `regexps`, or `pseudoUrls` objects
110
+ * have priority over this function. Some request options returned by `transformRequestFunction` may be overwritten by pattern-based options from `globs`, `regexps`, or `pseudoUrls`.
111
111
  */
112
112
  transformRequestFunction?: RequestTransform;
113
113
  /**
@@ -138,10 +138,13 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
138
138
  * RobotsTxtFile instance for the current request that triggered the `enqueueLinks`.
139
139
  * If provided, disallowed URLs will be ignored.
140
140
  */
141
- robotsTxtFile?: RobotsTxtFile;
141
+ robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
142
142
  /**
143
143
  * When a request is skipped for some reason, you can use this callback to act on it.
144
- * This is currently fired only for requests skipped based on robots.txt file.
144
+ * This is currently fired for requests skipped
145
+ * 1. based on robots.txt file,
146
+ * 2. because they don't match enqueueLinks filters,
147
+ * 3. or because the maxRequestsPerCrawl limit has been reached
145
148
  */
146
149
  onSkippedRequest?: SkippedRequestCallback;
147
150
  }
@@ -220,7 +223,11 @@ export declare enum EnqueueStrategy {
220
223
  * @param options All `enqueueLinks()` parameters are passed via an options object.
221
224
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
222
225
  */
223
- export declare function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'requestQueue' | 'urls'>): Promise<BatchAddRequestsResult>;
226
+ export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestQueue'>, 'urls'> & {
227
+ requestQueue: {
228
+ addRequestsBatched: (requests: Request<Dictionary>[], options: AddRequestsBatchedOptions) => Promise<AddRequestsBatchedResult>;
229
+ };
230
+ }): Promise<BatchAddRequestsResult>;
224
231
  /**
225
232
  * @internal
226
233
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -1 +1 @@
1
- {"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACpF,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAK7C,OAAO,KAAK,EAAE,eAAe,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AACrG,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,aAAa,CAAC;AAU9G,MAAM,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAA;CAAE,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAErG,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEzB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,eAAe,CAAC;IAE/B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;IAE/C;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IAEjC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAEvC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;IAErF;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;CAC7C;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;;;OAMG;IACH,YAAY,kBAAkB;IAE9B;;;;;;OAMG;IACH,UAAU,gBAAgB;IAE1B;;;;;;OAMG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAC9B,OAAO,EAAE,WAAW,CAAC,mBAAmB,EAAE,cAAc,GAAG,MAAM,CAAC,GACnE,OAAO,CAAC,sBAAsB,CAAC,CAmLjC;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
1
+ {"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAI7C,OAAO,KAAK,EAAE,OAAO,EAAkB,MAAM,eAAe,CAAC;AAC7D,OAAO,KAAK,EACR,yBAAyB,EACzB,wBAAwB,EACxB,eAAe,EACf,4BAA4B,EAC/B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EACR,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,sBAAsB,EAGzB,MAAM,aAAa,CAAC;AAUrB,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEzB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,eAAe,CAAC;IAE/B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;IAE/C;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IAEjC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAEvC;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;IAErF;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;;OAGG;IACH,aAAa,CAAC,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAEjD;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;CAC7C;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;;;OAMG;IACH,YAAY,kBAAkB;IAE9B;;;;;;OAMG;IACH,UAAU,gBAAgB;IAE1B;;;;;;OAMG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAC9B,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,GAAG;IACtE,YAAY,EAAE;QACV,kBAAkB,EAAE,CAChB,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,EAC/B,OAAO,EAAE,yBAAyB,KACjC,OAAO,CAAC,wBAAwB,CAAC,CAAC;KAC1C,CAAC;CACL,GACF,OAAO,CAAC,sBAAsB,CAAC,CA4NjC;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
@@ -87,7 +87,7 @@ export async function enqueueLinks(options) {
87
87
  }
88
88
  ow(options, ow.object.exactShape({
89
89
  urls: ow.array.ofType(ow.string),
90
- requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
90
+ requestQueue: ow.object.hasKeys('addRequestsBatched'),
91
91
  robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
92
92
  onSkippedRequest: ow.optional.function,
93
93
  forefront: ow.optional.boolean,
@@ -167,6 +167,16 @@ export async function enqueueLinks(options) {
167
167
  break;
168
168
  }
169
169
  }
170
+ async function reportSkippedRequests(skippedRequests, reason) {
171
+ if (onSkippedRequest && skippedRequests.length > 0) {
172
+ await Promise.all(skippedRequests.map((request) => {
173
+ return onSkippedRequest({
174
+ url: request.url,
175
+ reason: request.skippedReason ?? reason,
176
+ });
177
+ }));
178
+ }
179
+ }
170
180
  let requestOptions = createRequestOptions(urls, options);
171
181
  if (robotsTxtFile) {
172
182
  const skippedRequests = [];
@@ -177,30 +187,39 @@ export async function enqueueLinks(options) {
177
187
  skippedRequests.push(request);
178
188
  return false;
179
189
  });
180
- if (onSkippedRequest && skippedRequests.length > 0) {
181
- await Promise.all(skippedRequests.map((request) => {
182
- return onSkippedRequest({ url: request.url, reason: 'robotsTxt' });
183
- }));
184
- }
190
+ await reportSkippedRequests(skippedRequests, 'robotsTxt');
185
191
  }
186
192
  if (transformRequestFunction) {
193
+ const skippedRequests = [];
187
194
  requestOptions = requestOptions
188
- .map((request) => transformRequestFunction(request))
189
- .filter((r) => !!r);
195
+ .map((request) => {
196
+ const transformedRequest = transformRequestFunction(request);
197
+ if (!transformedRequest) {
198
+ skippedRequests.push(request);
199
+ }
200
+ return transformedRequest;
201
+ })
202
+ .filter((r) => Boolean(r));
203
+ await reportSkippedRequests(skippedRequests, 'filters');
190
204
  }
191
- function createFilteredRequests() {
205
+ async function createFilteredRequests() {
206
+ const skippedRequests = [];
192
207
  // No user provided patterns means we can skip an extra filtering step
193
208
  if (urlPatternObjects.length === 0) {
194
- return createRequests(requestOptions, enqueueStrategyPatterns, urlExcludePatternObjects, options.strategy);
209
+ return createRequests(requestOptions, enqueueStrategyPatterns, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
195
210
  }
196
211
  // Generate requests based on the user patterns first
197
- const generatedRequestsFromUserFilters = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy);
212
+ const generatedRequestsFromUserFilters = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
198
213
  // ...then filter them by the enqueue links strategy (making this an AND check)
199
- return filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns);
214
+ const filtered = filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns, (url) => skippedRequests.push(url));
215
+ await reportSkippedRequests(skippedRequests.map((url) => ({ url })), 'filters');
216
+ return filtered;
200
217
  }
201
- let requests = createFilteredRequests();
202
- if (limit)
218
+ let requests = await createFilteredRequests();
219
+ if (typeof limit === 'number' && limit < requests.length) {
220
+ await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');
203
221
  requests = requests.slice(0, limit);
222
+ }
204
223
  const { addedRequests } = await requestQueue.addRequestsBatched(requests, {
205
224
  forefront,
206
225
  waitForAllRequestsToBeAdded,
@@ -1 +1 @@
1
- {"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAGlC,OAAO,GAAG,MAAM,YAAY,CAAC;AAK7B,OAAO,EACH,6BAA6B,EAC7B,oCAAoC,EACpC,iCAAiC,EACjC,oBAAoB,EACpB,cAAc,EACd,wBAAwB,GAC3B,MAAM,aAAa,CAAC;AA+JrB;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAN,IAAY,eAgCX;AAhCD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;;;OAMG;IACH,iDAA8B,CAAA;IAE9B;;;;;;OAMG;IACH,6CAA0B,CAAA;IAE1B;;;;;;OAMG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAhCW,eAAe,KAAf,eAAe,QAgC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,OAAkE;IAElE,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,UAAU,CAChB;YACI,4JAA4J;YAC5J,kHAAkH;SACrH,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;IACN,CAAC;IAED,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACtD,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QACtC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;QACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAClE,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACnD,CAAC,CACL,CAAC;IAEF,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,OAAO,EACP,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,SAAS,EACT,2BAA2B,EAC3B,aAAa,EACb,gBAAgB,GACnB,GAAG,OAAO,CAAC;IAEZ,MAAM,wBAAwB,GAAuB,EAAE,CAAC;IACxD,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC7C,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACpD,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,GAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,oCAAoC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,CAAC,QAAQ,KAAK,eAAe,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE,CAAC;oBAClB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CACjD,CAAC;gBACN,CAAC;qBAAM,CAAC;oBACJ,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjF,CAAC;gBAED,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4EAA4E;gBAC5E,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxD,MAAM;QACd,CAAC;IACL,CAAC;IAED,IAAI,cAAc,GAAG,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,aAAa,EAAE,CAAC;QAChB,MAAM,eAAe,GAAqB,EAAE,CAAC;QAE7C,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/C,IAAI,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvC,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,IAAI,gBAAgB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,OAAO,CAAC,GAAG,CACb,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,OAAO,gBAAgB,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;YACvE,CAAC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,IAAI,wBAAwB,EAAE,CAAC;QAC3B,cAAc,GAAG,cAAc;aAC1B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC;aACnD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;IAChD,CAAC;IAED,SAAS,sBAAsB;QAC3B,sEAAsE;QACtE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,cAAc,CAAC,cAAc,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/G,CAAC;QAED,qDAAqD;QACrD,MAAM,gCAAgC,GAAG,cAAc,CACnD,cAAc,EACd,iBAAiB,EACjB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,+EAA+E;QAC/E,OAAO,wBAAwB,CAAC,gCAAgC,EAAE,uBAAuB,CAAC,CAAC;IAC/F,CAAC;IAED,IAAI,QAAQ,GAAG,sBAAsB,EAAE,CAAC;IACxC,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QACtE,SAAS;QACT,2BAA2B;KAC9B,CAAC,CAAC;IAEH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE,CAAC;QACtB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,SAAS,CAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO,cAAc,CAAC;QAC1B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAYD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC"}
1
+ {"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAGlC,OAAO,GAAG,MAAM,YAAY,CAAC;AAkB7B,OAAO,EACH,6BAA6B,EAC7B,oCAAoC,EACpC,iCAAiC,EACjC,oBAAoB,EACpB,cAAc,EACd,wBAAwB,GAC3B,MAAM,aAAa,CAAC;AAmKrB;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAN,IAAY,eAgCX;AAhCD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;;;OAMG;IACH,iDAA8B,CAAA;IAE9B;;;;;;OAMG;IACH,6CAA0B,CAAA;IAE1B;;;;;;OAMG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAhCW,eAAe,KAAf,eAAe,QAgC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,OAOC;IAED,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,UAAU,CAChB;YACI,4JAA4J;YAC5J,kHAAkH;SACrH,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;IACN,CAAC;IAED,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC;QACrD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACtD,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QACtC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;QACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAClE,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACnD,CAAC,CACL,CAAC;IAEF,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,OAAO,EACP,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,SAAS,EACT,2BAA2B,EAC3B,aAAa,EACb,gBAAgB,GACnB,GAAG,OAAO,CAAC;IAEZ,MAAM,wBAAwB,GAAuB,EAAE,CAAC;IACxD,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC7C,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACpD,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,GAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,oCAAoC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,CAAC,QAAQ,KAAK,eAAe,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE,CAAC;oBAClB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CACjD,CAAC;gBACN,CAAC;qBAAM,CAAC;oBACJ,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjF,CAAC;gBAED,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4EAA4E;gBAC5E,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxD,MAAM;QACd,CAAC;IACL,CAAC;IAED,KAAK,UAAU,qBAAqB,CAChC,eAAwE,EACxE,MAA4B;QAE5B,IAAI,gBAAgB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,OAAO,CAAC,GAAG,CACb,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,OAAO,gBAAgB,CAAC;oBACpB,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,MAAM,EAAE,OAAO,CAAC,aAAa,IAAI,MAAM;iBAC1C,CAAkB,CAAC;YACxB,CAAC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,IAAI,cAAc,GAAG,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,aAAa,EAAE,CAAC;QAChB,MAAM,eAAe,GAAqB,EAAE,CAAC;QAE7C,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/C,IAAI,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvC,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,MAAM,qBAAqB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,wBAAwB,EAAE,CAAC;QAC3B,MAAM,eAAe,GAAqB,EAAE,CAAC;QAE7C,cAAc,GAAG,cAAc;aAC1B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACb,MAAM,kBAAkB,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;YAC7D,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACtB,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;YACD,OAAO,kBAAkB,CAAC;QAC9B,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAqB,CAAC;QAEnD,MAAM,qBAAqB,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,UAAU,sBAAsB;QACjC,MAAM,eAAe,GAAa,EAAE,CAAC;QAErC,sEAAsE;QACtE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,cAAc,CACjB,cAAc,EACd,uBAAuB,EACvB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,EAChB,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CACrC,CAAC;QACN,CAAC;QAED,qDAAqD;QACrD,MAAM,gCAAgC,GAAG,cAAc,CACnD,cAAc,EACd,iBAAiB,EACjB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,EAChB,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CACrC,CAAC;QACF,+EAA+E;QAC/E,MAAM,QAAQ,GAAG,wBAAwB,CAAC,gCAAgC,EAAE,uBAAuB,EAAE,CAAC,GAAG,EAAE,EAAE,CACzG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAC5B,CAAC;QAEF,MAAM,qBAAqB,CACvB,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EACvC,SAAS,CACZ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,IAAI,QAAQ,GAAG,MAAM,sBAAsB,EAAE,CAAC;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvD,MAAM,qBAAqB,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,CAAC;QACnE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QACtE,SAAS;QACT,2BAA2B;KAC9B,CAAC,CAAC;IAEH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE,CAAC;QACtB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,SAAS,CAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO,cAAc,CAAC;QAC1B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAYD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC"}
@@ -1,3 +1,4 @@
1
+ import type { Awaitable } from '@crawlee/types';
1
2
  import type { RequestOptions } from '../request.js';
2
3
  import { Request } from '../request.js';
3
4
  import type { EnqueueLinksOptions } from './enqueue_links.js';
@@ -18,6 +19,11 @@ export type RegExpObject = {
18
19
  regexp: RegExp;
19
20
  } & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
20
21
  export type RegExpInput = RegExp | RegExpObject;
22
+ export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'redirect' | 'depth';
23
+ export type SkippedRequestCallback = (args: {
24
+ url: string;
25
+ reason: SkippedRequestReason;
26
+ }) => Awaitable<void>;
21
27
  /**
22
28
  * @ignore
23
29
  */
@@ -47,8 +53,8 @@ export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegE
47
53
  /**
48
54
  * @ignore
49
55
  */
50
- export declare function createRequests(requestOptions: (string | RequestOptions)[], urlPatternObjects?: UrlPatternObject[], excludePatternObjects?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy']): Request[];
51
- export declare function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[]): Request[];
56
+ export declare function createRequests(requestOptions: (string | RequestOptions)[], urlPatternObjects?: UrlPatternObject[], excludePatternObjects?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy'], onSkippedUrl?: (url: string) => void): Request[];
57
+ export declare function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[], onSkippedUrl?: (url: string) => void): Request[];
52
58
  /**
53
59
  * @ignore
54
60
  */
@@ -1 +1 @@
1
- {"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAYhD,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CACjD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAC5C,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAChD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAC1C,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAC9C,OAAO,EAAE,YAAY,GAAG,UAAU,GACnC,IAAI,CAMN;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,EAAE,CAiB1G;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,EAAE,CAkCvF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBjG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC1B,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAC3C,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,EACtC,qBAAqB,GAAE,gBAAgB,EAAO,EAC9C,QAAQ,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,GAC3C,OAAO,EAAE,CA8BX;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CAoBtG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,EACtD,OAAO,GAAE,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,gBAAgB,GAAG,UAAU,CAAM,GAC1G,cAAc,EAAE,CA+BlB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;OAGG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC;CACzE"}
1
+ {"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAKhD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAYhD,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CACjD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAC5C,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAChD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,WAAW,GAAG,OAAO,GAAG,cAAc,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,CAAC;AAE7G,MAAM,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAA;CAAE,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE9G;;GAEG;AACH,wBAAgB,8BAA8B,CAC1C,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAC9C,OAAO,EAAE,YAAY,GAAG,UAAU,GACnC,IAAI,CAMN;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,EAAE,CAiB1G;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,EAAE,CAkCvF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBjG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC1B,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAC3C,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,EACtC,qBAAqB,GAAE,gBAAgB,EAAO,EAC9C,QAAQ,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,EAC1C,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GACrC,OAAO,EAAE,CAqCX;AAED,wBAAgB,wBAAwB,CACpC,QAAQ,EAAE,OAAO,EAAE,EACnB,QAAQ,CAAC,EAAE,gBAAgB,EAAE,EAC7B,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GACrC,OAAO,EAAE,CAmBX;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,EACtD,OAAO,GAAE,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,gBAAgB,GAAG,UAAU,CAAM,GAC1G,cAAc,EAAE,CA+BlB;AAmBD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;OAGG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC;CACzE"}
@@ -1,5 +1,5 @@
1
1
  import { URL } from 'node:url';
2
- import { minimatch } from 'minimatch';
2
+ import { Minimatch } from 'minimatch';
3
3
  import { purlToRegExp } from '@apify/pseudo_url';
4
4
  import { Request } from '../request.js';
5
5
  export { tryAbsoluteURL } from '@crawlee/utils';
@@ -112,22 +112,25 @@ export function constructRegExpObjectsFromRegExps(regexps) {
112
112
  /**
113
113
  * @ignore
114
114
  */
115
- export function createRequests(requestOptions, urlPatternObjects, excludePatternObjects = [], strategy) {
115
+ export function createRequests(requestOptions, urlPatternObjects, excludePatternObjects = [], strategy, onSkippedUrl) {
116
+ const excludePatternObjectMatchers = excludePatternObjects.map(createPatternObjectMatcher);
117
+ const urlPatternObjectMatchers = urlPatternObjects?.map(createPatternObjectMatcher);
116
118
  return requestOptions
117
119
  .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
118
120
  .filter(({ url }) => {
119
- return !excludePatternObjects.some((excludePatternObject) => {
120
- const { regexp, glob } = excludePatternObject;
121
- return (regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }));
122
- });
121
+ const matchesExcludePatterns = excludePatternObjectMatchers.some(({ match }) => match(url));
122
+ if (matchesExcludePatterns) {
123
+ onSkippedUrl?.(url);
124
+ }
125
+ return !matchesExcludePatterns;
123
126
  })
124
127
  .map(({ url, opts }) => {
125
- if (!urlPatternObjects || !urlPatternObjects.length) {
128
+ if (!urlPatternObjectMatchers || !urlPatternObjectMatchers.length) {
126
129
  return new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts });
127
130
  }
128
- for (const urlPatternObject of urlPatternObjects) {
129
- const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;
130
- if ((regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }))) {
131
+ for (const urlPatternObject of urlPatternObjectMatchers) {
132
+ const { match, glob, regexp, ...requestRegExpOptions } = urlPatternObject;
133
+ if (match(url)) {
131
134
  const request = typeof opts === 'string'
132
135
  ? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
133
136
  : { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
@@ -135,23 +138,24 @@ export function createRequests(requestOptions, urlPatternObjects, excludePattern
135
138
  }
136
139
  }
137
140
  // didn't match any positive pattern
141
+ onSkippedUrl?.(url);
138
142
  return null;
139
143
  })
140
144
  .filter((request) => request);
141
145
  }
142
- export function filterRequestsByPatterns(requests, patterns) {
146
+ export function filterRequestsByPatterns(requests, patterns, onSkippedUrl) {
143
147
  if (!patterns?.length) {
144
148
  return requests;
145
149
  }
146
150
  const filtered = [];
151
+ const patternMatchers = patterns?.map(createPatternObjectMatcher);
147
152
  for (const request of requests) {
148
- for (const urlPatternObject of patterns) {
149
- const { regexp, glob } = urlPatternObject;
150
- if ((regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true }))) {
151
- filtered.push(request);
152
- // Break the pattern loop, as we already matched this request once
153
- break;
154
- }
153
+ const matchingPattern = patternMatchers.find(({ match }) => match(request.url));
154
+ if (matchingPattern !== undefined) {
155
+ filtered.push(request);
156
+ }
157
+ else {
158
+ onSkippedUrl?.(request.url);
155
159
  }
156
160
  }
157
161
  return filtered;
@@ -187,4 +191,22 @@ export function createRequestOptions(sources, options = {}) {
187
191
  return requestOptions;
188
192
  });
189
193
  }
194
+ /**
195
+ * @ignore
196
+ */
197
+ function createPatternObjectMatcher(urlPatternObject) {
198
+ const { regexp, glob } = urlPatternObject;
199
+ let match;
200
+ if (regexp) {
201
+ match = (url) => regexp.test(url);
202
+ }
203
+ else if (glob) {
204
+ const m = new Minimatch(glob, { nocase: true });
205
+ match = (url) => m.match(url);
206
+ }
207
+ else {
208
+ match = () => false;
209
+ }
210
+ return { ...urlPatternObject, match };
211
+ }
190
212
  //# sourceMappingURL=shared.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AA4B3C;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAC1C,IAA8C,EAC9C,OAAkC;IAElC,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oCAAoC,CAAC,UAAqC;IACtF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAAC,KAA2B;IACrE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAClG,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAAC,OAA+B;IAC7E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC1B,cAA2C,EAC3C,iBAAsC,EACtC,wBAA4C,EAAE,EAC9C,QAA0C;IAE1C,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC1E,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,oBAAoB,EAAE,EAAE;YACxD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,CAAC;YAC9C,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7F,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE;QACnB,IAAI,CAAC,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAClD,OAAO,IAAI,OAAO,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,KAAK,MAAM,gBAAgB,IAAI,iBAAiB,EAAE,CAAC;YAC/C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,GAAG,gBAAgB,CAAC;YACnE,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpF,MAAM,OAAO,GACT,OAAO,IAAI,KAAK,QAAQ;oBACpB,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE;oBACnE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;gBAE1E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAc,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,QAAmB,EAAE,QAA6B;IACvF,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,KAAK,MAAM,gBAAgB,IAAI,QAAQ,EAAE,CAAC;YACtC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,kEAAkE;gBAClE,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAChC,OAAsD,EACtD,UAAuG,EAAE;IAEzG,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACT,OAAO,GAAG,KAAK,QAAQ;QACnB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE;QACjD,CAAC,CAAE,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAqB,CAC1E;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC"}
1
+ {"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAG/B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AAgC3C;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAC1C,IAA8C,EAC9C,OAAkC;IAElC,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oCAAoC,CAAC,UAAqC;IACtF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAAC,KAA2B;IACrE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAClG,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAAC,OAA+B;IAC7E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC1B,cAA2C,EAC3C,iBAAsC,EACtC,wBAA4C,EAAE,EAC9C,QAA0C,EAC1C,YAAoC;IAEpC,MAAM,4BAA4B,GAAG,qBAAqB,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAC3F,MAAM,wBAAwB,GAAG,iBAAiB,EAAE,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAEpF,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC1E,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,MAAM,sBAAsB,GAAG,4BAA4B,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAE5F,IAAI,sBAAsB,EAAE,CAAC;YACzB,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,OAAO,CAAC,sBAAsB,CAAC;IACnC,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE;QACnB,IAAI,CAAC,wBAAwB,IAAI,CAAC,wBAAwB,CAAC,MAAM,EAAE,CAAC;YAChE,OAAO,IAAI,OAAO,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,KAAK,MAAM,gBAAgB,IAAI,wBAAwB,EAAE,CAAC;YACtD,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,oBAAoB,EAAE,GAAG,gBAAgB,CAAC;YAC1E,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACb,MAAM,OAAO,GACT,OAAO,IAAI,KAAK,QAAQ;oBACpB,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE;oBACnE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;gBAE1E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC;QACpB,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAc,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,wBAAwB,CACpC,QAAmB,EACnB,QAA6B,EAC7B,YAAoC;IAEpC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,eAAe,GAAG,QAAQ,EAAE,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAElE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,eAAe,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;QAEhF,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC;aAAM,CAAC;YACJ,YAAY,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAChC,OAAsD,EACtD,UAAuG,EAAE;IAEzG,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACT,OAAO,GAAG,KAAK,QAAQ;QACnB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE;QACjD,CAAC,CAAE,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAqB,CAC1E;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,gBAAkC;IAClE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;IAC1C,IAAI,KAAK,CAAC;IACV,IAAI,MAAM,EAAE,CAAC;QACT,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,IAAI,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;SAAM,CAAC;QACJ,KAAK,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;IACxB,CAAC;IACD,OAAO,EAAE,GAAG,gBAAgB,EAAE,KAAK,EAAE,CAAC;AAC1C,CAAC"}
package/errors.d.ts CHANGED
@@ -30,4 +30,16 @@ export declare class RetryRequestError extends Error {
30
30
  export declare class SessionError extends RetryRequestError {
31
31
  constructor(message?: string);
32
32
  }
33
+ export declare class ContextPipelineInterruptedError extends Error {
34
+ constructor(message?: string);
35
+ }
36
+ export declare class ContextPipelineInitializationError extends Error {
37
+ constructor(error: unknown, options?: ErrorOptions);
38
+ }
39
+ export declare class ContextPipelineCleanupError extends CriticalError {
40
+ constructor(error: unknown, options?: ErrorOptions);
41
+ }
42
+ export declare class RequestHandlerError extends Error {
43
+ constructor(error: unknown, options?: ErrorOptions);
44
+ }
33
45
  //# sourceMappingURL=errors.d.ts.map
package/errors.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,iBAAiB;gBACnC,OAAO,CAAC,EAAE,MAAM;CAG/B"}
1
+ {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,iBAAiB;gBACnC,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,+BAAgC,SAAQ,KAAK;gBAC1C,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,kCAAmC,SAAQ,KAAK;gBAC7C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,2BAA4B,SAAQ,aAAa;gBAC9C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,mBAAoB,SAAQ,KAAK;gBAC9B,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD"}