@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.130

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (280) hide show
  1. package/README.md +17 -13
  2. package/autoscaling/autoscaled_pool.d.ts +67 -172
  3. package/autoscaling/autoscaled_pool.js +182 -329
  4. package/autoscaling/concurrency_system.d.ts +268 -0
  5. package/autoscaling/concurrency_system.js +362 -0
  6. package/autoscaling/cpu_load_signal.d.ts +43 -0
  7. package/autoscaling/cpu_load_signal.js +47 -0
  8. package/autoscaling/event_loop_load_signal.d.ts +51 -0
  9. package/autoscaling/event_loop_load_signal.js +60 -0
  10. package/autoscaling/index.d.ts +6 -1
  11. package/autoscaling/index.js +6 -1
  12. package/autoscaling/load_signal.d.ts +100 -0
  13. package/autoscaling/load_signal.js +105 -0
  14. package/autoscaling/memory_load_signal.d.ts +47 -0
  15. package/autoscaling/memory_load_signal.js +106 -0
  16. package/autoscaling/snapshotter.d.ts +58 -163
  17. package/autoscaling/snapshotter.js +45 -263
  18. package/autoscaling/storage_backend_load_signal.d.ts +56 -0
  19. package/autoscaling/storage_backend_load_signal.js +73 -0
  20. package/autoscaling/system_status.d.ts +67 -89
  21. package/autoscaling/system_status.js +92 -122
  22. package/autoscaling/weighted_avg.d.ts +5 -0
  23. package/autoscaling/weighted_avg.js +14 -0
  24. package/byte_utils.d.ts +17 -0
  25. package/byte_utils.js +42 -0
  26. package/configuration.d.ts +96 -223
  27. package/configuration.js +170 -222
  28. package/cookie_utils.d.ts +3 -2
  29. package/cookie_utils.js +18 -7
  30. package/crawlers/context_pipeline.d.ts +10 -1
  31. package/crawlers/context_pipeline.js +31 -8
  32. package/crawlers/crawler_commons.d.ts +90 -129
  33. package/crawlers/crawler_commons.js +1 -116
  34. package/crawlers/error_snapshotter.d.ts +2 -5
  35. package/crawlers/error_snapshotter.js +7 -8
  36. package/crawlers/error_tracker.d.ts +0 -1
  37. package/crawlers/error_tracker.js +0 -1
  38. package/crawlers/index.d.ts +1 -3
  39. package/crawlers/index.js +0 -3
  40. package/crawlers/internals/types.d.ts +0 -1
  41. package/crawlers/internals/types.js +0 -1
  42. package/crawlers/statistics.d.ts +187 -63
  43. package/crawlers/statistics.js +354 -164
  44. package/debug.d.ts +36 -0
  45. package/debug.js +70 -0
  46. package/enqueue_links/enqueue_links.d.ts +68 -154
  47. package/enqueue_links/enqueue_links.js +38 -210
  48. package/enqueue_links/index.d.ts +0 -1
  49. package/enqueue_links/index.js +0 -1
  50. package/enqueue_links/shared.d.ts +52 -26
  51. package/enqueue_links/shared.js +115 -66
  52. package/errors.d.ts +72 -4
  53. package/errors.js +89 -5
  54. package/events/event_manager.d.ts +35 -9
  55. package/events/event_manager.js +12 -13
  56. package/events/index.d.ts +0 -1
  57. package/events/index.js +0 -1
  58. package/events/local_event_manager.d.ts +15 -3
  59. package/events/local_event_manager.js +39 -13
  60. package/http.d.ts +9 -0
  61. package/http.js +28 -0
  62. package/index.d.ts +7 -4
  63. package/index.js +6 -3
  64. package/iterables.d.ts +79 -0
  65. package/iterables.js +134 -0
  66. package/log.d.ts +82 -3
  67. package/log.js +106 -1
  68. package/memory-storage/consts.d.ts +4 -0
  69. package/memory-storage/consts.js +4 -0
  70. package/memory-storage/index.d.ts +1 -0
  71. package/memory-storage/index.js +1 -0
  72. package/memory-storage/memory-storage.d.ts +38 -0
  73. package/memory-storage/memory-storage.js +130 -0
  74. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  75. package/memory-storage/resource-clients/common/base-client.js +6 -0
  76. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  77. package/memory-storage/resource-clients/dataset.js +108 -0
  78. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  79. package/memory-storage/resource-clients/key-value-store.js +201 -0
  80. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  81. package/memory-storage/resource-clients/request-queue.js +409 -0
  82. package/memory-storage/utils.d.ts +16 -0
  83. package/memory-storage/utils.js +41 -0
  84. package/owned_or_injected.d.ts +58 -0
  85. package/owned_or_injected.js +98 -0
  86. package/package.json +14 -13
  87. package/proxy_configuration.d.ts +24 -132
  88. package/proxy_configuration.js +32 -147
  89. package/recoverable_state.d.ts +155 -0
  90. package/recoverable_state.js +231 -0
  91. package/request.d.ts +80 -16
  92. package/request.js +147 -68
  93. package/router.d.ts +193 -21
  94. package/router.js +188 -43
  95. package/serialization.d.ts +0 -1
  96. package/serialization.js +15 -15
  97. package/service_locator.d.ts +165 -0
  98. package/service_locator.js +253 -0
  99. package/session_pool/consts.d.ts +1 -2
  100. package/session_pool/consts.js +1 -2
  101. package/session_pool/errors.d.ts +0 -1
  102. package/session_pool/errors.js +0 -1
  103. package/session_pool/fingerprint.d.ts +9 -0
  104. package/session_pool/fingerprint.js +30 -0
  105. package/session_pool/index.d.ts +0 -2
  106. package/session_pool/index.js +0 -2
  107. package/session_pool/session.d.ts +35 -88
  108. package/session_pool/session.js +101 -159
  109. package/session_pool/session_pool.d.ts +73 -91
  110. package/session_pool/session_pool.js +172 -165
  111. package/storages/batched_adds.d.ts +37 -0
  112. package/storages/batched_adds.js +73 -0
  113. package/storages/dataset.d.ts +114 -54
  114. package/storages/dataset.js +291 -148
  115. package/storages/index.d.ts +10 -8
  116. package/storages/index.js +8 -8
  117. package/storages/key_value_store.d.ts +185 -42
  118. package/storages/key_value_store.js +445 -169
  119. package/storages/key_value_store_codec.d.ts +32 -0
  120. package/storages/key_value_store_codec.js +113 -0
  121. package/storages/request_dedup_cache.d.ts +22 -0
  122. package/storages/request_dedup_cache.js +48 -0
  123. package/storages/request_list.d.ts +52 -115
  124. package/storages/request_list.js +183 -152
  125. package/storages/request_loader.d.ts +101 -0
  126. package/storages/request_loader.js +1 -0
  127. package/storages/request_manager.d.ts +33 -0
  128. package/storages/request_manager.js +1 -0
  129. package/storages/request_manager_tandem.d.ts +97 -0
  130. package/storages/request_manager_tandem.js +197 -0
  131. package/storages/request_queue.d.ts +290 -47
  132. package/storages/request_queue.js +762 -216
  133. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +45 -89
  134. package/storages/sitemap_request_loader.js +438 -0
  135. package/storages/storage_instance_manager.d.ts +87 -0
  136. package/storages/storage_instance_manager.js +256 -0
  137. package/storages/storage_stats.d.ts +48 -0
  138. package/storages/storage_stats.js +29 -0
  139. package/storages/throttling_request_manager.d.ts +239 -0
  140. package/storages/throttling_request_manager.js +646 -0
  141. package/storages/transaction.d.ts +252 -0
  142. package/storages/transaction.js +251 -0
  143. package/storages/utils.d.ts +59 -11
  144. package/storages/utils.js +75 -15
  145. package/system-info/cpu-info.d.ts +67 -0
  146. package/system-info/cpu-info.js +216 -0
  147. package/system-info/memory-info.d.ts +31 -0
  148. package/system-info/memory-info.js +115 -0
  149. package/system-info/ps-tree.d.ts +17 -0
  150. package/system-info/ps-tree.js +144 -0
  151. package/system-info/runtime.d.ts +14 -0
  152. package/system-info/runtime.js +80 -0
  153. package/typedefs.d.ts +0 -6
  154. package/typedefs.js +0 -1
  155. package/url.d.ts +9 -0
  156. package/url.js +11 -0
  157. package/validators.d.ts +22 -18
  158. package/validators.js +13 -18
  159. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  160. package/autoscaling/autoscaled_pool.js.map +0 -1
  161. package/autoscaling/index.d.ts.map +0 -1
  162. package/autoscaling/index.js.map +0 -1
  163. package/autoscaling/snapshotter.d.ts.map +0 -1
  164. package/autoscaling/snapshotter.js.map +0 -1
  165. package/autoscaling/system_status.d.ts.map +0 -1
  166. package/autoscaling/system_status.js.map +0 -1
  167. package/configuration.d.ts.map +0 -1
  168. package/configuration.js.map +0 -1
  169. package/cookie_utils.d.ts.map +0 -1
  170. package/cookie_utils.js.map +0 -1
  171. package/crawlers/context_pipeline.d.ts.map +0 -1
  172. package/crawlers/context_pipeline.js.map +0 -1
  173. package/crawlers/crawler_commons.d.ts.map +0 -1
  174. package/crawlers/crawler_commons.js.map +0 -1
  175. package/crawlers/crawler_utils.d.ts +0 -10
  176. package/crawlers/crawler_utils.d.ts.map +0 -1
  177. package/crawlers/crawler_utils.js +0 -12
  178. package/crawlers/crawler_utils.js.map +0 -1
  179. package/crawlers/error_snapshotter.d.ts.map +0 -1
  180. package/crawlers/error_snapshotter.js.map +0 -1
  181. package/crawlers/error_tracker.d.ts.map +0 -1
  182. package/crawlers/error_tracker.js.map +0 -1
  183. package/crawlers/index.d.ts.map +0 -1
  184. package/crawlers/index.js.map +0 -1
  185. package/crawlers/internals/types.d.ts.map +0 -1
  186. package/crawlers/internals/types.js.map +0 -1
  187. package/crawlers/statistics.d.ts.map +0 -1
  188. package/crawlers/statistics.js.map +0 -1
  189. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  190. package/enqueue_links/enqueue_links.js.map +0 -1
  191. package/enqueue_links/index.d.ts.map +0 -1
  192. package/enqueue_links/index.js.map +0 -1
  193. package/enqueue_links/shared.d.ts.map +0 -1
  194. package/enqueue_links/shared.js.map +0 -1
  195. package/errors.d.ts.map +0 -1
  196. package/errors.js.map +0 -1
  197. package/events/event_manager.d.ts.map +0 -1
  198. package/events/event_manager.js.map +0 -1
  199. package/events/index.d.ts.map +0 -1
  200. package/events/index.js.map +0 -1
  201. package/events/local_event_manager.d.ts.map +0 -1
  202. package/events/local_event_manager.js.map +0 -1
  203. package/http_clients/base-http-client.d.ts +0 -140
  204. package/http_clients/base-http-client.d.ts.map +0 -1
  205. package/http_clients/base-http-client.js +0 -40
  206. package/http_clients/base-http-client.js.map +0 -1
  207. package/http_clients/form-data-like.d.ts +0 -67
  208. package/http_clients/form-data-like.d.ts.map +0 -1
  209. package/http_clients/form-data-like.js +0 -5
  210. package/http_clients/form-data-like.js.map +0 -1
  211. package/http_clients/got-scraping-http-client.d.ts +0 -20
  212. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  213. package/http_clients/got-scraping-http-client.js +0 -82
  214. package/http_clients/got-scraping-http-client.js.map +0 -1
  215. package/http_clients/index.d.ts +0 -3
  216. package/http_clients/index.d.ts.map +0 -1
  217. package/http_clients/index.js +0 -3
  218. package/http_clients/index.js.map +0 -1
  219. package/index.d.ts.map +0 -1
  220. package/index.js.map +0 -1
  221. package/log.d.ts.map +0 -1
  222. package/log.js.map +0 -1
  223. package/proxy_configuration.d.ts.map +0 -1
  224. package/proxy_configuration.js.map +0 -1
  225. package/request.d.ts.map +0 -1
  226. package/request.js.map +0 -1
  227. package/router.d.ts.map +0 -1
  228. package/router.js.map +0 -1
  229. package/serialization.d.ts.map +0 -1
  230. package/serialization.js.map +0 -1
  231. package/session_pool/consts.d.ts.map +0 -1
  232. package/session_pool/consts.js.map +0 -1
  233. package/session_pool/errors.d.ts.map +0 -1
  234. package/session_pool/errors.js.map +0 -1
  235. package/session_pool/events.d.ts +0 -3
  236. package/session_pool/events.d.ts.map +0 -1
  237. package/session_pool/events.js +0 -3
  238. package/session_pool/events.js.map +0 -1
  239. package/session_pool/index.d.ts.map +0 -1
  240. package/session_pool/index.js.map +0 -1
  241. package/session_pool/session.d.ts.map +0 -1
  242. package/session_pool/session.js.map +0 -1
  243. package/session_pool/session_pool.d.ts.map +0 -1
  244. package/session_pool/session_pool.js.map +0 -1
  245. package/storages/access_checking.d.ts +0 -13
  246. package/storages/access_checking.d.ts.map +0 -1
  247. package/storages/access_checking.js +0 -14
  248. package/storages/access_checking.js.map +0 -1
  249. package/storages/dataset.d.ts.map +0 -1
  250. package/storages/dataset.js.map +0 -1
  251. package/storages/index.d.ts.map +0 -1
  252. package/storages/index.js.map +0 -1
  253. package/storages/key_value_store.d.ts.map +0 -1
  254. package/storages/key_value_store.js.map +0 -1
  255. package/storages/request_list.d.ts.map +0 -1
  256. package/storages/request_list.js.map +0 -1
  257. package/storages/request_provider.d.ts +0 -308
  258. package/storages/request_provider.d.ts.map +0 -1
  259. package/storages/request_provider.js +0 -555
  260. package/storages/request_provider.js.map +0 -1
  261. package/storages/request_queue.d.ts.map +0 -1
  262. package/storages/request_queue.js.map +0 -1
  263. package/storages/request_queue_v2.d.ts +0 -87
  264. package/storages/request_queue_v2.d.ts.map +0 -1
  265. package/storages/request_queue_v2.js +0 -438
  266. package/storages/request_queue_v2.js.map +0 -1
  267. package/storages/sitemap_request_list.d.ts.map +0 -1
  268. package/storages/sitemap_request_list.js +0 -430
  269. package/storages/sitemap_request_list.js.map +0 -1
  270. package/storages/storage_manager.d.ts +0 -58
  271. package/storages/storage_manager.d.ts.map +0 -1
  272. package/storages/storage_manager.js +0 -105
  273. package/storages/storage_manager.js.map +0 -1
  274. package/storages/utils.d.ts.map +0 -1
  275. package/storages/utils.js.map +0 -1
  276. package/tsconfig.build.tsbuildinfo +0 -1
  277. package/typedefs.d.ts.map +0 -1
  278. package/typedefs.js.map +0 -1
  279. package/validators.d.ts.map +0 -1
  280. package/validators.js.map +0 -1
package/router.d.ts CHANGED
@@ -1,14 +1,91 @@
1
- import type { Dictionary } from '@crawlee/types';
2
- import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons.js';
1
+ import type { Awaitable, Dictionary } from '@crawlee/types';
2
+ import type { StandardSchemaV1 } from '@standard-schema/spec';
3
+ import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/crawler_commons.js';
3
4
  import type { Request } from './request.js';
4
- import type { Awaitable } from './typedefs.js';
5
- export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext> extends Router<Context> {
5
+ /**
6
+ * The key of the default route the fallback handler registered via {@link Router.addDefaultHandler}.
7
+ * Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
8
+ * that falls through to the default handler (i.e. whose label has no route of its own).
9
+ */
10
+ export declare const defaultRoute: unique symbol;
11
+ /**
12
+ * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`, and
13
+ * `addRequests`/`enqueueLinks` typed according to the router's route map (`Routes`) so that enqueuing a
14
+ * request under a declared label requires the matching `userData` shape.
15
+ */
16
+ export type RouterHandlerContext<Context, UserData extends Dictionary, Routes extends Record<keyof Routes, Dictionary>> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
17
+ request: LoadedRequest<Request<UserData>>;
18
+ addRequests: TypedContextAddRequests<Routes>;
19
+ } & (Context extends {
20
+ enqueueLinks: infer EnqueueLinks;
21
+ } ? {
22
+ enqueueLinks: TypedContextEnqueueLinks<EnqueueLinks, Routes>;
23
+ } : {});
24
+ /**
25
+ * A map of request labels to a [Standard Schema](https://standardschema.dev) (Zod, Valibot, ArkType, …)
26
+ * validating that label's `request.userData`. Pass it to {@link Router.create} or a `createXRouter`
27
+ * factory to derive the per-label `request.userData` types *and* validate them at runtime. The optional
28
+ * {@link defaultRoute} key registers a schema for requests handled by the default route.
29
+ */
30
+ export type RouteSchemas = Record<string, StandardSchemaV1> & {
31
+ [defaultRoute]?: StandardSchemaV1;
32
+ };
33
+ /** Infers a label's `userData` type from its schema, falling back to a plain {@link Dictionary}. */
34
+ type SchemaUserData<Schema extends StandardSchemaV1> = StandardSchemaV1.InferOutput<Schema> extends Dictionary ? StandardSchemaV1.InferOutput<Schema> : Dictionary;
35
+ /**
36
+ * Derives a route map (label → `userData` type) from a {@link RouteSchemas} map by inferring each schema's
37
+ * output type. Outputs that are not object-shaped fall back to a plain {@link Dictionary}. The
38
+ * {@link defaultRoute} schema is kept under its symbol key so {@link Router.addDefaultHandler} can pick it
39
+ * up; string labels (the ones {@link Router.addHandler} and the crawler-level typing accept) ignore it.
40
+ */
41
+ export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
42
+ [Label in Extract<keyof Schemas, string>]: SchemaUserData<Schemas[Label]>;
43
+ } & (Schemas extends {
44
+ [defaultRoute]: StandardSchemaV1;
45
+ } ? {
46
+ [defaultRoute]: SchemaUserData<Schemas[typeof defaultRoute]>;
47
+ } : {});
48
+ /**
49
+ * The `userData` type of the default route: inferred from the {@link defaultRoute} schema when the route map
50
+ * carries one, otherwise the provided `Fallback`.
51
+ */
52
+ export type DefaultRouteUserData<Routes, Fallback extends Dictionary> = Routes extends {
53
+ [defaultRoute]: infer DefaultUserData extends Dictionary;
54
+ } ? DefaultUserData : Fallback;
55
+ /**
56
+ * Validates `userData` against a {@link RouteSchemas|Standard Schema}, returning the parsed (and coerced)
57
+ * value. Throws a {@link RequestValidationError} when validation fails.
58
+ * @internal
59
+ */
60
+ export declare function validateUserData(label: string | symbol, schema: StandardSchemaV1, userData: unknown): Promise<Dictionary>;
61
+ /**
62
+ * The set of labels accepted by {@link Router.addHandler}. When the router declares a concrete
63
+ * route map (e.g. `{ PRODUCT: ...; CATEGORY: ... }`), only those labels (plus symbols) are
64
+ * allowed — unknown labels become a compile-time error. When the map is left open (the default
65
+ * `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
66
+ */
67
+ export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;
68
+ export interface RouterHandler<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
6
69
  (ctx: Context): Awaitable<void>;
7
70
  }
8
71
  export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
9
- export type RouterRoutes<Context, UserData extends Dictionary> = {
10
- [label in string | symbol]: (ctx: Omit<Context, 'request'> & {
11
- request: Request<UserData>;
72
+ /**
73
+ * Per-route overrides, passed as the last argument of {@link Router.addHandler|`addHandler`} and
74
+ * {@link Router.addDefaultHandler|`addDefaultHandler`}.
75
+ */
76
+ export interface RouteOptions {
77
+ /**
78
+ * Overrides the crawler's `requestHandlerTimeoutSecs` for this route only. Useful when one kind of page
79
+ * needs markedly more time than the rest - a listing page behind an infinite scroll, say - and you do not
80
+ * want to raise the timeout for every other page to accommodate it.
81
+ *
82
+ * Applies only to this route's handler. The navigation and the navigation hooks keep their own timeouts.
83
+ */
84
+ requestHandlerTimeoutSecs?: number;
85
+ }
86
+ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
87
+ [Label in keyof Routes]: (ctx: Omit<Context, 'request'> & {
88
+ request: Request<Routes[Label]>;
12
89
  }) => Awaitable<void>;
13
90
  };
14
91
  /**
@@ -75,36 +152,129 @@ export type RouterRoutes<Context, UserData extends Dictionary> = {
75
152
  * ctx.log.info('...');
76
153
  * });
77
154
  * ```
155
+ *
156
+ * To get `request.userData` typed per label, declare a route map and pass it as the second
157
+ * type argument. The label passed to {@link Router.addHandler} then drives the type of
158
+ * `request.userData`, and unknown labels are rejected at compile time:
159
+ *
160
+ * ```ts
161
+ * import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
162
+ *
163
+ * interface Routes {
164
+ * PRODUCT: { sku: string; price: number };
165
+ * CATEGORY: { categoryId: string };
166
+ * }
167
+ *
168
+ * const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
169
+ *
170
+ * router.addHandler('PRODUCT', async ({ request }) => {
171
+ * request.userData.sku; // string
172
+ * request.userData.price; // number
173
+ * });
174
+ *
175
+ * router.addHandler('TYPO', async () => {}); // compile error: not a known label
176
+ * ```
177
+ *
178
+ * Passing a [Standard Schema](https://standardschema.dev) per label instead of a plain type both infers the
179
+ * `request.userData` types *and* validates them at runtime — when the request is handled, and when it is
180
+ * added to the crawler (`crawler.addRequests`, `context.addRequests`, `enqueueLinks`). A failing request
181
+ * throws a {@link RequestValidationError}.
182
+ *
183
+ * ```ts
184
+ * import { z } from 'zod';
185
+ * import { createCheerioRouter } from 'crawlee';
186
+ *
187
+ * const router = createCheerioRouter({
188
+ * PRODUCT: z.object({ sku: z.string(), price: z.number() }),
189
+ * CATEGORY: z.object({ categoryId: z.string() }),
190
+ * });
191
+ *
192
+ * router.addHandler('PRODUCT', async ({ request }) => {
193
+ * request.userData.price; // number, inferred from the schema and validated at runtime
194
+ * });
195
+ * ```
196
+ *
197
+ * A single route can take longer than the rest without raising the crawler-wide
198
+ * `requestHandlerTimeoutSecs` for everything - pass a per-route timeout as the last argument:
199
+ *
200
+ * ```ts
201
+ * // LIST pages scroll through a lot of content, DETAIL pages are quick
202
+ * router.addHandler('LIST', async (ctx) => { ... }, { requestHandlerTimeoutSecs: 120 });
203
+ * router.addHandler('DETAIL', async (ctx) => { ... }); // keeps the crawler's default
204
+ * ```
205
+ *
206
+ * When the time a route needs is only apparent once it is already running, call
207
+ * {@link CrawlingContext.extendTimeout|`context.extendTimeout`} from inside the handler:
208
+ *
209
+ * ```ts
210
+ * router.addHandler('LIST', async ({ page, extendTimeout }) => {
211
+ * const pageCount = await countPages(page);
212
+ * extendTimeout(pageCount * 10); // ask for 10 more seconds per page
213
+ * await scrapeAllPages(page);
214
+ * });
215
+ * ```
78
216
  */
79
- export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>> {
80
- private readonly routes;
81
- private readonly middlewares;
217
+ export declare class Router<Context extends RestrictedCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
218
+ #private;
82
219
  /**
83
220
  * use Router.create() instead!
84
221
  * @ignore
85
222
  */
86
- protected constructor();
223
+ private constructor();
224
+ /**
225
+ * Registers new route handler for given label. When the router declares a route map, the
226
+ * `label` is restricted to the declared labels and `request.userData` is typed accordingly. Pass
227
+ * {@link RouteOptions|`options`} to give this route its own `requestHandlerTimeoutSecs`,
228
+ * overriding the crawler's default for requests with this label.
229
+ */
230
+ addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>, options?: RouteOptions): void;
231
+ /**
232
+ * Registers new route handler for given label, explicitly typing `request.userData` via the
233
+ * `UserData` type argument. Useful when the router has no declared route map (the open default)
234
+ * and you want to type a single handler, or to register a handler under a `symbol` label.
235
+ */
236
+ addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options?: RouteOptions): void;
87
237
  /**
88
- * Registers new route handler for given label.
238
+ * Registers default route handler. As a fallback it can receive any request (including labels not
239
+ * declared in the route map). When the router was created with a {@link defaultRoute} schema,
240
+ * `request.userData` is typed from it; otherwise it defaults to the context's (loosely typed) `userData`.
241
+ * Pass an explicit `UserData` type argument to narrow it. Pass {@link RouteOptions|`options`} to give the
242
+ * default route its own `requestHandlerTimeoutSecs`, overriding the crawler's default for requests that fall
243
+ * through to it.
89
244
  */
90
- addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: string | symbol, handler: (ctx: Omit<Context, 'request'> & {
91
- request: LoadedRequest<Request<UserData>>;
92
- }) => Awaitable<void>): void;
245
+ addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options?: RouteOptions): void;
93
246
  /**
94
- * Registers default route handler.
247
+ * Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
248
+ * to validate `request.userData` when requests are added.
249
+ * @internal
95
250
  */
96
- addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: Omit<Context, 'request'> & {
97
- request: LoadedRequest<Request<UserData>>;
98
- }) => Awaitable<void>): void;
251
+ getSchema(label?: string | symbol): StandardSchemaV1 | undefined;
99
252
  /**
100
253
  * Registers a middleware that will be fired before the matching route handler.
101
254
  * Multiple middlewares can be registered, they will be fired in the same order.
102
255
  */
103
256
  use(middleware: (ctx: Context) => Awaitable<void>): void;
257
+ /**
258
+ * Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
259
+ * override it and the crawler's own timeout should apply. Falls back to the default route the same way
260
+ * {@link Router.getHandler|`getHandler`} does, so a label with no route of its own inherits whatever
261
+ * the default route asked for. Used by the crawler; not meant to be called directly.
262
+ */
263
+ getTimeoutSecs(label?: string | symbol): number | undefined;
264
+ /**
265
+ * The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
266
+ * The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
267
+ */
268
+ getMaxTimeoutSecs(): number | undefined;
104
269
  /**
105
270
  * Returns route handler for given label. If no label is provided, the default request handler will be returned.
106
271
  */
107
272
  getHandler(label?: string | symbol): (ctx: Context) => Awaitable<void>;
273
+ /**
274
+ * Validates `request.userData` against the schema registered for its label (if any), replacing it with
275
+ * the parsed value. Throws a {@link RequestValidationError} when validation fails.
276
+ */
277
+ private validateRequest;
108
278
  /**
109
279
  * Throws when the label already exists in our registry.
110
280
  */
@@ -129,6 +299,8 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
129
299
  * await crawler.run();
130
300
  * ```
131
301
  */
132
- static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
302
+ static create<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
303
+ static create<Context extends RestrictedCrawlingContext = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
304
+ static create<Context extends RestrictedCrawlingContext = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
133
305
  }
134
- //# sourceMappingURL=router.d.ts.map
306
+ export {};
package/router.js CHANGED
@@ -1,5 +1,39 @@
1
- import { MissingRouteError } from './errors.js';
2
- const defaultRoute = Symbol('default-route');
1
+ import { MissingRouteError, RequestValidationError } from './errors.js';
2
+ /**
3
+ * The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
4
+ * Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
5
+ * that falls through to the default handler (i.e. whose label has no route of its own).
6
+ */
7
+ export const defaultRoute = Symbol('default-route');
8
+ /** Whether a validation issue points at the top-level `label` key. */
9
+ function isLabelIssue(issue) {
10
+ if (issue.path?.length !== 1) {
11
+ return false;
12
+ }
13
+ const [segment] = issue.path;
14
+ return (typeof segment === 'object' ? segment.key : segment) === 'label';
15
+ }
16
+ /**
17
+ * Validates `userData` against a {@link RouteSchemas|Standard Schema}, returning the parsed (and coerced)
18
+ * value. Throws a {@link RequestValidationError} when validation fails.
19
+ * @internal
20
+ */
21
+ export async function validateUserData(label, schema, userData) {
22
+ const { label: _label, ...rest } = (userData ?? {});
23
+ // `label` is a Crawlee-managed key that lives inside `userData`, so validating it is opt-in: we validate
24
+ // without it first, letting schemas that don't describe it pass (including `.strict()` ones). A schema that
25
+ // *does* declare `label` reports an issue for the now-missing key — so we re-validate with it included,
26
+ // honouring the declaration. Unlike `userData.__crawlee`, `label` is enumerable, so schemas do see it.
27
+ let result = await schema['~standard'].validate(rest);
28
+ if (result.issues?.some(isLabelIssue)) {
29
+ result = await schema['~standard'].validate({ ...rest, label });
30
+ }
31
+ if (result.issues) {
32
+ throw new RequestValidationError(label, result.issues);
33
+ }
34
+ // Restore the label so it survives schemas that strip undeclared keys.
35
+ return { ...result.value, label };
36
+ }
3
37
  /**
4
38
  * Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.
5
39
  *
@@ -64,95 +98,207 @@ const defaultRoute = Symbol('default-route');
64
98
  * ctx.log.info('...');
65
99
  * });
66
100
  * ```
101
+ *
102
+ * To get `request.userData` typed per label, declare a route map and pass it as the second
103
+ * type argument. The label passed to {@link Router.addHandler} then drives the type of
104
+ * `request.userData`, and unknown labels are rejected at compile time:
105
+ *
106
+ * ```ts
107
+ * import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
108
+ *
109
+ * interface Routes {
110
+ * PRODUCT: { sku: string; price: number };
111
+ * CATEGORY: { categoryId: string };
112
+ * }
113
+ *
114
+ * const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
115
+ *
116
+ * router.addHandler('PRODUCT', async ({ request }) => {
117
+ * request.userData.sku; // string
118
+ * request.userData.price; // number
119
+ * });
120
+ *
121
+ * router.addHandler('TYPO', async () => {}); // compile error: not a known label
122
+ * ```
123
+ *
124
+ * Passing a [Standard Schema](https://standardschema.dev) per label instead of a plain type both infers the
125
+ * `request.userData` types *and* validates them at runtime — when the request is handled, and when it is
126
+ * added to the crawler (`crawler.addRequests`, `context.addRequests`, `enqueueLinks`). A failing request
127
+ * throws a {@link RequestValidationError}.
128
+ *
129
+ * ```ts
130
+ * import { z } from 'zod';
131
+ * import { createCheerioRouter } from 'crawlee';
132
+ *
133
+ * const router = createCheerioRouter({
134
+ * PRODUCT: z.object({ sku: z.string(), price: z.number() }),
135
+ * CATEGORY: z.object({ categoryId: z.string() }),
136
+ * });
137
+ *
138
+ * router.addHandler('PRODUCT', async ({ request }) => {
139
+ * request.userData.price; // number, inferred from the schema and validated at runtime
140
+ * });
141
+ * ```
142
+ *
143
+ * A single route can take longer than the rest without raising the crawler-wide
144
+ * `requestHandlerTimeoutSecs` for everything - pass a per-route timeout as the last argument:
145
+ *
146
+ * ```ts
147
+ * // LIST pages scroll through a lot of content, DETAIL pages are quick
148
+ * router.addHandler('LIST', async (ctx) => { ... }, { requestHandlerTimeoutSecs: 120 });
149
+ * router.addHandler('DETAIL', async (ctx) => { ... }); // keeps the crawler's default
150
+ * ```
151
+ *
152
+ * When the time a route needs is only apparent once it is already running, call
153
+ * {@link CrawlingContext.extendTimeout|`context.extendTimeout`} from inside the handler:
154
+ *
155
+ * ```ts
156
+ * router.addHandler('LIST', async ({ page, extendTimeout }) => {
157
+ * const pageCount = await countPages(page);
158
+ * extendTimeout(pageCount * 10); // ask for 10 more seconds per page
159
+ * await scrapeAllPages(page);
160
+ * });
161
+ * ```
67
162
  */
68
163
  export class Router {
69
- routes = new Map();
70
- middlewares = [];
164
+ #routes = new Map();
165
+ #schemas = new Map();
166
+ #timeouts = new Map();
167
+ #middlewares = [];
71
168
  /**
72
169
  * use Router.create() instead!
73
170
  * @ignore
74
171
  */
75
172
  constructor() { }
76
- /**
77
- * Registers new route handler for given label.
78
- */
79
- addHandler(label, handler) {
173
+ addHandler(label, handler, options = {}) {
80
174
  this.validate(label);
81
- this.routes.set(label, handler);
175
+ this.#routes.set(label, handler);
176
+ if (options.requestHandlerTimeoutSecs !== undefined) {
177
+ this.#timeouts.set(label, options.requestHandlerTimeoutSecs);
178
+ }
82
179
  }
83
180
  /**
84
- * Registers default route handler.
181
+ * Registers default route handler. As a fallback it can receive any request (including labels not
182
+ * declared in the route map). When the router was created with a {@link defaultRoute} schema,
183
+ * `request.userData` is typed from it; otherwise it defaults to the context's (loosely typed) `userData`.
184
+ * Pass an explicit `UserData` type argument to narrow it. Pass {@link RouteOptions|`options`} to give the
185
+ * default route its own `requestHandlerTimeoutSecs`, overriding the crawler's default for requests that fall
186
+ * through to it.
85
187
  */
86
- addDefaultHandler(handler) {
188
+ addDefaultHandler(handler, options = {}) {
87
189
  this.validate(defaultRoute);
88
- this.routes.set(defaultRoute, handler);
190
+ this.#routes.set(defaultRoute, handler);
191
+ if (options.requestHandlerTimeoutSecs !== undefined) {
192
+ this.#timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
193
+ }
194
+ }
195
+ /**
196
+ * Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
197
+ * to validate `request.userData` when requests are added.
198
+ * @internal
199
+ */
200
+ getSchema(label) {
201
+ if (label != null) {
202
+ const schema = this.#schemas.get(label);
203
+ if (schema) {
204
+ return schema;
205
+ }
206
+ // A label with its own route is fully specified; don't fall back to the default-route schema.
207
+ if (this.#routes.has(label)) {
208
+ return undefined;
209
+ }
210
+ }
211
+ // Requests with no route of their own fall through to the default handler, so validate their
212
+ // `userData` against the default-route schema, if one was registered.
213
+ return this.#schemas.get(defaultRoute);
89
214
  }
90
215
  /**
91
216
  * Registers a middleware that will be fired before the matching route handler.
92
217
  * Multiple middlewares can be registered, they will be fired in the same order.
93
218
  */
94
219
  use(middleware) {
95
- this.middlewares.push(middleware);
220
+ this.#middlewares.push(middleware);
221
+ }
222
+ /**
223
+ * Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
224
+ * override it and the crawler's own timeout should apply. Falls back to the default route the same way
225
+ * {@link Router.getHandler|`getHandler`} does, so a label with no route of its own inherits whatever
226
+ * the default route asked for. Used by the crawler; not meant to be called directly.
227
+ */
228
+ getTimeoutSecs(label) {
229
+ if (label && this.#routes.has(label)) {
230
+ return this.#timeouts.get(label);
231
+ }
232
+ return this.#timeouts.get(defaultRoute);
233
+ }
234
+ /**
235
+ * The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
236
+ * The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
237
+ */
238
+ getMaxTimeoutSecs() {
239
+ return this.#timeouts.size > 0 ? Math.max(...this.#timeouts.values()) : undefined;
96
240
  }
97
241
  /**
98
242
  * Returns route handler for given label. If no label is provided, the default request handler will be returned.
99
243
  */
100
244
  getHandler(label) {
101
- if (label && this.routes.has(label)) {
102
- return this.routes.get(label);
245
+ if (label && this.#routes.has(label)) {
246
+ return this.#routes.get(label);
103
247
  }
104
- if (this.routes.has(defaultRoute)) {
105
- return this.routes.get(defaultRoute);
248
+ if (this.#routes.has(defaultRoute)) {
249
+ return this.#routes.get(defaultRoute);
106
250
  }
107
251
  throw new MissingRouteError(`Route not found for label '${String(label)}'.` +
108
252
  ' You must set up a route for this label or a default route.' +
109
253
  ' Use `requestHandler`, `router.addHandler` or `router.addDefaultHandler`.');
110
254
  }
255
+ /**
256
+ * Validates `request.userData` against the schema registered for its label (if any), replacing it with
257
+ * the parsed value. Throws a {@link RequestValidationError} when validation fails.
258
+ */
259
+ async validateRequest(context) {
260
+ const label = context.request.label;
261
+ const schema = this.getSchema(label);
262
+ if (schema) {
263
+ context.request.userData = (await validateUserData(label, schema, context.request.userData));
264
+ }
265
+ }
111
266
  /**
112
267
  * Throws when the label already exists in our registry.
113
268
  */
114
269
  validate(label) {
115
- if (this.routes.has(label)) {
270
+ if (this.#routes.has(label)) {
116
271
  const message = label === defaultRoute
117
272
  ? `Default route is already defined!`
118
273
  : `Route for label '${String(label)}' is already defined!`;
119
274
  throw new Error(message);
120
275
  }
121
276
  }
122
- /**
123
- * Creates new router instance. This instance can then serve as a `requestHandler` of your crawler.
124
- *
125
- * ```ts
126
- * import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
127
- *
128
- * const router = Router.create<CheerioCrawlingContext>();
129
- * router.addHandler('label-a', async (ctx) => {
130
- * ctx.log.info('...');
131
- * });
132
- * router.addDefaultHandler(async (ctx) => {
133
- * ctx.log.info('...');
134
- * });
135
- *
136
- * const crawler = new CheerioCrawler({
137
- * requestHandler: router,
138
- * });
139
- * await crawler.run();
140
- * ```
141
- */
142
- static create(routes) {
277
+ static create(routesOrSchemas) {
143
278
  const router = new Router();
144
279
  const obj = Object.create(Function.prototype);
145
280
  obj.addHandler = router.addHandler.bind(router);
146
281
  obj.addDefaultHandler = router.addDefaultHandler.bind(router);
282
+ obj.getSchema = router.getSchema.bind(router);
147
283
  obj.getHandler = router.getHandler.bind(router);
284
+ obj.getTimeoutSecs = router.getTimeoutSecs.bind(router);
285
+ obj.getMaxTimeoutSecs = router.getMaxTimeoutSecs.bind(router);
148
286
  obj.use = router.use.bind(router);
149
- for (const [label, handler] of Object.entries(routes ?? {})) {
150
- router.addHandler(label, handler);
287
+ // `Reflect.ownKeys` (unlike `Object.entries`) also yields the `defaultRoute` symbol key.
288
+ for (const label of Reflect.ownKeys(routesOrSchemas ?? {})) {
289
+ const value = routesOrSchemas[label];
290
+ if (typeof value === 'function') {
291
+ router.addHandler(label, value);
292
+ }
293
+ else {
294
+ router.#schemas.set(label, value);
295
+ }
151
296
  }
152
297
  const func = async function (context) {
153
298
  const { url, loadedUrl, label } = context.request;
154
299
  context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
155
- for (const middleware of router.middlewares) {
300
+ await router.validateRequest(context);
301
+ for (const middleware of router.#middlewares) {
156
302
  await middleware(context);
157
303
  }
158
304
  return router.getHandler(label)(context);
@@ -161,4 +307,3 @@ export class Router {
161
307
  return func;
162
308
  }
163
309
  }
164
- //# sourceMappingURL=router.js.map
@@ -29,4 +29,3 @@ export declare function deserializeArray<T extends string | Buffer>(compressedDa
29
29
  * @internal
30
30
  */
31
31
  export declare function createDeserialize(compressedData: Buffer | Uint8Array): Readable;
32
- //# sourceMappingURL=serialization.d.ts.map
package/serialization.js CHANGED
@@ -1,9 +1,11 @@
1
1
  import { pipeline as streamPipeline, Readable, Writable } from 'node:stream';
2
2
  import util from 'node:util';
3
3
  import zlib from 'node:zlib';
4
- import ow from 'ow';
5
4
  import StreamArray from 'stream-json/streamers/StreamArray.js';
5
+ import { z } from 'zod';
6
+ import { parseArgument, schemas } from './validators.js';
6
7
  const pipeline = util.promisify(streamPipeline);
8
+ const uint8ArraySchema = z.instanceof(Uint8Array);
7
9
  /**
8
10
  * Transforms an array of items to a JSON in a streaming
9
11
  * fashion to save memory. It operates in batches to speed
@@ -11,32 +13,31 @@ const pipeline = util.promisify(streamPipeline);
11
13
  * @internal
12
14
  */
13
15
  class ArrayToJson extends Readable {
14
- data;
15
- offset = 0;
16
- batchSize;
16
+ #offset = 0;
17
+ #batchSize;
18
+ #data;
17
19
  constructor(data, options = {}) {
18
20
  super({
19
21
  ...options,
20
22
  autoDestroy: true,
21
23
  emitClose: true,
22
24
  });
23
- this.data = data;
24
25
  const { batchSize = 10000 } = options;
25
- this.batchSize = batchSize;
26
- this.data = data;
26
+ this.#batchSize = batchSize;
27
+ this.#data = data;
27
28
  this.push('[');
28
29
  }
29
30
  _read() {
30
31
  try {
31
- const items = this.data.slice(this.offset, this.offset + this.batchSize);
32
+ const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);
32
33
  if (items.length) {
33
34
  const json = JSON.stringify(items);
34
35
  // Strip brackets to flatten the batch.
35
36
  const itemString = json.substring(1, json.length - 1);
36
- if (this.offset > 0)
37
+ if (this.#offset > 0)
37
38
  this.push(',', 'utf8');
38
39
  this.push(itemString, 'utf8');
39
- this.offset += this.batchSize;
40
+ this.#offset += this.#batchSize;
40
41
  }
41
42
  else {
42
43
  this.push(']');
@@ -58,7 +59,7 @@ class ArrayToJson extends Readable {
58
59
  * @internal
59
60
  */
60
61
  export async function serializeArray(data) {
61
- ow(data, ow.array);
62
+ parseArgument(data, schemas.anyArray);
62
63
  const { chunks, collector } = createChunkCollector();
63
64
  await pipeline(new ArrayToJson(data), zlib.createGzip(), collector);
64
65
  return Buffer.concat(chunks);
@@ -73,7 +74,7 @@ export async function serializeArray(data) {
73
74
  * @internal
74
75
  */
75
76
  export async function deserializeArray(compressedData) {
76
- ow(compressedData, ow.uint8Array);
77
+ parseArgument(compressedData, uint8ArraySchema);
77
78
  const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
78
79
  await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);
79
80
  return chunks;
@@ -88,7 +89,7 @@ export async function deserializeArray(compressedData) {
88
89
  * @internal
89
90
  */
90
91
  export function createDeserialize(compressedData) {
91
- ow(compressedData, ow.uint8Array);
92
+ parseArgument(compressedData, uint8ArraySchema);
92
93
  const streamArray = StreamArray.withParser();
93
94
  const destination = pluckValue(streamArray);
94
95
  streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err) => destination.emit(err));
@@ -116,7 +117,6 @@ function createChunkCollector(options = {}) {
116
117
  }
117
118
  function pluckValue(streamArray) {
118
119
  const realPush = streamArray.push.bind(streamArray);
119
- streamArray.push = (obj) => realPush(obj && obj.value);
120
+ streamArray.push = (obj) => realPush(obj?.value ?? null);
120
121
  return streamArray;
121
122
  }
122
- //# sourceMappingURL=serialization.js.map