@crawlee/basic 4.0.0-beta.85 → 4.0.0-beta.87

package/internals/basic-crawler.d.ts

@@ -1,4 +1,4 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IProxyConfiguration, IRequestLoader, IRequestManager, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier } from '@crawlee/core';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IProxyConfiguration, IRequestLoader, IRequestManager, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier, TypedRequestsLike } from '@crawlee/core';
 import { AutoscaledPool, ContextPipeline, Dataset, RequestQueue, Statistics } from '@crawlee/core';
 import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
@@ -16,17 +16,17 @@ export type RequestHandler<Context extends CrawlingContext = CrawlingContext> =
  * difference between `BaseContext` and `ExtendedContext`) are only available as a `Partial`.
  */
 export type ErrorHandler<BaseContext extends CrawlingContext = CrawlingContext, ExtendedContext extends BaseContext = BaseContext> = (inputs: BaseContext & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
-export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> {
+export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> {
     state: StatisticState;
     crawler: Crawler;
     previousState: StatisticState;
     message: string;
 }
-export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
+export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
 export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
     contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
 };
-export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
+export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
@@ -44,7 +44,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * The exceptions are logged to the request using the
      * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
      */
-    requestHandler?: RequestHandler<ExtendedContext>;
+    requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
     /**
      * Allows the user to extend the crawling context with custom functionality (helpers, references, etc.).
      *
@@ -387,7 +387,7 @@ interface CrawlerIdentity {
     /** Whether `id` came from the user (as opposed to being derived from `instanceIndex`). */
     readonly hasExplicitId: boolean;
 }
-export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
+export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
     #private;
     protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
     /**
@@ -447,7 +447,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
-    readonly router: RouterHandler<Context>;
+    readonly router: RouterHandler<Context, Routes>;
     private _basicContextPipeline?;
     /**
      * The basic part of the context pipeline. Unlike the subclass pipeline, this
@@ -565,7 +565,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
-    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>);
+    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> & RequireContextPipeline<CrawlingContext, Context>);
     /**
      * Determines if the given HTTP status code is an error status code given
      * the default behaviour and user-set preferences.
@@ -617,7 +617,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @param [requests] The requests to add.
      * @param [options] Options for the request queue.
      */
-    run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics>;
+    run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
     /**
      * Gracefully stops the current run of the crawler.
      *
@@ -674,7 +674,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @param requests The requests to add
      * @param options Options for the request queue
      */
-    addRequests(requests: ReadonlyDeep<RequestsLike>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
+    addRequests(requests: ReadonlyDeep<TypedRequestsLike<Routes>>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
     /**
      * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
      */

package/internals/basic-crawler.js

@@ -643,7 +643,7 @@ export class BasicCrawler {
         await purgeDefaultStorages({
             onlyPurgeOnce: true,
             storageBackend: serviceLocator.getStorageBackend(),
-            config: serviceLocator.getConfiguration(),
+            configuration: serviceLocator.getConfiguration(),
         });
         if (requests) {
             await this.addRequests(requests, addRequestsOptions);
@@ -754,7 +754,7 @@ export class BasicCrawler {
         // The first crawler instance uses the default queue (null identifier);
         // subsequent instances get their own queue via a unique alias so they don't collide.
         const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
-        const requestQueue = await RequestQueue.open(identifier, { config: serviceLocator.getConfiguration() });
+        const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
         return this.ownedRequestQueue.set(requestQueue);
     }
     /**
@@ -794,7 +794,7 @@ export class BasicCrawler {
         target.userData = await validateUserData(target.label, schema, target.userData ?? {});
     }
     async useState(defaultValue = {}) {
-        const kvs = await KeyValueStore.open(null, { config: serviceLocator.getConfiguration() });
+        const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
         if (this.identity.hasExplicitId) {
             const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
             return kvs.getAutoSavedValue(stateKey, defaultValue);
@@ -913,7 +913,7 @@ export class BasicCrawler {
      */
     async getDataset(identifier) {
         return Dataset.open(identifier, {
-            config: serviceLocator.getConfiguration(),
+            configuration: serviceLocator.getConfiguration(),
         });
     }
     /**
@@ -1042,7 +1042,7 @@ export class BasicCrawler {
                 await this.requestManager.persistState().catch((err) => {
                     if (err.message.includes('Cannot persist state.')) {
                         this.log.error("The crawler attempted to persist its request list's state and failed due to missing or " +
-                            'invalid config. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
+                            'invalid configuration. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
                             'constructor to ensure your crawling state is persisted through host migrations and restarts.');
                     }
                     else {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.85",
+    "version": "4.0.0-beta.87",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.85",
-        "@crawlee/http-client": "4.0.0-beta.85",
-        "@crawlee/types": "4.0.0-beta.85",
-        "@crawlee/utils": "4.0.0-beta.85",
+        "@crawlee/core": "4.0.0-beta.87",
+        "@crawlee/http-client": "4.0.0-beta.87",
+        "@crawlee/types": "4.0.0-beta.87",
+        "@crawlee/utils": "4.0.0-beta.87",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
         "ow": "^2.0.0",
@@ -54,7 +54,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.85"
+        "@crawlee/impit-client": "^4.0.0-beta.87"
     },
     "lerna": {
         "command": {
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "8d6d3cd9da3dc9cb0b7822b28eb317019b6806bc"
+    "gitHead": "1b7604cd77b694460aac4448b4555444fa1b2bdb"
 }