@crawlee/types 4.0.0-beta.20 → 4.0.0-beta.21

package/http-client.d.ts

@@ -0,0 +1,90 @@
+import type { Readable } from 'node:stream';
+import type { ISession } from './session.js';
+import type { AllowedHttpMethods } from './utility-types.js';
+export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
+interface ToughCookieJar {
+    getCookieString: ((currentUrl: string, options: Record<string, unknown>, callback: (error: Error | null, cookies: string) => void) => string) & ((url: string, callback: (error: Error | null, cookieHeader: string) => void) => string);
+    setCookie: ((cookieOrString: unknown, currentUrl: string, options: Record<string, unknown>, callback: (error: Error | null, cookie: unknown) => void) => void) & ((rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void) => void);
+}
+interface PromiseCookieJar {
+    getCookieString: (url: string) => Promise<string>;
+    setCookie: (rawCookie: string, url: string) => Promise<unknown>;
+}
+/**
+ * HTTP Request as accepted by {@link BaseHttpClient} methods.
+ */
+export interface HttpRequest {
+    url: string | URL;
+    method?: AllowedHttpMethods;
+    headers?: Headers;
+    body?: Readable;
+    signal?: AbortSignal;
+    timeout?: number;
+    cookieJar?: ToughCookieJar | PromiseCookieJar;
+    followRedirect?: boolean | ((response: any) => boolean);
+    maxRedirects?: number;
+    encoding?: BufferEncoding;
+    throwHttpErrors?: boolean;
+    proxyUrl?: string;
+    headerGeneratorOptions?: Record<string, unknown>;
+    useHeaderGenerator?: boolean;
+    headerGenerator?: {
+        getHeaders: (options: Record<string, unknown>) => Record<string, string>;
+    };
+    insecureHTTPParser?: boolean;
+    sessionToken?: object;
+}
+/**
+ * Additional options for HTTP requests that need to be handled separately before passing to {@link BaseHttpClient}.
+ */
+export interface HttpRequestOptions extends HttpRequest {
+    /** Search (query string) parameters to be appended to the request URL */
+    searchParams?: SearchParams;
+    /** A form to be sent in the HTTP request body (URL encoding will be used) */
+    form?: Record<string, string>;
+    /** Arbitrary object to be JSON-serialized and sent as the HTTP request body */
+    json?: unknown;
+    /** Basic HTTP Auth username */
+    username?: string;
+    /** Basic HTTP Auth password */
+    password?: string;
+}
+export interface IResponseWithUrl extends Response {
+    url: string;
+}
+/**
+ * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
+ */
+export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
+    url?: string | URL;
+    headers: Headers;
+}) => void;
+export interface SendRequestOptions {
+    session?: ISession;
+    cookieJar?: ToughCookieJar;
+    timeout?: number;
+    /**
+     * Overrides the proxy URL set in the `session` for this request.
+     *
+     * Note that setting this manually can interfere with session proxy rotation.
+     */
+    proxyUrl?: string;
+}
+export interface StreamOptions extends SendRequestOptions {
+    onRedirect?: RedirectHandler;
+}
+/**
+ * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.
+ */
+export interface BaseHttpClient {
+    /**
+     * Perform an HTTP Request and return the complete response.
+     */
+    sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
+    /**
+     * Perform an HTTP Request and return after the response headers are received. The body may be read from a stream contained in the response.
+     */
+    stream(request: Request, options?: StreamOptions): Promise<Response>;
+}
+export {};
+//# sourceMappingURL=http-client.d.ts.map

package/http-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http-client.d.ts","sourceRoot":"","sources":["../src/http-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE7D,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;AAGnH,UAAU,cAAc;IACpB,eAAe,EAAE,CAAC,CACd,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,KACvD,MAAM,CAAC,GACR,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,YAAY,EAAE,MAAM,KAAK,IAAI,KAAK,MAAM,CAAC,CAAC;IAC7F,SAAS,EAAE,CAAC,CACR,cAAc,EAAE,OAAO,EACvB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,KACvD,IAAI,CAAC,GACN,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;CAC5G;AAED,UAAU,gBAAgB;IACtB,eAAe,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACnE;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;IAClB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAEhB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,SAAS,CAAC,EAAE,cAAc,GAAG,gBAAgB,CAAC;IAC9C,cAAc,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAG1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sBAAsB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,eAAe,CAAC,EAAE;QACd,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC5E,CAAC;IACF,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAmB,SAAQ,WAAW;IACnD,yEAAyE;IACzE,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,6EAA6E;IAC7E,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,+EAA+E;IAC/E,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAiB,SAAQ,QAAQ;IAC9C,GAAG,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,CAC1B,gBAAgB,EAAE,QAAQ,EAC1B,cAAc,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,KACvD,IAAI,CAAC;AAEV,MAAM,WAAW,kBAAkB;IAC/B,OAAO,CAAC,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,cAAc,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAc,SAAQ,kBAAkB;IACrD,UAAU,CAAC,EAAE,eAAe,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE/E;;OAEG;IACH,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;CACxE"}

package/http-client.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=http-client.js.map

package/http-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../src/http-client.ts"],"names":[],"mappings":""}

package/index.d.ts

@@ -1,4 +1,6 @@
 export * from './storages.js';
 export * from './utility-types.js';
 export * from './browser.js';
+export * from './http-client.js';
+export * from './session.js';
 //# sourceMappingURL=index.d.ts.map

package/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC"}

package/index.js

@@ -1,4 +1,6 @@
 export * from './storages.js';
 export * from './utility-types.js';
 export * from './browser.js';
+export * from './http-client.js';
+export * from './session.js';
 //# sourceMappingURL=index.js.map

package/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/types",
-    "version": "4.0.0-beta.20",
+    "version": "4.0.0-beta.21",
     "description": "Shared types for the crawlee projects",
     "engines": {
         "node": ">=22.0.0"
@@ -43,6 +43,7 @@
         "access": "public"
     },
     "dependencies": {
+        "tough-cookie": "^6.0.0",
         "tslib": "^2.8.1"
     },
     "lerna": {
@@ -52,5 +53,5 @@
             }
         }
     },
-    "gitHead": "8e04fea33d3b7ca5a386a941c990e9212275e221"
+    "gitHead": "e370ab2f4ffcf4f63b52a61c9b6e97081d525e64"
 }

package/session.d.ts

@@ -0,0 +1,178 @@
+import type { CookieJar, SerializedCookieJar } from 'tough-cookie';
+import type { Cookie } from './browser.js';
+import type { Dictionary } from './utility-types.js';
+/**
+ * The main purpose of the ProxyInfo object is to provide information
+ * about the current proxy connection used by the crawler for the request.
+ * Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const proxyConfiguration = new ProxyConfiguration({
+ *   proxyUrls: ['...', '...'] // List of Proxy URLs to rotate
+ * });
+ *
+ * // Getting proxyInfo object by calling class method directly
+ * const proxyInfo = await proxyConfiguration.newProxyInfo();
+ *
+ * // In crawler
+ * const crawler = new CheerioCrawler({
+ *   // ...
+ *   proxyConfiguration,
+ *   requestHandler({ proxyInfo }) {
+ *      // Getting used proxy URL
+ *       const proxyUrl = proxyInfo.url;
+ *   }
+ * })
+ *
+ * ```
+ */
+export interface ProxyInfo {
+    /**
+     * The URL of the proxy.
+     */
+    url: string;
+    /**
+     * Username for the proxy.
+     */
+    username?: string;
+    /**
+     * User's password for the proxy.
+     */
+    password: string;
+    /**
+     * Hostname of your proxy.
+     */
+    hostname: string;
+    /**
+     * Proxy port.
+     */
+    port: number | string;
+    /**
+     * Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
+     */
+    proxyTier?: number;
+    /**
+     * When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content.
+     *
+     * @default false
+     */
+    ignoreTlsErrors?: boolean;
+}
+/**
+ * Persistable {@link Session} state.
+ */
+export interface SessionState {
+    id: string;
+    cookieJar: SerializedCookieJar;
+    proxyInfo?: ProxyInfo;
+    userData: object;
+    errorScore: number;
+    maxErrorScore: number;
+    errorScoreDecrement: number;
+    usageCount: number;
+    maxUsageCount: number;
+    expiresAt: string;
+    createdAt: string;
+}
+/**
+ * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
+ * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
+ * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
+ * @category Scaling
+ */
+export interface ISession {
+    readonly id: string;
+    userData: Dictionary;
+    errorScore: number;
+    usageCount: number;
+    maxErrorScore: number;
+    errorScoreDecrement: number;
+    expiresAt: Date;
+    createdAt: Date;
+    maxUsageCount: number;
+    cookieJar: CookieJar;
+    proxyInfo?: ProxyInfo;
+    /**
+     * Indicates whether the session is blocked.
+     * Session is blocked once it reaches the `maxErrorScore`.
+     */
+    isBlocked(): boolean;
+    /**
+     * Indicates whether the session is expired.
+     * Session expiration is determined by the `maxAgeSecs`.
+     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
+     */
+    isExpired(): boolean;
+    /**
+     * Indicates whether the session is used maximum number of times.
+     * Session maximum usage count can be changed by `maxUsageCount` parameter.
+     */
+    isMaxUsageCountReached(): boolean;
+    /**
+     * Indicates whether the session can be used for next requests.
+     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
+     */
+    isUsable(): boolean;
+    /**
+     * This method should be called after a successful session usage.
+     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
+     */
+    markGood(): void;
+    /**
+     * Gets session state for persistence in KeyValueStore.
+     * @returns Represents session internal state.
+     */
+    getState(): SessionState;
+    /**
+     * Marks session as blocked and emits event on the `SessionPool`
+     * This method should be used if the session usage was unsuccessful
+     * and you are sure that it is because of the session configuration and not any external matters.
+     * For example when server returns 403 status code.
+     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
+     */
+    retire(): void;
+    /**
+     * Increases usage and error count.
+     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
+     */
+    markBad(): void;
+    /**
+     * With certain status codes: `401`, `403` or `429` we can be certain
+     * that the target website is blocking us. This function helps to do this conveniently
+     * by retiring the session when such code is received. Optionally, the default status
+     * codes can be extended in the second parameter.
+     * @param statusCode HTTP status code.
+     * @returns Whether the session was retired.
+     */
+    retireOnBlockedStatusCodes(statusCode: number): boolean;
+    /**
+     * Saves cookies from an HTTP response to be used with the session.
+     * It expects an object with a `headers` property that's either an `Object`
+     * (typical Node.js responses) or a `Function` (Puppeteer Response).
+     *
+     * It then parses and saves the cookies from the `set-cookie` header, if available.
+     */
+    setCookiesFromResponse(response: Response): void;
+    /**
+     * Saves an array with cookie objects to be used with the session.
+     * The objects should be in the format that
+     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
+     * but you can also use this function to set cookies manually:
+     *
+     * ```
+     * [
+     *   { name: 'cookie1', value: 'my-cookie' },
+     *   { name: 'cookie2', value: 'your-cookie' }
+     * ]
+     * ```
+     */
+    setCookies(cookies: Cookie[], url: string): void;
+    /**
+     * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
+     * @param url website url. Only cookies stored for this url will be returned
+     */
+    getCookies(url: string): Cookie[];
+}
+//# sourceMappingURL=session.d.ts.map

package/session.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../src/session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEnE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,WAAW,SAAS;IACtB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,mBAAmB,CAAC;IAC/B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,MAAM,WAAW,QAAQ;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,UAAU,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;;OAGG;IACH,SAAS,IAAI,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,IAAI,OAAO,CAAC;IAErB;;;OAGG;IACH,sBAAsB,IAAI,OAAO,CAAC;IAElC;;;OAGG;IACH,QAAQ,IAAI,OAAO,CAAC;IAEpB;;;OAGG;IACH,QAAQ,IAAI,IAAI,CAAC;IAEjB;;;OAGG;IACH,QAAQ,IAAI,YAAY,CAAC;IAEzB;;;;;;OAMG;IACH,MAAM,IAAI,IAAI,CAAC;IAEf;;;OAGG;IACH,OAAO,IAAI,IAAI,CAAC;IAEhB;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;IAExD;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;IAEjD;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAEjD;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CACrC"}

package/session.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=session.js.map

package/session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session.js","sourceRoot":"","sources":["../src/session.ts"],"names":[],"mappings":""}