apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/storages/request_queue.d.ts

@@ -1,318 +0,0 @@
-export const QUERY_HEAD_MIN_LENGTH: 100;
-export const QUERY_HEAD_BUFFER: 3;
-export const API_PROCESSED_REQUESTS_DELAY_MILLIS: number;
-export const MAX_QUERIES_FOR_CONSISTENCY: 6;
-export const STORAGE_CONSISTENCY_DELAY_MILLIS: 3000;
-export function getRequestId(uniqueKey: any): string;
-/**
- * A helper class that is used to report results from various
- * {@link RequestQueue} functions as well as
- * {@link utils#enqueueLinks}.
- *
- * @typedef QueueOperationInfo
- * @property {boolean} wasAlreadyPresent Indicates if request was already present in the queue.
- * @property {boolean} wasAlreadyHandled Indicates if request was already marked as handled.
- * @property {string} requestId The ID of the added request
- * @property {Request} request The original {@link Request} object passed to the `RequestQueue` function.
- */
-/**
- * Represents a queue of URLs to crawl, which is used for deep crawling of websites
- * where you start with several URLs and then recursively
- * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
- *
- * Each URL is represented using an instance of the {@link Request} class.
- * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
- * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
- * To add a single URL multiple times to the queue,
- * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
- *
- * Do not instantiate this class directly, use the
- * {@link Apify#openRequestQueue} function instead.
- *
- * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
- * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
- * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
- *
- * `RequestQueue` stores its data either on local disk or in the Apify Cloud,
- * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set.
- *
- * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the queue data is stored in
- * that directory in an SQLite database file.
- *
- * If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
- * [Apify Request Queue](https://docs.apify.com/storage/request-queue)
- * cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
- * option to {@link Apify#openRequestQueue} function,
- * even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
- *
- * **Example usage:**
- *
- * ```javascript
- * // Open the default request queue associated with the actor run
- * const queue = await Apify.openRequestQueue();
- *
- * // Open a named request queue
- * const queueWithName = await Apify.openRequestQueue('some-name');
- *
- * // Enqueue few requests
- * await queue.addRequest({ url: 'http://example.com/aaa' });
- * await queue.addRequest({ url: 'http://example.com/bbb' });
- * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
- * ```
- * @hideconstructor
- */
-export class RequestQueue {
-    /**
-     * @param {object} options
-     * @param {string} options.id
-     * @param {string} [options.name]
-     * @param {boolean} options.isLocal
-     * @param {ApifyClient|ApifyStorageLocal} options.client
-     */
-    constructor(options: {
-        id: string;
-        name?: string | undefined;
-        isLocal: boolean;
-        client: ApifyClient | ApifyStorageLocal;
-    });
-    id: string;
-    name: string | undefined;
-    isLocal: boolean;
-    timeoutSecs: number;
-    clientKey: string;
-    client: import("@apify/storage-local/dist/resource_clients/request_queue").RequestQueueClient | import("apify-client").RequestQueueClient;
-    log: import("@apify/log/log").Log;
-    /**
-     * @type {*}
-     * @ignore
-     */
-    queueHeadDict: any;
-    queryQueueHeadPromise: any;
-    inProgress: Set<any>;
-    lastActivity: Date;
-    internalTimeoutMillis: number;
-    recentlyHandled: LruCache<any>;
-    assumedTotalCount: number;
-    assumedHandledCount: number;
-    requestsCache: LruCache<any>;
-    /**
-     * @ignore
-     */
-    inProgressCount(): number;
-    /**
-     * Adds a request to the queue.
-     *
-     * If a request with the same `uniqueKey` property is already present in the queue,
-     * it will not be updated. You can find out whether this happened from the resulting
-     * {@link QueueOperationInfo} object.
-     *
-     * To add multiple requests to the queue by extracting links from a webpage,
-     * see the {@link utils#enqueueLinks} helper function.
-     *
-     * @param {(Request|RequestOptions)} requestLike {@link Request} object or vanilla object with request data.
-     * Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
-     * @param {Object} [options]
-     * @param {boolean} [options.forefront=false] If `true`, the request will be added to the foremost position in the queue.
-     * @return {Promise<QueueOperationInfo>}
-     */
-    addRequest(requestLike: (Request | RequestOptions), options?: {
-        forefront?: boolean | undefined;
-    } | undefined): Promise<QueueOperationInfo>;
-    /**
-     * Gets the request from the queue specified by ID.
-     *
-     * @param {string} id ID of the request.
-     * @return {Promise<(Request|null)>} Returns the request object, or `null` if it was not found.
-     */
-    getRequest(id: string): Promise<(Request | null)>;
-    /**
-     * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
-     *
-     * Once you successfully finish processing of the request, you need to call
-     * {@link RequestQueue#markRequestHandled}
-     * to mark the request as handled in the queue. If there was some error in processing the request,
-     * call {@link RequestQueue#reclaimRequest} instead,
-     * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
-     *
-     * Note that the `null` return value doesn't mean the queue processing finished,
-     * it means there are currently no pending requests.
-     * To check whether all requests in queue were finished,
-     * use {@link RequestQueue#isFinished} instead.
-     *
-     * @returns {Promise<(Request|null)>}
-     * Returns the request object or `null` if there are no more pending requests.
-     */
-    fetchNextRequest(): Promise<(Request | null)>;
-    /**
-     * Marks a request that was previously returned by the
-     * {@link RequestQueue#fetchNextRequest}
-     * function as handled after successful processing.
-     * Handled requests will never again be returned by the `fetchNextRequest` function.
-     *
-     * @param {Request} request
-     * @return {Promise<QueueOperationInfo | null>}
-     */
-    markRequestHandled(request: Request): Promise<QueueOperationInfo | null>;
-    /**
-     * Reclaims a failed request back to the queue, so that it can be returned for processed later again
-     * by another call to {@link RequestQueue#fetchNextRequest}.
-     * The request record in the queue is updated using the provided `request` parameter.
-     * For example, this lets you store the number of retries or error messages for the request.
-     *
-     * @param {Request} request
-     * @param {object} [options]
-     * @param {boolean} [options.forefront=false]
-     * If `true` then the request it placed to the beginning of the queue, so that it's returned
-     * in the next call to {@link RequestQueue#fetchNextRequest}.
-     * By default, it's put to the end of the queue.
-     * @return {Promise<QueueOperationInfo | null>}
-     */
-    reclaimRequest(request: Request, options?: {
-        forefront?: boolean | undefined;
-    } | undefined): Promise<QueueOperationInfo | null>;
-    /**
-     * Resolves to `true` if the next call to {@link RequestQueue#fetchNextRequest}
-     * would return `null`, otherwise it resolves to `false`.
-     * Note that even if the queue is empty, there might be some pending requests currently being processed.
-     * If you need to ensure that there is no activity in the queue, use {@link RequestQueue#isFinished}.
-     *
-     * @returns {Promise<boolean>}
-     */
-    isEmpty(): Promise<boolean>;
-    /**
-     * Resolves to `true` if all requests were already handled and there are no more left.
-     * Due to the nature of distributed storage used by the queue,
-     * the function might occasionally return a false negative,
-     * but it will never return a false positive.
-     *
-     * @returns {Promise<boolean>}
-     */
-    isFinished(): Promise<boolean>;
-    /**
-     * @internal
-     */
-    _reset(): void;
-    /**
-     * Caches information about request to beware of unneeded addRequest() calls.
-     * @param {string} cacheKey
-     * @param {object} queueOperationInfo
-     * @param {string} queueOperationInfo.requestId
-     * @param {boolean} queueOperationInfo.wasAlreadyHandled
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _cacheRequest(cacheKey: string, queueOperationInfo: {
-        requestId: string;
-        wasAlreadyHandled: boolean;
-    }): void;
-    /**
-     * We always request more items than is in progress to ensure that something falls into head.
-     *
-     * @param {boolean} [ensureConsistency=false] If true then query for queue head is retried until queueModifiedAt
-     *   is older than queryStartedAt by at least API_PROCESSED_REQUESTS_DELAY_MILLIS to ensure that queue
-     *   head is consistent.
-     * @param {number} [limit] How many queue head items will be fetched.
-     * @param {number} [iteration] Used when this function is called recursively to limit the recursion.
-     * @return {Promise<boolean>} Indicates if queue head is consistent (true) or inconsistent (false).
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _ensureHeadIsNonEmpty(ensureConsistency?: boolean | undefined, limit?: number | undefined, iteration?: number | undefined): Promise<boolean>;
-    /**
-     * Adds a request straight to the queueHeadDict, to improve performance.
-     * @private
-     */
-    private _maybeAddRequestToQueueHead;
-    /**
-     * Removes the queue either from the Apify Cloud storage or from the local database,
-     * depending on the mode of operation.
-     *
-     * @return {Promise<void>}
-     */
-    drop(): Promise<void>;
-    /**
-     * Returns the number of handled requests.
-     *
-     * This function is just a convenient shortcut for:
-     *
-     * ```javascript
-     * const { handledRequestCount } = await queue.getInfo();
-     * ```
-     *
-     * @return {Promise<number>}
-     */
-    handledCount(): Promise<number>;
-    /**
-     * Returns an object containing general information about the request queue.
-     *
-     * The function returns the same object as the Apify API Client's
-     * [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
-     * function, which in turn calls the
-     * [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
-     * API endpoint.
-     *
-     * **Example:**
-     * ```
-     * {
-     *   id: "WkzbQMuFYuamGv3YF",
-     *   name: "my-queue",
-     *   userId: "wRsJZtadYvn4mBZmm",
-     *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
-     *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
-     *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
-     *   totalRequestCount: 25,
-     *   handledRequestCount: 5,
-     *   pendingRequestCount: 20,
-     * }
-     * ```
-     *
-     * @returns {Promise<RequestQueueInfo>}
-     */
-    getInfo(): Promise<RequestQueueInfo>;
-}
-export function openRequestQueue(queueIdOrName?: string | undefined, options?: {
-    forceCloud?: boolean | undefined;
-} | undefined): Promise<RequestQueue>;
-/**
- * A helper class that is used to report results from various
- * {@link RequestQueue } functions as well as
- * {@link utilsenqueueLinks }.
- */
-export type QueueOperationInfo = {
-    /**
-     * Indicates if request was already present in the queue.
-     */
-    wasAlreadyPresent: boolean;
-    /**
-     * Indicates if request was already marked as handled.
-     */
-    wasAlreadyHandled: boolean;
-    /**
-     * The ID of the added request
-     */
-    requestId: string;
-    /**
-     * The original {@link Request } object passed to the `RequestQueue` function.
-     */
-    request: Request;
-};
-export type RequestQueueInfo = {
-    id: string;
-    name: string;
-    userId: string;
-    createdAt: Date;
-    modifiedAt: Date;
-    accessedAt: Date;
-    totalRequestCount: number;
-    handledRequestCount: number;
-    pendingRequestCount: number;
-};
-import { LruCache } from "@apify/datastructures/lru_cache";
-import Request from "../request";
-import { RequestOptions } from "../request";
-import { ApifyClient } from "apify-client/dist/apify_client";
-import { ApifyStorageLocal } from "@apify/storage-local";
-//# sourceMappingURL=request_queue.d.ts.map

package/build/storages/request_queue.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue.js"],"names":[],"mappings":"AAqBA,wCAAyC;AACzC,kCAAmC;AAInC,yDAA6D;AAG7D,4CAA6C;AAQ7C,oDAAqD;AAW9C,qDAQN;AAED;;;;;;;;;;GAUG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH;IACI;;;;;;OAMG;IACH;QAL2B,EAAE,EAAlB,MAAM;QACW,IAAI;QACJ,OAAO,EAAxB,OAAO;QACgC,MAAM,EAA7C,WAAW,GAAC,iBAAiB;OAiDvC;IA9CG,WAAoB;IACpB,yBAAwB;IACxB,iBAA8B;IAC9B,oBAAqB;IACrB,kBAAuC;IACvC,0IAGE;IACF,kCAAgD;IAKhD;;;OAGG;IACH,mBAAyC;IACzC,2BAAiC;IAIjC,qBAA2B;IAK3B,mBAA8B;IAC9B,8BAAqC;IAKrC,+BAA+E;IAI/E,0BAA0B;IAC1B,4BAA4B;IAM5B,6BAAqE;IAGzE;;OAEG;IACH,0BAEC;IAED;;;;;;;;;;;;;;;OAeG;IACH,wBANW,CAAC,OAAO,GAAC,cAAc,CAAC;;oBAIvB,QAAQ,kBAAkB,CAAC,CA+CtC;IAED;;;;;OAKG;IACH,eAHW,MAAM,GACL,QAAQ,CAAC,OAAO,GAAC,IAAI,CAAC,CAAC,CAmBlC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,oBAHa,QAAQ,CAAC,OAAO,GAAC,IAAI,CAAC,CAAC,CA2DnC;IAED;;;;;;;;OAQG;IACH,4BAHW,OAAO,GACN,QAAQ,kBAAkB,GAAG,IAAI,CAAC,CA+B7C;IAED;;;;;;;;;;;;;OAaG;IACH,wBARW,OAAO;;oBAMN,QAAQ,kBAAkB,GAAG,IAAI,CAAC,CAwC7C;IAED;;;;;;;OAOG;IACH,WAFa,QAAQ,OAAO,CAAC,CAK5B;IAED;;;;;;;OAOG;IACH,cAFa,QAAQ,OAAO,CAAC,CAa5B;IAED;;OAEG;IACH,eASC;IAED;;;;;;;;;OASG;IACH,kCARW,MAAM;QAEqB,SAAS,EAApC,MAAM;QACsB,iBAAiB,EAA7C,OAAO;aAUjB;IAED;;;;;;;;;;;;OAYG;IACH,sIALY,QAAQ,OAAO,CAAC,CAqF3B;IAED;;;OAGG;IACH,oCAMC;IAED;;;;;OAKG;IACH,QAFY,QAAQ,IAAI,CAAC,CAMxB;IAED;;;;;;;;;;OAUG;IACH,gBAFY,QAAQ,MAAM,CAAC,CAM1B;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,WAFa,QAAQ,gBAAgB,CAAC,CAIrC;CACJ;AAyBM;;gBALM,QAAQ,YAAY,CAAC,CAYjC;;;;;;;;;;uBA5nBa,OAAO;;;;uBACP,OAAO;;;;eACP,MAAM;;;;aACN,OAAO;;;QA6nBP,MAAM;UACN,MAAM;YACN,MAAM;eACN,IAAI;gBACJ,IAAI;gBACJ,IAAI;uBACJ,MAAM;yBACN,MAAM;yBACN,MAAM"}