apify 2.3.1-beta.4 → 3.0.0-alpha.10

package/build/storages/dataset.d.ts

@@ -1,288 +0,0 @@
-export const DATASET_ITERATORS_DEFAULT_LIMIT: 10000;
-export const LOCAL_FILENAME_DIGITS: 9;
-export function checkAndSerialize(item: any, limitBytes: number, index?: number | undefined): string;
-export function chunkBySize(items: Array<string>, limitBytes: number): Array<string>;
-/**
- * The `Dataset` class represents a store for structured data where each object stored has the same attributes,
- * such as online store products or real estate offers. You can imagine it as a table,
- * where each object is a row and its attributes are columns.
- * Dataset is an append-only storage - you can only add new records to it but you cannot modify or remove existing records.
- * Typically it is used to store crawling results.
- *
- * Do not instantiate this class directly, use the
- * {@link Apify#openDataset} function instead.
- *
- * `Dataset` stores its data either on local disk or in the Apify cloud,
- * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.
- *
- * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in
- * the local directory in the following files:
- * ```
- * {APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json
- * ```
- * Note that `{DATASET_ID}` is the name or ID of the dataset. The default dataset has ID: `default`,
- * unless you override it by setting the `APIFY_DEFAULT_DATASET_ID` environment variable.
- * Each dataset item is stored as a separate JSON file, where `{INDEX}` is a zero-based index of the item in the dataset.
- *
- * If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` not, the data is stored in the
- * [Apify Dataset](https://docs.apify.com/storage/dataset)
- * cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
- * option to {@link Apify#openDataset} function,
- * even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
- *
- * **Example usage:**
- *
- * ```javascript
- * // Write a single row to the default dataset
- * await Apify.pushData({ col1: 123, col2: 'val2' });
- *
- * // Open a named dataset
- * const dataset = await Apify.openDataset('some-name');
- *
- * // Write a single row
- * await dataset.pushData({ foo: 'bar' });
- *
- * // Write multiple rows
- * await dataset.pushData([
- *   { foo: 'bar2', col2: 'val2' },
- *   { col3: 123 },
- * ]);
- * ```
- * @hideconstructor
- */
-export class Dataset {
-    /**
-     * @param {object} options
-     * @param {string} options.id
-     * @param {string} [options.name]
-     * @param {ApifyClient|ApifyStorageLocal} options.client
-     * @param {boolean} options.isLocal
-     */
-    constructor(options: {
-        id: string;
-        name?: string | undefined;
-        client: ApifyClient | ApifyStorageLocal;
-        isLocal: boolean;
-    });
-    id: string;
-    name: string | undefined;
-    isLocal: boolean;
-    client: import("@apify/storage-local/dist/resource_clients/dataset").DatasetClient | import("apify-client").DatasetClient<Record<string | number, unknown>>;
-    log: import("@apify/log/log").Log;
-    /**
-     * Stores an object or an array of objects to the dataset.
-     * The function returns a promise that resolves when the operation finishes.
-     * It has no result, but throws on invalid args or other errors.
-     *
-     * **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
-     * otherwise the actor process might finish before the data is stored!
-     *
-     * The size of the data is limited by the receiving API and therefore `pushData()` will only
-     * allow objects whose JSON representation is smaller than 9MB. When an array is passed,
-     * none of the included objects
-     * may be larger than 9MB, but the array itself may be of any size.
-     *
-     * The function internally
-     * chunks the array into separate items and pushes them sequentially.
-     * The chunking process is stable (keeps order of data), but it does not provide a transaction
-     * safety mechanism. Therefore, in the event of an uploading error (after several automatic retries),
-     * the function's Promise will reject and the dataset will be left in a state where some of
-     * the items have already been saved to the dataset while other items from the source array were not.
-     * To overcome this limitation, the developer may, for example, read the last item saved in the dataset
-     * and re-attempt the save of the data from this item onwards to prevent duplicates.
-     * @param {object|Array<object>} data Object or array of objects containing data to be stored in the default dataset.
-     * The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
-     * @return {Promise<void>}
-     */
-    pushData(data: object | Array<object>): Promise<void>;
-    /**
-     * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
-     *
-     * If you need to get data in an unparsed format, use the {@link Apify#newClient} function to get a new
-     * `apify-client` instance and call
-     * [`datasetClient.downloadItems()`](https://github.com/apify/apify-client-js#DatasetClient+downloadItems)
-     *
-     * @param {Object} [options] All `getData()` parameters are passed
-     *   via an options object with the following keys:
-     * @param {number} [options.offset=0]
-     *   Number of array elements that should be skipped at the start.
-     * @param {number} [options.limit=250000]
-     *   Maximum number of array elements to return.
-     * @param {boolean} [options.desc=false]
-     *   If `true` then the objects are sorted by `createdAt` in descending order.
-     *   Otherwise they are sorted in ascending order.
-     * @param {string[]} [options.fields]
-     *   An array of field names that will be included in the result. If omitted, all fields are included in the results.
-     * @param {string} [options.unwind]
-     *   Specifies a name of the field in the result objects that will be used to unwind the resulting objects.
-     *   By default, the results are returned as they are.
-     * @param {boolean} [options.clean=false]
-     *   If `true` then the function returns only non-empty items and skips hidden fields (i.e. fields starting with `#` character).
-     *   Note that the `clean` parameter is a shortcut for `skipHidden: true` and `skipEmpty: true` options.
-     * @param {boolean} [options.skipHidden=false]
-     *   If `true` then the function doesn't return hidden fields (fields starting with "#" character).
-     * @param {boolean} [options.skipEmpty=false]
-     *   If `true` then the function doesn't return empty items.
-     *   Note that in this case the returned number of items might be lower than limit parameter and pagination must be done using the `limit` value.
-     * @return {Promise<DatasetContent>}
-     */
-    getData(options?: {
-        offset?: number | undefined;
-        limit?: number | undefined;
-        desc?: boolean | undefined;
-        fields?: string[] | undefined;
-        unwind?: string | undefined;
-        clean?: boolean | undefined;
-        skipHidden?: boolean | undefined;
-        skipEmpty?: boolean | undefined;
-    } | undefined): Promise<DatasetContent>;
-    /**
-     * Returns an object containing general information about the dataset.
-     *
-     * The function returns the same object as the Apify API Client's
-     * [getDataset](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-datasets-getDataset)
-     * function, which in turn calls the
-     * [Get dataset](https://apify.com/docs/api/v2#/reference/datasets/dataset/get-dataset)
-     * API endpoint.
-     *
-     * **Example:**
-     * ```
-     * {
-     *   id: "WkzbQMuFYuamGv3YF",
-     *   name: "my-dataset",
-     *   userId: "wRsJZtadYvn4mBZmm",
-     *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
-     *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
-     *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
-     *   itemCount: 14,
-     *   cleanItemCount: 10
-     * }
-     * ```
-     *
-     * @returns {Promise<object>}
-     */
-    getInfo(): Promise<object>;
-    /**
-     * Iterates over dataset items, yielding each in turn to an `iteratee` function.
-     * Each invocation of `iteratee` is called with two arguments: `(item, index)`.
-     *
-     * If the `iteratee` function returns a Promise then it is awaited before the next call.
-     * If it throws an error, the iteration is aborted and the `forEach` function throws the error.
-     *
-     * **Example usage**
-     * ```javascript
-     * const dataset = await Apify.openDataset('my-results');
-     * await dataset.forEach(async (item, index) => {
-     *   console.log(`Item at ${index}: ${JSON.stringify(item)}`);
-     * });
-     * ```
-     *
-     * @param {DatasetConsumer} iteratee A function that is called for every item in the dataset.
-     * @param {Object} [options] All `forEach()` parameters are passed
-     *   via an options object with the following keys:
-     * @param {boolean} [options.desc=false] If `true` then the objects are sorted by `createdAt` in descending order.
-     * @param {string[]} [options.fields] If provided then returned objects will only contain specified keys.
-     * @param {string} [options.unwind] If provided then objects will be unwound based on provided field.
-     * @param {number} [index=0] Specifies the initial index number passed to the `iteratee` function.
-     * @return {Promise<void>}
-     */
-    forEach(iteratee: DatasetConsumer, options?: {
-        desc?: boolean | undefined;
-        fields?: string[] | undefined;
-        unwind?: string | undefined;
-    } | undefined, index?: number | undefined): Promise<void>;
-    /**
-     * Produces a new array of values by mapping each value in list through a transformation function `iteratee()`.
-     * Each invocation of `iteratee()` is called with two arguments: `(element, index)`.
-     *
-     * If `iteratee` returns a `Promise` then it's awaited before a next call.
-     *
-     * @param {DatasetMapper} iteratee
-     * @param {Object} [options] All `map()` parameters are passed
-     *   via an options object with the following keys:
-     * @param {boolean} [options.desc=false] If `true` then the objects are sorted by createdAt in descending order.
-     * @param {string[]} [options.fields] If provided then returned objects will only contain specified keys
-     * @param {string} [options.unwind] If provided then objects will be unwound based on provided field.
-     * @return {Promise<Array<object>>}
-     */
-    map(iteratee: DatasetMapper, options?: {
-        desc?: boolean | undefined;
-        fields?: string[] | undefined;
-        unwind?: string | undefined;
-    } | undefined): Promise<Array<object>>;
-    /**
-     * Reduces a list of values down to a single value.
-     *
-     * Memo is the initial state of the reduction, and each successive step of it should be returned by `iteratee()`.
-     * The `iteratee()` is passed three arguments: the `memo`, then the `value` and `index` of the iteration.
-     *
-     * If no `memo` is passed to the initial invocation of reduce, the `iteratee()` is not invoked on the first element of the list.
-     * The first element is instead passed as the memo in the invocation of the `iteratee()` on the next element in the list.
-     *
-     * If `iteratee()` returns a `Promise` then it's awaited before a next call.
-     *
-     * @param {DatasetReducer} iteratee
-     * @param {object} memo Initial state of the reduction.
-     * @param {Object} [options] All `reduce()` parameters are passed
-     *   via an options object with the following keys:
-     * @param {boolean} [options.desc=false] If `true` then the objects are sorted by createdAt in descending order.
-     * @param {string[]} [options.fields] If provided then returned objects will only contain specified keys
-     * @param {string} [options.unwind] If provided then objects will be unwound based on provided field.
-     * @return {Promise<object>}
-     */
-    reduce(iteratee: DatasetReducer, memo: object, options?: {
-        desc?: boolean | undefined;
-        fields?: string[] | undefined;
-        unwind?: string | undefined;
-    } | undefined): Promise<object>;
-    /**
-     * Removes the dataset either from the Apify cloud storage or from the local directory,
-     * depending on the mode of operation.
-     *
-     * @return {Promise<void>}
-     */
-    drop(): Promise<void>;
-}
-export function openDataset(datasetIdOrName?: string | undefined, options?: {
-    forceCloud?: boolean | undefined;
-    config?: Configuration | undefined;
-} | undefined): Promise<Dataset>;
-export function pushData(item: object): Promise<void>;
-export type DatasetContent = {
-    /**
-     * Dataset entries based on chosen format parameter.
-     */
-    items: Array<object>;
-    /**
-     * Total count of entries in the dataset.
-     */
-    total: number;
-    /**
-     * Position of the first returned entry in the dataset.
-     */
-    offset: number;
-    /**
-     * Count of dataset entries returned in this set.
-     */
-    count: number;
-    /**
-     * Maximum number of dataset entries requested.
-     */
-    limit: number;
-};
-/**
- * User-function used in the `Dataset.forEach()` API.
- */
-export type DatasetConsumer = (item: object, index: number) => object;
-/**
- * User-function used in the `Dataset.map()` API.
- */
-export type DatasetMapper = (item: object, index: number) => object;
-/**
- * User-function used in the `Dataset.reduce()` API.
- */
-export type DatasetReducer = (memo: object, item: object, index: number) => object;
-import { ApifyClient } from "apify-client/dist/apify_client";
-import { ApifyStorageLocal } from "@apify/storage-local";
-import { Configuration } from "../configuration";
-//# sourceMappingURL=dataset.d.ts.map

package/build/storages/dataset.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.js"],"names":[],"mappings":"AAcA,oDAAqD;AACrD,sCAAuC;AAchC,yDALI,MAAM,+BAEJ,MAAM,CAuBlB;AAeM,mCALI,MAAM,MAAM,CAAC,cACb,MAAM,GACJ,MAAM,MAAM,CAAC,CA6BzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH;IACI;;;;;;OAMG;IACH;QAL2B,EAAE,EAAlB,MAAM;QACW,IAAI;QACkB,MAAM,EAA7C,WAAW,GAAC,iBAAiB;QACZ,OAAO,EAAxB,OAAO;OAQjB;IALG,WAAoB;IACpB,yBAAwB;IACxB,iBAA8B;IAC9B,4JAA6C;IAC7C,kCAA2C;IAG/C;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,eAJW,MAAM,GAAC,MAAM,MAAM,CAAC,GAEnB,QAAQ,IAAI,CAAC,CAsBxB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH;;;;;;;;;oBAFY,QAAQ,cAAc,CAAC,CAWlC;IAED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,WAFa,QAAQ,MAAM,CAAC,CAI3B;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,kBATW,eAAe;;;;gDAOd,QAAQ,IAAI,CAAC,CAkBxB;IAED;;;;;;;;;;;;;OAaG;IACH,cARW,aAAa;;;;oBAMZ,QAAQ,MAAM,MAAM,CAAC,CAAC,CAejC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,iBATW,cAAc,QACd,MAAM;;;;oBAML,QAAQ,MAAM,CAAC,CAqB1B;IAED;;;;;OAKG;IACH,QAFY,QAAQ,IAAI,CAAC,CAMxB;CACJ;AAwBM;;;gBALM,QAAQ,OAAO,CAAC,CAc5B;AA8BM,+BARI,MAAM,GAEJ,QAAQ,IAAI,CAAC,CAUzB;;;;;WAIa,MAAM,MAAM,CAAC;;;;WACb,MAAM;;;;YACN,MAAM;;;;WACN,MAAM;;;;WACN,MAAM;;;;;qCAOT,MAAM,SACN,MAAM,KACJ,MAAM;;;;mCAOR,MAAM,SACN,MAAM,KACJ,MAAM;;;;oCAMR,MAAM,QACN,MAAM,SACN,MAAM,KACJ,MAAM"}