@algolia/ingestion 1.28.0 → 1.30.0

package/dist/builds/worker.js

@@ -3,8 +3,8 @@ import { createMemoryCache, createNullCache, createNullLogger } from "@algolia/c
 import { createFetchRequester } from "@algolia/requester-fetch";
 
 // src/ingestionClient.ts
-import { createAuth, createTransporter, getAlgoliaAgent } from "@algolia/client-common";
-var apiClientVersion = "1.28.0";
+import { createAuth, createIterablePromise, createTransporter, getAlgoliaAgent } from "@algolia/client-common";
+var apiClientVersion = "1.30.0";
 var REGIONS = ["eu", "us"];
 function getDefaultHosts(region) {
   const url = "data.{region}.algolia.com".replace("{region}", region);
@@ -90,6 +90,69 @@ function createIngestionClient({
         transporter.baseQueryParameters["x-algolia-api-key"] = apiKey;
       }
     },
+    /**
+     * Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `push` requests by leveraging the Transformation pipeline setup in the Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/).
+     *
+     * @summary Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `batch` requests.
+     * @param chunkedPush - The `chunkedPush` object.
+     * @param chunkedPush.indexName - The `indexName` to replace `objects` in.
+     * @param chunkedPush.objects - The array of `objects` to store in the given Algolia `indexName`.
+     * @param chunkedPush.action - The `batch` `action` to perform on the given array of `objects`, defaults to `addObject`.
+     * @param chunkedPush.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
+     * @param chunkedPush.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
+     * @param chunkedPush.referenceIndexName - This is required when targeting an index that does not have a push connector setup (e.g. a tmp index), but you wish to attach another index's transformation to it (e.g. the source index name).
+     * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `getEvent` method and merged with the transporter requestOptions.
+     */
+    async chunkedPush({
+      indexName,
+      objects,
+      action = "addObject",
+      waitForTasks,
+      batchSize = 1e3,
+      referenceIndexName
+    }, requestOptions) {
+      let records = [];
+      const responses = [];
+      const objectEntries = objects.entries();
+      for (const [i, obj] of objectEntries) {
+        records.push(obj);
+        if (records.length === batchSize || i === objects.length - 1) {
+          responses.push(
+            await this.push({ indexName, pushTaskPayload: { action, records }, referenceIndexName }, requestOptions)
+          );
+          records = [];
+        }
+      }
+      let retryCount = 0;
+      if (waitForTasks) {
+        for (const resp of responses) {
+          if (!resp.eventID) {
+            throw new Error("received unexpected response from the push endpoint, eventID must not be undefined");
+          }
+          await createIterablePromise({
+            func: async () => {
+              if (resp.eventID === void 0 || !resp.eventID) {
+                throw new Error("received unexpected response from the push endpoint, eventID must not be undefined");
+              }
+              return this.getEvent({ runID: resp.runID, eventID: resp.eventID }).catch((error) => {
+                if (error.status === 404) {
+                  return void 0;
+                }
+                throw error;
+              });
+            },
+            validate: (response) => response !== void 0,
+            aggregator: () => retryCount += 1,
+            error: {
+              validate: () => retryCount >= 50,
+              message: () => `The maximum number of retries exceeded. (${retryCount}/${50})`
+            },
+            timeout: () => Math.min(retryCount * 500, 5e3)
+          });
+        }
+      }
+      return responses;
+    },
     /**
      * Creates a new authentication resource.
      *
@@ -284,7 +347,7 @@ function createIngestionClient({
     /**
      * This method lets you send requests to the Algolia REST API.
      * @param customDelete - The customDelete object.
-     * @param customDelete.path - Path of the endpoint, anything after \"/1\" must be specified.
+     * @param customDelete.path - Path of the endpoint, for example `1/newFeature`.
      * @param customDelete.parameters - Query parameters to apply to the current query.
      * @param requestOptions - The requestOptions to send along with the query, they will be merged with the transporter requestOptions.
      */
@@ -306,7 +369,7 @@ function createIngestionClient({
     /**
      * This method lets you send requests to the Algolia REST API.
      * @param customGet - The customGet object.
-     * @param customGet.path - Path of the endpoint, anything after \"/1\" must be specified.
+     * @param customGet.path - Path of the endpoint, for example `1/newFeature`.
      * @param customGet.parameters - Query parameters to apply to the current query.
      * @param requestOptions - The requestOptions to send along with the query, they will be merged with the transporter requestOptions.
      */
@@ -328,7 +391,7 @@ function createIngestionClient({
     /**
      * This method lets you send requests to the Algolia REST API.
      * @param customPost - The customPost object.
-     * @param customPost.path - Path of the endpoint, anything after \"/1\" must be specified.
+     * @param customPost.path - Path of the endpoint, for example `1/newFeature`.
      * @param customPost.parameters - Query parameters to apply to the current query.
      * @param customPost.body - Parameters to send with the custom request.
      * @param requestOptions - The requestOptions to send along with the query, they will be merged with the transporter requestOptions.
@@ -352,7 +415,7 @@ function createIngestionClient({
     /**
      * This method lets you send requests to the Algolia REST API.
      * @param customPut - The customPut object.
-     * @param customPut.path - Path of the endpoint, anything after \"/1\" must be specified.
+     * @param customPut.path - Path of the endpoint, for example `1/newFeature`.
      * @param customPut.parameters - Query parameters to apply to the current query.
      * @param customPut.body - Parameters to send with the custom request.
      * @param requestOptions - The requestOptions to send along with the query, they will be merged with the transporter requestOptions.