@algolia/ingestion 1.27.0 → 1.29.0

package/dist/builds/worker.js

@@ -3,8 +3,8 @@ import { createMemoryCache, createNullCache, createNullLogger } from "@algolia/c
 import { createFetchRequester } from "@algolia/requester-fetch";
 
 // src/ingestionClient.ts
-import { createAuth, createTransporter, getAlgoliaAgent } from "@algolia/client-common";
-var apiClientVersion = "1.27.0";
+import { createAuth, createIterablePromise, createTransporter, getAlgoliaAgent } from "@algolia/client-common";
+var apiClientVersion = "1.29.0";
 var REGIONS = ["eu", "us"];
 function getDefaultHosts(region) {
   const url = "data.{region}.algolia.com".replace("{region}", region);
@@ -90,6 +90,69 @@ function createIngestionClient({
         transporter.baseQueryParameters["x-algolia-api-key"] = apiKey;
       }
     },
+    /**
+     * Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `push` requests by leveraging the Transformation pipeline setup in the Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/).
+     *
+     * @summary Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `batch` requests.
+     * @param chunkedPush - The `chunkedPush` object.
+     * @param chunkedPush.indexName - The `indexName` to replace `objects` in.
+     * @param chunkedPush.objects - The array of `objects` to store in the given Algolia `indexName`.
+     * @param chunkedPush.action - The `batch` `action` to perform on the given array of `objects`, defaults to `addObject`.
+     * @param chunkedPush.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
+     * @param chunkedPush.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
+     * @param chunkedPush.referenceIndexName - This is required when targeting an index that does not have a push connector setup (e.g. a tmp index), but you wish to attach another index's transformation to it (e.g. the source index name).
+     * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `getEvent` method and merged with the transporter requestOptions.
+     */
+    async chunkedPush({
+      indexName,
+      objects,
+      action = "addObject",
+      waitForTasks,
+      batchSize = 1e3,
+      referenceIndexName
+    }, requestOptions) {
+      let records = [];
+      const responses = [];
+      const objectEntries = objects.entries();
+      for (const [i, obj] of objectEntries) {
+        records.push(obj);
+        if (records.length === batchSize || i === objects.length - 1) {
+          responses.push(
+            await this.push({ indexName, pushTaskPayload: { action, records }, referenceIndexName }, requestOptions)
+          );
+          records = [];
+        }
+      }
+      let retryCount = 0;
+      if (waitForTasks) {
+        for (const resp of responses) {
+          if (!resp.eventID) {
+            throw new Error("received unexpected response from the push endpoint, eventID must not be undefined");
+          }
+          await createIterablePromise({
+            func: async () => {
+              if (resp.eventID === void 0 || !resp.eventID) {
+                throw new Error("received unexpected response from the push endpoint, eventID must not be undefined");
+              }
+              return this.getEvent({ runID: resp.runID, eventID: resp.eventID }).catch((error) => {
+                if (error.status === 404) {
+                  return void 0;
+                }
+                throw error;
+              });
+            },
+            validate: (response) => response !== void 0,
+            aggregator: () => retryCount += 1,
+            error: {
+              validate: () => retryCount >= 50,
+              message: () => `The maximum number of retries exceeded. (${retryCount}/${50})`
+            },
+            timeout: () => Math.min(retryCount * 500, 5e3)
+          });
+        }
+      }
+      return responses;
+    },
     /**
      * Creates a new authentication resource.
      *
@@ -269,12 +332,6 @@ function createIngestionClient({
       if (!transformationCreate.name) {
         throw new Error("Parameter `transformationCreate.name` is required when calling `createTransformation`.");
       }
-      if (!transformationCreate.type) {
-        throw new Error("Parameter `transformationCreate.type` is required when calling `createTransformation`.");
-      }
-      if (!transformationCreate.input) {
-        throw new Error("Parameter `transformationCreate.input` is required when calling `createTransformation`.");
-      }
       const requestPath = "/1/transformations";
       const headers = {};
       const queryParameters = {};
@@ -290,7 +347,7 @@ function createIngestionClient({
     /**
      * This method lets you send requests to the Algolia REST API.
      * @param customDelete - The customDelete object.
-     * @param customDelete.path - Path of the endpoint, anything after \"/1\" must be specified.
+     * @param customDelete.path - Path of the endpoint, for example `1/newFeature`.
      * @param customDelete.parameters - Query parameters to apply to the current query.
      * @param requestOptions - The requestOptions to send along with the query, they will be merged with the transporter requestOptions.
      */
@@ -312,7 +369,7 @@ function createIngestionClient({
     /**
      * This method lets you send requests to the Algolia REST API.
      * @param customGet - The customGet object.
-     * @param customGet.path - Path of the endpoint, anything after \"/1\" must be specified.
+     * @param customGet.path - Path of the endpoint, for example `1/newFeature`.
      * @param customGet.parameters - Query parameters to apply to the current query.
      * @param requestOptions - The requestOptions to send along with the query, they will be merged with the transporter requestOptions.
      */
@@ -334,7 +391,7 @@ function createIngestionClient({
     /**
      * This method lets you send requests to the Algolia REST API.
      * @param customPost - The customPost object.
-     * @param customPost.path - Path of the endpoint, anything after \"/1\" must be specified.
+     * @param customPost.path - Path of the endpoint, for example `1/newFeature`.
      * @param customPost.parameters - Query parameters to apply to the current query.
      * @param customPost.body - Parameters to send with the custom request.
      * @param requestOptions - The requestOptions to send along with the query, they will be merged with the transporter requestOptions.
@@ -358,7 +415,7 @@ function createIngestionClient({
     /**
      * This method lets you send requests to the Algolia REST API.
      * @param customPut - The customPut object.
-     * @param customPut.path - Path of the endpoint, anything after \"/1\" must be specified.
+     * @param customPut.path - Path of the endpoint, for example `1/newFeature`.
      * @param customPut.parameters - Query parameters to apply to the current query.
      * @param customPut.body - Parameters to send with the custom request.
      * @param requestOptions - The requestOptions to send along with the query, they will be merged with the transporter requestOptions.
@@ -1307,9 +1364,10 @@ function createIngestionClient({
      * @param push.indexName - Name of the index on which to perform the operation.
      * @param push.pushTaskPayload - The pushTaskPayload object.
      * @param push.watch - When provided, the push operation will be synchronous and the API will wait for the ingestion to be finished before responding.
+     * @param push.referenceIndexName - This is required when targeting an index that does not have a push connector setup (e.g. a tmp index), but you wish to attach another index\'s transformation to it (e.g. the source index name).
      * @param requestOptions - The requestOptions to send along with the query, they will be merged with the transporter requestOptions.
      */
-    push({ indexName, pushTaskPayload, watch }, requestOptions) {
+    push({ indexName, pushTaskPayload, watch, referenceIndexName }, requestOptions) {
       if (!indexName) {
         throw new Error("Parameter `indexName` is required when calling `push`.");
       }
@@ -1328,6 +1386,9 @@ function createIngestionClient({
       if (watch !== void 0) {
         queryParameters["watch"] = watch.toString();
       }
+      if (referenceIndexName !== void 0) {
+        queryParameters["referenceIndexName"] = referenceIndexName.toString();
+      }
       const request = {
         method: "POST",
         path: requestPath,
@@ -1704,9 +1765,6 @@ function createIngestionClient({
       if (!transformationTry) {
         throw new Error("Parameter `transformationTry` is required when calling `tryTransformation`.");
       }
-      if (!transformationTry.code) {
-        throw new Error("Parameter `transformationTry.code` is required when calling `tryTransformation`.");
-      }
       if (!transformationTry.sampleRecord) {
         throw new Error("Parameter `transformationTry.sampleRecord` is required when calling `tryTransformation`.");
       }
@@ -1741,9 +1799,6 @@ function createIngestionClient({
       if (!transformationTry) {
         throw new Error("Parameter `transformationTry` is required when calling `tryTransformationBeforeUpdate`.");
       }
-      if (!transformationTry.code) {
-        throw new Error("Parameter `transformationTry.code` is required when calling `tryTransformationBeforeUpdate`.");
-      }
       if (!transformationTry.sampleRecord) {
         throw new Error(
           "Parameter `transformationTry.sampleRecord` is required when calling `tryTransformationBeforeUpdate`."
@@ -1934,12 +1989,6 @@ function createIngestionClient({
       if (!transformationCreate.name) {
         throw new Error("Parameter `transformationCreate.name` is required when calling `updateTransformation`.");
       }
-      if (!transformationCreate.type) {
-        throw new Error("Parameter `transformationCreate.type` is required when calling `updateTransformation`.");
-      }
-      if (!transformationCreate.input) {
-        throw new Error("Parameter `transformationCreate.input` is required when calling `updateTransformation`.");
-      }
       const requestPath = "/1/transformations/{transformationID}".replace(
         "{transformationID}",
         encodeURIComponent(transformationID)