@algolia/ingestion 1.55.0 → 1.55.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/browser.d.ts +1 -1
- package/dist/builds/browser.js +37 -28
- package/dist/builds/browser.js.map +1 -1
- package/dist/builds/browser.min.js +4 -4
- package/dist/builds/browser.min.js.map +1 -1
- package/dist/builds/browser.umd.js +4 -4
- package/dist/builds/fetch.js +37 -28
- package/dist/builds/fetch.js.map +1 -1
- package/dist/builds/node.cjs +37 -28
- package/dist/builds/node.cjs.map +1 -1
- package/dist/builds/node.js +37 -28
- package/dist/builds/node.js.map +1 -1
- package/dist/builds/worker.js +37 -28
- package/dist/builds/worker.js.map +1 -1
- package/dist/fetch.d.ts +1 -1
- package/dist/node.d.cts +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/src/ingestionClient.cjs +37 -28
- package/dist/src/ingestionClient.cjs.map +1 -1
- package/dist/src/ingestionClient.js +42 -33
- package/dist/src/ingestionClient.js.map +1 -1
- package/dist/worker.d.ts +1 -1
- package/package.json +9 -9
package/dist/builds/node.js
CHANGED
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
getAlgoliaAgent,
|
|
12
12
|
validateRequired
|
|
13
13
|
} from "@algolia/client-common";
|
|
14
|
-
var apiClientVersion = "1.55.
|
|
14
|
+
var apiClientVersion = "1.55.2";
|
|
15
15
|
var REGIONS = ["eu", "us"];
|
|
16
16
|
function getDefaultHosts(region) {
|
|
17
17
|
const url = "data.{region}.algolia.com".replace("{region}", region);
|
|
@@ -120,10 +120,41 @@ function createIngestionClient({
|
|
|
120
120
|
referenceIndexName,
|
|
121
121
|
maxRetries = 100
|
|
122
122
|
}, requestOptions) {
|
|
123
|
+
if (batchSize < 1) {
|
|
124
|
+
throw new Error("`batchSize` must be at least 1.");
|
|
125
|
+
}
|
|
123
126
|
let records = [];
|
|
124
127
|
let offset = 0;
|
|
125
128
|
const responses = [];
|
|
126
129
|
const waitBatchSize = Math.floor(batchSize / 10) || batchSize;
|
|
130
|
+
const waitForBatch = async (start, end) => {
|
|
131
|
+
for (const resp of responses.slice(start, end)) {
|
|
132
|
+
if (!resp.eventID) {
|
|
133
|
+
throw new Error("received unexpected response from the push endpoint, eventID must not be undefined");
|
|
134
|
+
}
|
|
135
|
+
let retryCount = 0;
|
|
136
|
+
await createIterablePromise({
|
|
137
|
+
func: async () => {
|
|
138
|
+
if (resp.eventID === void 0 || !resp.eventID) {
|
|
139
|
+
throw new Error("received unexpected response from the push endpoint, eventID must not be undefined");
|
|
140
|
+
}
|
|
141
|
+
return this.getEvent({ runID: resp.runID, eventID: resp.eventID }).catch((error) => {
|
|
142
|
+
if (error.status === 404) {
|
|
143
|
+
return void 0;
|
|
144
|
+
}
|
|
145
|
+
throw error;
|
|
146
|
+
});
|
|
147
|
+
},
|
|
148
|
+
validate: (response) => response !== void 0,
|
|
149
|
+
aggregator: () => retryCount += 1,
|
|
150
|
+
error: {
|
|
151
|
+
validate: () => retryCount >= maxRetries,
|
|
152
|
+
message: () => `Stopped waiting for the task after ${maxRetries} retries. This does not mean the operation failed; it may still complete. If you need to keep polling, retry with a higher maxRetries.`
|
|
153
|
+
},
|
|
154
|
+
timeout: () => Math.min(retryCount * 1500, 5e3)
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
};
|
|
127
158
|
const objectEntries = objects.entries();
|
|
128
159
|
for (const [i, obj] of objectEntries) {
|
|
129
160
|
records.push(obj);
|
|
@@ -133,36 +164,14 @@ function createIngestionClient({
|
|
|
133
164
|
);
|
|
134
165
|
records = [];
|
|
135
166
|
}
|
|
136
|
-
if (waitForTasks && responses.length
|
|
137
|
-
|
|
138
|
-
if (!resp.eventID) {
|
|
139
|
-
throw new Error("received unexpected response from the push endpoint, eventID must not be undefined");
|
|
140
|
-
}
|
|
141
|
-
let retryCount = 0;
|
|
142
|
-
await createIterablePromise({
|
|
143
|
-
func: async () => {
|
|
144
|
-
if (resp.eventID === void 0 || !resp.eventID) {
|
|
145
|
-
throw new Error("received unexpected response from the push endpoint, eventID must not be undefined");
|
|
146
|
-
}
|
|
147
|
-
return this.getEvent({ runID: resp.runID, eventID: resp.eventID }).catch((error) => {
|
|
148
|
-
if (error.status === 404) {
|
|
149
|
-
return void 0;
|
|
150
|
-
}
|
|
151
|
-
throw error;
|
|
152
|
-
});
|
|
153
|
-
},
|
|
154
|
-
validate: (response) => response !== void 0,
|
|
155
|
-
aggregator: () => retryCount += 1,
|
|
156
|
-
error: {
|
|
157
|
-
validate: () => retryCount >= maxRetries,
|
|
158
|
-
message: () => `Stopped waiting for the task after ${maxRetries} retries. This does not mean the operation failed; it may still complete. If you need to keep polling, retry with a higher maxRetries.`
|
|
159
|
-
},
|
|
160
|
-
timeout: () => Math.min(retryCount * 1500, 5e3)
|
|
161
|
-
});
|
|
162
|
-
}
|
|
167
|
+
if (waitForTasks && responses.length - offset >= waitBatchSize) {
|
|
168
|
+
await waitForBatch(offset, offset + waitBatchSize);
|
|
163
169
|
offset += waitBatchSize;
|
|
164
170
|
}
|
|
165
171
|
}
|
|
172
|
+
if (waitForTasks) {
|
|
173
|
+
await waitForBatch(offset, responses.length);
|
|
174
|
+
}
|
|
166
175
|
return responses;
|
|
167
176
|
},
|
|
168
177
|
/**
|