@merkl/api 0.10.413 → 0.10.415
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,7 @@ if (!process.env.ENV || !process.env.FILENAME)
|
|
3
3
|
throw new Error("[ENV]: missing variable");
|
4
4
|
import { log } from "../../utils/logger";
|
5
5
|
import { apiDbClient } from "../../utils/prisma";
|
6
|
+
import { withRetry } from "@sdk";
|
6
7
|
import { S3Client } from "bun";
|
7
8
|
import moment from "moment";
|
8
9
|
// ─── Constants ───────────────────────────────────────────────
|
@@ -17,7 +18,6 @@ const gcsClient = new S3Client({
|
|
17
18
|
bucket: `merkl-rewards-lake-${process.env.ENV}`,
|
18
19
|
});
|
19
20
|
const file = gcsClient.file(`pendings/${process.env.FILENAME}`);
|
20
|
-
const failedBatches = [];
|
21
21
|
// ─── Extract ─────────────────────────────────────────────────────────────────
|
22
22
|
const extract = async () => {
|
23
23
|
if (!file.exists())
|
@@ -25,8 +25,8 @@ const extract = async () => {
|
|
25
25
|
const textDecoder = new TextDecoder();
|
26
26
|
let buffer = "";
|
27
27
|
const stream = file.stream();
|
28
|
-
let
|
29
|
-
const
|
28
|
+
let data = [];
|
29
|
+
const loadPromises = [];
|
30
30
|
for await (const chunk of stream) {
|
31
31
|
buffer += textDecoder.decode(chunk);
|
32
32
|
const lines = buffer.split("\n");
|
@@ -36,26 +36,21 @@ const extract = async () => {
|
|
36
36
|
}
|
37
37
|
if (data.length < MAX_BATCH_SIZE)
|
38
38
|
continue;
|
39
|
-
|
40
|
-
|
41
|
-
}
|
42
|
-
catch (err) {
|
43
|
-
console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
|
44
|
-
failedBatches.push(data);
|
45
|
-
data.length = 0;
|
46
|
-
}
|
39
|
+
loadPromises.push(withRetry(load, [data], 5, 60_000));
|
40
|
+
data = [];
|
47
41
|
}
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
42
|
+
// Final batch
|
43
|
+
loadPromises.push(withRetry(load, [data], 5, 10_000));
|
44
|
+
let count = 0;
|
45
|
+
let failed = 0;
|
46
|
+
const promiseResults = await Promise.allSettled(loadPromises);
|
47
|
+
for (const x of promiseResults) {
|
48
|
+
if (x.status === "rejected")
|
49
|
+
failed += 1;
|
50
|
+
else
|
51
|
+
count += x.value;
|
57
52
|
}
|
58
|
-
return count;
|
53
|
+
return { count, failed };
|
59
54
|
};
|
60
55
|
// ─── Transform & Load ────────────────────────────────────────────────────────
|
61
56
|
const load = async (pendings) => {
|
@@ -193,17 +188,18 @@ const updatePendings = async (data) => {
|
|
193
188
|
};
|
194
189
|
// ─────────────────────────────────────────────────────────────────────────────
|
195
190
|
export const main = async () => {
|
191
|
+
log.info(`✅ Running for ${process.env.FILENAME}`);
|
196
192
|
const start = moment().unix();
|
197
193
|
// ─── Start Rewards ETL ───────────────────────────────────────────────
|
198
|
-
const count = await extract();
|
194
|
+
const { count, failed } = await extract();
|
199
195
|
log.info(`✅ Successfully created ${count} new records for ${process.env.FILENAME} in ${moment().unix() - start} sec`);
|
200
|
-
if (
|
201
|
-
log.error("pendings", `${
|
196
|
+
if (failed !== 0) {
|
197
|
+
log.error("pendings", `${failed} batches failed.`);
|
202
198
|
process.exit(1);
|
203
199
|
}
|
204
|
-
if (
|
205
|
-
|
206
|
-
|
200
|
+
if (failed === 0) {
|
201
|
+
await file.delete();
|
202
|
+
log.info("Object deleted");
|
207
203
|
}
|
208
204
|
};
|
209
205
|
main();
|