@merkl/api 0.10.412 → 0.10.414

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,8 +3,11 @@ if (!process.env.ENV || !process.env.FILENAME)
3
3
  throw new Error("[ENV]: missing variable");
4
4
  import { log } from "../../utils/logger";
5
5
  import { apiDbClient } from "../../utils/prisma";
6
+ import { withRetry } from "@sdk";
6
7
  import { S3Client } from "bun";
7
8
  import moment from "moment";
9
+ // ─── Constants ───────────────────────────────────────────────
10
+ const MAX_BATCH_SIZE = 10_000;
8
11
  // ─── Global Variables ────────────────────────────────────────
9
12
  const [chainIdString, root, campaignId] = process.env.FILENAME.replace(".gz", "").split("_");
10
13
  const chainId = Number.parseInt(chainIdString);
@@ -15,7 +18,6 @@ const gcsClient = new S3Client({
15
18
  bucket: `merkl-rewards-lake-${process.env.ENV}`,
16
19
  });
17
20
  const file = gcsClient.file(`pendings/${process.env.FILENAME}`);
18
- const failedBatches = [];
19
21
  // ─── Extract ─────────────────────────────────────────────────────────────────
20
22
  const extract = async () => {
21
23
  if (!file.exists())
@@ -23,8 +25,8 @@ const extract = async () => {
23
25
  const textDecoder = new TextDecoder();
24
26
  let buffer = "";
25
27
  const stream = file.stream();
26
- let count = 0;
27
- const data = [];
28
+ let data = [];
29
+ const loadPromises = [];
28
30
  for await (const chunk of stream) {
29
31
  buffer += textDecoder.decode(chunk);
30
32
  const lines = buffer.split("\n");
@@ -32,16 +34,23 @@ const extract = async () => {
32
34
  for (const line of lines) {
33
35
  data.push(JSON.parse(line));
34
36
  }
35
- try {
36
- count += await load(data);
37
- }
38
- catch (err) {
39
- console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
40
- failedBatches.push(data);
41
- data.length = 0;
42
- }
37
+ if (data.length < MAX_BATCH_SIZE)
38
+ continue;
39
+ loadPromises.push(withRetry(load, [data], 5, 60_000));
40
+ data = [];
41
+ }
42
+ // Final batch
43
+ loadPromises.push(withRetry(load, [data], 5, 60_000));
44
+ let count = 0;
45
+ let failed = 0;
46
+ const promiseResults = await Promise.allSettled(loadPromises);
47
+ for (const x of promiseResults) {
48
+ if (x.status === "rejected")
49
+ failed += 1;
50
+ else
51
+ count += x.value;
43
52
  }
44
- return count;
53
+ return { count, failed };
45
54
  };
46
55
  // ─── Transform & Load ────────────────────────────────────────────────────────
47
56
  const load = async (pendings) => {
@@ -181,15 +190,15 @@ const updatePendings = async (data) => {
181
190
  export const main = async () => {
182
191
  const start = moment().unix();
183
192
  // ─── Start Rewards ETL ───────────────────────────────────────────────
184
- const count = await extract();
193
+ const { count, failed } = await extract();
185
194
  log.info(`✅ Successfully created ${count} new records for ${process.env.FILENAME} in ${moment().unix() - start} sec`);
186
- if (failedBatches.length !== 0) {
187
- log.error("pendings", `${failedBatches.length} batches failed.`);
195
+ if (failed !== 0) {
196
+ log.error("pendings", `${failed} batches failed.`);
188
197
  process.exit(1);
189
198
  }
190
- if (failedBatches.length === 0) {
191
- // await file.delete();
192
- // log.info("Object deleted");
199
+ if (failed === 0) {
200
+ await file.delete();
201
+ log.info("Object deleted");
193
202
  }
194
203
  };
195
204
  main();
@@ -3,6 +3,7 @@ if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
3
3
  throw new Error("[ENV]: missing variable");
4
4
  import { log } from "../../utils/logger";
5
5
  import { apiDbClient } from "../../utils/prisma";
6
+ import { withRetry } from "@sdk";
6
7
  import { S3Client } from "bun";
7
8
  import moment from "moment";
8
9
  // ─── Global Variables ────────────────────────────────────────
@@ -32,7 +33,7 @@ const extract = async () => {
32
33
  data.push(...transform(JSON.parse(line)));
33
34
  }
34
35
  try {
35
- count += await load(data);
36
+ count += await withRetry(load, [data], 5, 60_000);
36
37
  data.length = 0;
37
38
  }
38
39
  catch (err) {
@@ -79,8 +80,7 @@ export const main = async () => {
79
80
  process.exit(1);
80
81
  }
81
82
  if (failedBatches.length === 0) {
82
- // await file.delete();
83
- // log.info("Object deleted");
83
+ await file.delete();
84
84
  }
85
85
  };
86
86
  main();
@@ -3,6 +3,7 @@ if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
3
3
  throw new Error("[ENV]: missing variable");
4
4
  import { log } from "../../utils/logger";
5
5
  import { apiDbClient } from "../../utils/prisma";
6
+ import { withRetry } from "@sdk";
6
7
  import { S3Client } from "bun";
7
8
  import moment from "moment";
8
9
  // ─── Global Variables ────────────────────────────────────────
@@ -32,7 +33,7 @@ const extract = async () => {
32
33
  data.push(transform(JSON.parse(line)));
33
34
  }
34
35
  try {
35
- count += await load(data);
36
+ count += await withRetry(load, [data], 5, 60_000);
36
37
  data.length = 0;
37
38
  }
38
39
  catch (err) {