@merkl/api 0.10.412 → 0.10.414

package/dist/src/jobs/etl/pendings.js

@@ -3,8 +3,11 @@ if (!process.env.ENV || !process.env.FILENAME)
     throw new Error("[ENV]: missing variable");
 import { log } from "../../utils/logger";
 import { apiDbClient } from "../../utils/prisma";
+import { withRetry } from "@sdk";
 import { S3Client } from "bun";
 import moment from "moment";
+// ─── Constants ───────────────────────────────────────────────
+const MAX_BATCH_SIZE = 10_000;
 // ─── Global Variables ────────────────────────────────────────
 const [chainIdString, root, campaignId] = process.env.FILENAME.replace(".gz", "").split("_");
 const chainId = Number.parseInt(chainIdString);
@@ -15,7 +18,6 @@ const gcsClient = new S3Client({
     bucket: `merkl-rewards-lake-${process.env.ENV}`,
 });
 const file = gcsClient.file(`pendings/${process.env.FILENAME}`);
-const failedBatches = [];
 // ─── Extract ─────────────────────────────────────────────────────────────────
 const extract = async () => {
     if (!file.exists())
@@ -23,8 +25,8 @@ const extract = async () => {
     const textDecoder = new TextDecoder();
     let buffer = "";
     const stream = file.stream();
-    let count = 0;
-    const data = [];
+    let data = [];
+    const loadPromises = [];
     for await (const chunk of stream) {
         buffer += textDecoder.decode(chunk);
         const lines = buffer.split("\n");
@@ -32,16 +34,23 @@ const extract = async () => {
         for (const line of lines) {
             data.push(JSON.parse(line));
         }
-        try {
-            count += await load(data);
-        }
-        catch (err) {
-            console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
-            failedBatches.push(data);
-            data.length = 0;
-        }
+        if (data.length < MAX_BATCH_SIZE)
+            continue;
+        loadPromises.push(withRetry(load, [data], 5, 60_000));
+        data = [];
+    }
+    // Final batch
+    loadPromises.push(withRetry(load, [data], 5, 60_000));
+    let count = 0;
+    let failed = 0;
+    const promiseResults = await Promise.allSettled(loadPromises);
+    for (const x of promiseResults) {
+        if (x.status === "rejected")
+            failed += 1;
+        else
+            count += x.value;
     }
-    return count;
+    return { count, failed };
 };
 // ─── Transform & Load ────────────────────────────────────────────────────────
 const load = async (pendings) => {
@@ -181,15 +190,15 @@ const updatePendings = async (data) => {
 export const main = async () => {
     const start = moment().unix();
     // ─── Start Rewards ETL ───────────────────────────────────────────────
-    const count = await extract();
+    const { count, failed } = await extract();
     log.info(`✅ Successfully created ${count} new records for ${process.env.FILENAME} in ${moment().unix() - start} sec`);
-    if (failedBatches.length !== 0) {
-        log.error("pendings", `${failedBatches.length} batches failed.`);
+    if (failed !== 0) {
+        log.error("pendings", `${failed} batches failed.`);
         process.exit(1);
     }
-    if (failedBatches.length === 0) {
-        // await file.delete();
-        // log.info("Object deleted");
+    if (failed === 0) {
+        await file.delete();
+        log.info("Object deleted");
     }
 };
 main();

package/dist/src/jobs/etl/reward-breakdowns.js

@@ -3,6 +3,7 @@ if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
     throw new Error("[ENV]: missing variable");
 import { log } from "../../utils/logger";
 import { apiDbClient } from "../../utils/prisma";
+import { withRetry } from "@sdk";
 import { S3Client } from "bun";
 import moment from "moment";
 // ─── Global Variables ────────────────────────────────────────
@@ -32,7 +33,7 @@ const extract = async () => {
                 data.push(...transform(JSON.parse(line)));
             }
         try {
-            count += await load(data);
+            count += await withRetry(load, [data], 5, 60_000);
             data.length = 0;
         }
         catch (err) {
@@ -79,8 +80,7 @@ export const main = async () => {
         process.exit(1);
     }
     if (failedBatches.length === 0) {
-        // await file.delete();
-        // log.info("Object deleted");
+        await file.delete();
     }
 };
 main();

package/dist/src/jobs/etl/rewards.js

@@ -3,6 +3,7 @@ if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
     throw new Error("[ENV]: missing variable");
 import { log } from "../../utils/logger";
 import { apiDbClient } from "../../utils/prisma";
+import { withRetry } from "@sdk";
 import { S3Client } from "bun";
 import moment from "moment";
 // ─── Global Variables ────────────────────────────────────────
@@ -32,7 +33,7 @@ const extract = async () => {
                 data.push(transform(JSON.parse(line)));
         }
         try {
-            count += await load(data);
+            count += await withRetry(load, [data], 5, 60_000);
             data.length = 0;
         }
         catch (err) {