@merkl/api 0.10.413 → 0.10.414

package/dist/src/jobs/etl/pendings.js

@@ -3,6 +3,7 @@ if (!process.env.ENV || !process.env.FILENAME)
     throw new Error("[ENV]: missing variable");
 import { log } from "../../utils/logger";
 import { apiDbClient } from "../../utils/prisma";
+import { withRetry } from "@sdk";
 import { S3Client } from "bun";
 import moment from "moment";
 // ─── Constants ───────────────────────────────────────────────
@@ -17,7 +18,6 @@ const gcsClient = new S3Client({
     bucket: `merkl-rewards-lake-${process.env.ENV}`,
 });
 const file = gcsClient.file(`pendings/${process.env.FILENAME}`);
-const failedBatches = [];
 // ─── Extract ─────────────────────────────────────────────────────────────────
 const extract = async () => {
     if (!file.exists())
@@ -25,8 +25,8 @@ const extract = async () => {
     const textDecoder = new TextDecoder();
     let buffer = "";
     const stream = file.stream();
-    let count = 0;
-    const data = [];
+    let data = [];
+    const loadPromises = [];
     for await (const chunk of stream) {
         buffer += textDecoder.decode(chunk);
         const lines = buffer.split("\n");
@@ -36,26 +36,21 @@ const extract = async () => {
         }
         if (data.length < MAX_BATCH_SIZE)
             continue;
-        try {
-            count += await load(data);
-        }
-        catch (err) {
-            console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
-            failedBatches.push(data);
-            data.length = 0;
-        }
+        loadPromises.push(withRetry(load, [data], 5, 60_000));
+        data = [];
     }
-    if (data.length === 0)
-        return count;
-    try {
-        count += await load(data);
-    }
-    catch (err) {
-        console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
-        failedBatches.push(data);
-        data.length = 0;
+    // Final batch
+    loadPromises.push(withRetry(load, [data], 5, 60_000));
+    let count = 0;
+    let failed = 0;
+    const promiseResults = await Promise.allSettled(loadPromises);
+    for (const x of promiseResults) {
+        if (x.status === "rejected")
+            failed += 1;
+        else
+            count += x.value;
     }
-    return count;
+    return { count, failed };
 };
 // ─── Transform & Load ────────────────────────────────────────────────────────
 const load = async (pendings) => {
@@ -195,15 +190,15 @@ const updatePendings = async (data) => {
 export const main = async () => {
     const start = moment().unix();
     // ─── Start Rewards ETL ───────────────────────────────────────────────
-    const count = await extract();
+    const { count, failed } = await extract();
     log.info(`✅ Successfully created ${count} new records for ${process.env.FILENAME} in ${moment().unix() - start} sec`);
-    if (failedBatches.length !== 0) {
-        log.error("pendings", `${failedBatches.length} batches failed.`);
+    if (failed !== 0) {
+        log.error("pendings", `${failed} batches failed.`);
         process.exit(1);
     }
-    if (failedBatches.length === 0) {
-        // await file.delete();
-        // log.info("Object deleted");
+    if (failed === 0) {
+        await file.delete();
+        log.info("Object deleted");
     }
 };
 main();