@merkl/api 0.10.392 → 0.10.393

package/dist/src/jobs/etl/pendings.js

@@ -1,55 +1,52 @@
 // ─── Pending Rewards Etl ─────────────────────────────────────────────────────
 if (!process.env.ENV || !process.env.FILENAME)
     throw new Error("[ENV]: missing variable");
-import { BucketService } from "../../modules/v4/bucket/bucket.service";
 import { RewardService } from "../../modules/v4/reward";
-import { withRetry } from "@sdk";
+import { S3Client } from "bun";
 import moment from "moment";
-import { log } from "../../utils/logger";
 // ─── Constants ───────────────────────────────────────────────
 const BATCH_SIZE = 20_000;
 // ─── Global Variables ────────────────────────────────────────
 const [chainIdString, root, campaignId] = process.env.FILENAME.split("_");
 const chainId = Number.parseInt(chainIdString);
-const pendingsToCreate = [[]];
+const gcsClient = new S3Client({
+    accessKeyId: process.env.GCS_ACCESS_ID,
+    secretAccessKey: process.env.GCS_SECRET,
+    endpoint: process.env.GCS_ENDPOINT,
+    bucket: `merkl-rewards-lake-${process.env.ENV}`,
+});
+const file = gcsClient.file(`pendings/${process.env.FILENAME}`);
 const failedBatches = [];
 // ─── Extract ─────────────────────────────────────────────────────────────────
 const extract = async () => {
+    if (!file.exists())
+        throw new Error("File does not exist.");
+    const textDecoder = new TextDecoder();
+    let buffer = "";
+    const stream = file.stream();
     let count = 0;
-    let currentBatchIndex = 0;
-    await BucketService.readStreamFromBucket(`pendings/${process.env.FILENAME}.gz`, `merkl-rewards-lake-${process.env.ENV}`, `merkl-data-${process.env.ENV}`, async (x) => {
-        pendingsToCreate[currentBatchIndex].push(transform(JSON.parse(x)));
-        if (pendingsToCreate[currentBatchIndex].length >= BATCH_SIZE) {
-            try {
-                count += await withRetry(load, [pendingsToCreate[currentBatchIndex]], 5, 10_000);
-                log.info(`Successfully inserted a batch of ${count} rewards`);
+    const data = [];
+    for await (const chunk of stream) {
+        buffer += textDecoder.decode(chunk);
+        const lines = buffer.split("\n");
+        buffer = lines.pop() || "";
+        for (const line of lines) {
+            data.push(JSON.parse(line));
+            if (data.length >= BATCH_SIZE) {
+                try {
+                    count += await load(data);
+                }
+                catch (err) {
+                    console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
+                    failedBatches.push(data);
+                    data.length = 0;
+                }
             }
-            catch (err) {
-                console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
-                failedBatches.push(currentBatchIndex);
-            }
-            currentBatchIndex++;
-            pendingsToCreate.push([]);
         }
-        return;
-    });
-    // ─── Current Batch Not In DB Yet ─────────────────────────────────────
-    try {
-        const count = await load(pendingsToCreate[currentBatchIndex]);
-        if (count !== 0)
-            log.info(`Successfully inserted a batch of ${count} rewards`);
-    }
-    catch (err) {
-        console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
-        failedBatches.push(currentBatchIndex);
     }
     return count;
 };
-// ─── Transform ───────────────────────────────────────────────────────────────
-const transform = (pending) => {
-    return pending;
-};
-// ─── Load ────────────────────────────────────────────────────────────────────
+// ─── Transform & Load ────────────────────────────────────────────────────────
 const load = async (pendings) => {
     const { updated, created } = await RewardService.updatePendings({
         distributionChainId: chainId,
@@ -65,11 +62,14 @@ export const main = async () => {
     const start = moment().unix();
     // ─── Start Rewards ETL ───────────────────────────────────────────────
     const count = await extract();
-    log.info(`✅ Successfully created ${count} new records in ${moment().unix() - start} sec`);
-    if (failedBatches.length !== 0)
-        log.info(`${failedBatches.length}/${pendingsToCreate.length} batches failed.`);
+    console.log(`✅ Successfully created ${count} new records in ${moment().unix() - start} sec`);
+    if (failedBatches.length !== 0) {
+        console.log(`${failedBatches.length} batches failed.`);
+        process.exit(1);
+    }
     if (failedBatches.length === 0) {
-        await BucketService.deleteFile(`rewards/${process.env.FILENAME}.gz`, `merkl-rewards-lake-${process.env.ENV}`, `merkl-data-${process.env.ENV}`);
+        await file.delete();
+        console.log("Object deleted");
     }
 };
 main();