@merkl/api 0.10.416 → 0.10.417

package/dist/src/jobs/etl/pendings.js

@@ -7,7 +7,7 @@ import { withRetry } from "@sdk";
 import { S3Client } from "bun";
 import moment from "moment";
 // ─── Constants ───────────────────────────────────────────────
-const MAX_BATCH_SIZE = 10_000;
+const MAX_BATCH_SIZE = 1_000;
 // ─── Global Variables ────────────────────────────────────────
 const [chainIdString, root, campaignId] = process.env.FILENAME.replace(".gz", "").split("_");
 const chainId = Number.parseInt(chainIdString);
@@ -36,24 +36,32 @@ const extract = async () => {
         }
         if (data.length < MAX_BATCH_SIZE)
             continue;
-        loadPromises.push(withRetry(load, [data], 5, 10_000));
+        const promise = withRetry(load, [data], 5, 10_000);
+        loadPromises.push(promise);
+        await promise;
         data = [];
     }
     // Final batch
-    loadPromises.push(withRetry(load, [data], 5, 10_000));
-    let count = 0;
+    const promise = withRetry(load, [data], 5, 10_000);
+    loadPromises.push(promise);
+    await promise;
+    let created = 0;
+    let updated = 0;
     let failed = 0;
     const promiseResults = await Promise.allSettled(loadPromises);
     for (const x of promiseResults) {
         if (x.status === "rejected")
             failed += 1;
-        else
-            count += x.value;
+        else {
+            updated += x.value.updated;
+            created += x.value.created;
+        }
     }
-    return { count, failed, batch: promiseResults.length };
+    return { created, updated, failed, batch: promiseResults.length };
 };
 // ─── Transform & Load ────────────────────────────────────────────────────────
 const load = async (pendings) => {
+    log.info(`pushing ${pendings.length} pendings`);
     const { updated, created } = await updatePendings({
         distributionChainId: chainId,
         rewardToken: pendings[0].rewardToken,
@@ -61,7 +69,8 @@ const load = async (pendings) => {
         root,
         data: pendings,
     });
-    return updated + created;
+    log.info(`✅ updated ${updated} and created ${created} pendings`);
+    return { updated, created };
 };
 const updatePendings = async (data) => {
     const rewardTokenId = Bun.hash(`${data.distributionChainId}${data.rewardToken}`).toString();
@@ -100,7 +109,9 @@ const updatePendings = async (data) => {
     for (const [pointIndex, point] of data.data.entries()) {
         updateRewardUniques(point.recipient, point.pending, breakdownExists[pointIndex]?.pending);
         if (!!breakdownExists[pointIndex]) {
-            breakdownToUpdate.push(point);
+            if (point.pending !== breakdownExists[pointIndex].pending) {
+                breakdownToUpdate.push(point);
+            }
         }
         else {
             breakdownToCreate.push(point);
@@ -118,72 +129,82 @@ const updatePendings = async (data) => {
     })));
     for (const [pointIndex, point] of Object.values(rewardUniques).entries()) {
         if (!!exists[pointIndex]) {
-            toUpdate.push({
-                ...point,
-                pending: (BigInt(exists[pointIndex].pending) + BigInt(point.pending)).toString(), // Store the delta
-            });
+            if (point.pending !== "0") {
+                toUpdate.push({
+                    ...point,
+                    pending: (BigInt(exists[pointIndex].pending) + BigInt(point.pending)).toString(), // Store the delta
+                });
+            }
         }
         else {
             toCreate.push(point);
         }
     }
-    await apiDbClient.$transaction(toUpdate.map(x => {
-        const rewardId = Bun.hash(`${data.root}${x.recipient}${rewardTokenId}`).toString();
-        return apiDbClient.reward.update({
-            where: {
-                id: rewardId,
-            },
-            data: {
-                pending: x.pending,
-            },
+    console.log(`rewards to create: ${toCreate.length}, to update: ${toUpdate.length}`);
+    if (toCreate.length > 0) {
+        const users = toCreate.map(x => x.recipient);
+        await apiDbClient.user.createMany({ data: users.map(x => ({ address: x, tags: [] })), skipDuplicates: true });
+        await apiDbClient.reward.createMany({
+            data: toCreate.map(x => {
+                const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
+                return {
+                    id: rewardId,
+                    root,
+                    recipient: x.recipient,
+                    rewardTokenId,
+                    proofs: [],
+                    amount: "0",
+                    pending: x.pending,
+                    claimed: "0",
+                };
+            }),
         });
-    }));
-    let users = toCreate.map(x => x.recipient);
-    await apiDbClient.user.createMany({ data: users.map(x => ({ address: x, tags: [] })), skipDuplicates: true });
-    await apiDbClient.reward.createMany({
-        data: toCreate.map(x => {
-            const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
-            return {
-                id: rewardId,
-                root,
-                recipient: x.recipient,
-                rewardTokenId,
-                proofs: [],
-                amount: "0",
-                pending: x.pending,
-                claimed: "0",
-            };
-        }),
-    });
-    await apiDbClient.$transaction(breakdownToUpdate.map(x => {
-        return apiDbClient.rewardBreakdown.update({
-            where: {
-                rewardId_campaignId_reason: {
-                    rewardId: Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString(),
-                    campaignId: campaignId,
-                    reason: x.reason,
+    }
+    if (toUpdate.length > 0) {
+        await apiDbClient.$transaction(toUpdate.map(x => {
+            const rewardId = Bun.hash(`${data.root}${x.recipient}${rewardTokenId}`).toString();
+            return apiDbClient.reward.update({
+                where: {
+                    id: rewardId,
                 },
-            },
-            data: {
-                pending: x.pending,
-            },
+                data: {
+                    pending: x.pending,
+                },
+            });
+        }));
+    }
+    console.log(`breakdowns to create: ${breakdownToCreate.length}, to update: ${breakdownToUpdate.length}`);
+    if (breakdownToUpdate.length > 0) {
+        await apiDbClient.$transaction(breakdownToUpdate.map(x => {
+            return apiDbClient.rewardBreakdown.update({
+                where: {
+                    rewardId_campaignId_reason: {
+                        rewardId: Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString(),
+                        campaignId: campaignId,
+                        reason: x.reason,
+                    },
+                },
+                data: {
+                    pending: x.pending,
+                },
+            });
+        }));
+    }
+    if (breakdownToCreate.length > 0) {
+        await apiDbClient.rewardBreakdown.createMany({
+            data: breakdownToCreate.map(x => {
+                const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
+                return {
+                    reason: x.reason,
+                    amount: "0",
+                    pending: x.pending,
+                    claimed: "0",
+                    rewardId,
+                    campaignId,
+                };
+            }),
         });
-    }));
-    users = toCreate.map(x => x.recipient);
-    await apiDbClient.user.createMany({ data: users.map(x => ({ address: x, tags: [] })), skipDuplicates: true });
-    await apiDbClient.rewardBreakdown.createMany({
-        data: breakdownToCreate.map(x => {
-            const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
-            return {
-                reason: x.reason,
-                amount: "0",
-                pending: x.pending,
-                claimed: "0",
-                rewardId,
-                campaignId,
-            };
-        }),
-    });
+    }
     return { created: breakdownToCreate.length, updated: breakdownToUpdate.length };
 };
 // ─────────────────────────────────────────────────────────────────────────────
@@ -191,8 +212,8 @@ export const main = async () => {
     log.info(`✅ Running for ${process.env.FILENAME}`);
     const start = moment().unix();
     // ─── Start Rewards ETL ───────────────────────────────────────────────
-    const { count, failed, batch } = await extract();
-    log.info(`✅ Successfully created ${count} new records in ${batch} batches for ${process.env.FILENAME} in ${moment().unix() - start} sec`);
+    const { created, updated, failed, batch } = await extract();
+    log.info(`✅ Successfully created ${created} and updated ${updated} pendings records in ${batch} batches for ${process.env.FILENAME} in ${moment().unix() - start} sec`);
     if (failed !== 0) {
         log.error("pendings", `${failed} batches failed.`);
         process.exit(1);