@merkl/api 0.15.41 → 0.15.42

package/dist/src/eden/index.d.ts

@@ -1753,9 +1753,12 @@ declare const eden: {
                     fetch?: RequestInit | undefined;
                 }) => Promise<import("@elysiajs/eden").Treaty.TreatyResponse<{
                     200: {
-                        name: string;
-                        id: number;
-                        icon: string;
+                        Explorer: {
+                            type: import("../../database/api/.generated").$Enums.ExplorerType;
+                            url: string;
+                            id: string;
+                            chainId: number;
+                        }[];
                     };
                 }>>;
             };
@@ -4872,9 +4875,12 @@ declare const eden: {
                     fetch?: RequestInit | undefined;
                 }) => Promise<import("@elysiajs/eden").Treaty.TreatyResponse<{
                     200: {
-                        name: string;
-                        id: number;
-                        icon: string;
+                        Explorer: {
+                            type: import("../../database/api/.generated").$Enums.ExplorerType;
+                            url: string;
+                            id: string;
+                            chainId: number;
+                        }[];
                     };
                 }>>;
             };
@@ -9062,9 +9068,12 @@ export declare const MerklApi: (domain: string | import("elysia").default<"", fa
                     };
                     response: {
                         200: {
-                            name: string;
-                            id: number;
-                            icon: string;
+                            Explorer: {
+                                type: import("../../database/api/.generated").$Enums.ExplorerType;
+                                url: string;
+                                id: string;
+                                chainId: number;
+                            }[];
                         };
                     };
                 };
@@ -13466,9 +13475,12 @@ export declare const MerklApi: (domain: string | import("elysia").default<"", fa
                     fetch?: RequestInit | undefined;
                 }) => Promise<import("@elysiajs/eden").Treaty.TreatyResponse<{
                     200: {
-                        name: string;
-                        id: number;
-                        icon: string;
+                        Explorer: {
+                            type: import("../../database/api/.generated").$Enums.ExplorerType;
+                            url: string;
+                            id: string;
+                            chainId: number;
+                        }[];
                     };
                 }>>;
             };
@@ -16585,9 +16597,12 @@ export declare const MerklApi: (domain: string | import("elysia").default<"", fa
                     fetch?: RequestInit | undefined;
                 }) => Promise<import("@elysiajs/eden").Treaty.TreatyResponse<{
                     200: {
-                        name: string;
-                        id: number;
-                        icon: string;
+                        Explorer: {
+                            type: import("../../database/api/.generated").$Enums.ExplorerType;
+                            url: string;
+                            id: string;
+                            chainId: number;
+                        }[];
                     };
                 }>>;
             };

package/dist/src/index.d.ts

@@ -2126,9 +2126,12 @@ declare const app: Elysia<"", false, {
                     };
                     response: {
                         200: {
-                            name: string;
-                            id: number;
-                            icon: string;
+                            Explorer: {
+                                type: import("../database/api/.generated").$Enums.ExplorerType;
+                                url: string;
+                                id: string;
+                                chainId: number;
+                            }[];
                         };
                     };
                 };

package/dist/src/jobs/etl/reward-breakdowns.js

@@ -1,9 +1,9 @@
 // ─── Reward Breakdowns ETL ───────────────────────────────────────────────────
-if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
+if (!process.env.DATABASE_API_URL || !process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
     throw new Error("[ENV]: missing variable");
+import { BucketService } from "../../modules/v4/bucket/bucket.service";
 import { log } from "../../utils/logger";
 import { apiDbClient } from "../../utils/prisma";
-import { withRetry } from "@sdk";
 import { S3Client } from "bun";
 import moment from "moment";
 // ─── Global Variables ────────────────────────────────────────
@@ -13,10 +13,9 @@ const gcsClient = new S3Client({
     endpoint: process.env.GCS_ENDPOINT,
     bucket: `merkl-rewards-lake-${process.env.ENV}`,
 });
-let file = gcsClient.file(`breakdowns/${process.env.CHAIN_ID}-${process.env.ROOT}`);
-if (!(await file.exists()))
-    file = gcsClient.file(`breakdowns/${process.env.CHAIN_ID}-${process.env.ROOT}.gz`);
+const file = gcsClient.file(`breakdowns/${process.env.CHAIN_ID}-${process.env.ROOT}`);
 const failedBatches = [];
+const missingCampaigns = [];
 // ─── Extract ─────────────────────────────────────────────────────────────────
 const extract = async () => {
     if (!file.exists())
@@ -30,106 +29,115 @@ const extract = async () => {
         buffer += textDecoder.decode(chunk);
         const lines = buffer.split("\n");
         buffer = lines.pop() || "";
-        for (const line of lines)
-            if (line.trim()) {
+        for (const line of lines) {
+            if (line.trim())
                 data.push(...(await transform(JSON.parse(line))));
-            }
+        }
         if (data.length < 30_000)
             continue;
         try {
-            count += await withRetry(load, [data], 5, 60_000);
+            count += await load(data);
             data = [];
         }
         catch (err) {
-            console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
+            // log.error("Failed to insert a batch, adding it to the fail queue.", err);
             failedBatches.push(data);
             data = [];
         }
     }
     try {
-        count += await withRetry(load, [data], 5, 60_000);
+        count += await load(data);
         data = [];
     }
     catch (err) {
-        console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
+        // log.error("Failed to insert a batch, adding it to the fail queue.", err);
         failedBatches.push(data);
         data = [];
     }
     return count;
 };
 // ─── Transform ───────────────────────────────────────────────────────────────
-const transform = async (rewardBreakdowns) => {
-    const transformedRewardBreakdowns = [];
-    for (const rewardBreakdown of rewardBreakdowns) {
-        const rewardTokenId = Bun.hash(`${process.env.CHAIN_ID}${rewardBreakdown.token}`).toString();
-        const rewardId = Bun.hash(`${process.env.ROOT}${rewardBreakdown.recipient}${rewardTokenId}`).toString();
-        transformedRewardBreakdowns.push({
+const transform = (rewardBreakdowns) => {
+    const transformedBreakdowns = Array(rewardBreakdowns.length);
+    let i = 0;
+    for (i; i < rewardBreakdowns.length; i++) {
+        const rewardTokenId = Bun.hash(`${process.env.CHAIN_ID}${rewardBreakdowns[i].token}`).toString();
+        const rewardId = Bun.hash(`${process.env.ROOT}${rewardBreakdowns[i].recipient}${rewardTokenId}`).toString();
+        const campaignId = Bun.hash(`${process.env.CHAIN_ID}${rewardBreakdowns[i].campaignId}`).toString();
+        transformedBreakdowns[i] = {
             rewardId,
-            protocolId: rewardBreakdown.protocolId ? rewardBreakdown.protocolId : undefined,
-            campaignId: rewardBreakdown.campaignId,
-            reason: rewardBreakdown.reason ? rewardBreakdown.reason : "",
-            amount: rewardBreakdown.amount,
-            claimed: rewardBreakdown.claimed,
-            pending: rewardBreakdown.pending,
-        });
+            protocolId: rewardBreakdowns[i].protocolId ? rewardBreakdowns[i].protocolId : undefined,
+            campaignId,
+            reason: rewardBreakdowns[i].reason ? rewardBreakdowns[i].reason : "",
+            amount: rewardBreakdowns[i].amount,
+            claimed: rewardBreakdowns[i].claimed,
+            pending: rewardBreakdowns[i].pending,
+            merklCampaignId: rewardBreakdowns[i].campaignId,
+        };
     }
-    return transformedRewardBreakdowns;
+    return transformedBreakdowns;
 };
 // ─── Load ────────────────────────────────────────────────────────────────────
 const load = async (rewardBreakdowns) => {
-    log.info(`pushing ${rewardBreakdowns.length} rewardBreakdowns`);
-    // const missingCampaigns: string[] = [];
-    // const foundCampaigns: string[] = [];
-    // const campaigns = rewardBreakdowns.reduce(
-    //   (acc, x) => {
-    //     const campaignUnique = {
-    //       campaignId: x.campaignId,
-    //       distributionChain: Number.parseInt(process.env.CHAIN_ID!),
-    //     };
-    //     if (!Object.keys(acc).includes(CampaignService.hashId(campaignUnique)))
-    //       acc[CampaignService.hashId(campaignUnique)] = {
-    //         campaignId: x.campaignId,
-    //         distributionChain: Number.parseInt(process.env.CHAIN_ID!),
-    //       };
-    //     return acc;
-    //   },
-    //   {} as Record<string, CampaignUnique>
-    // );
-    // for (const campaign of Object.values(campaigns)) {
-    //   const campaignExists = await CampaignService.checkIfExist(campaign);
-    //   if (!campaignExists) {
-    //     const { success, fail } = await CampaignService.fill([campaign]);
-    //     if (fail === 1 || success !== 1) {
-    //       missingCampaigns.push(CampaignService.hashId(campaign));
-    //       log.warn(`createManyBreakdown - Missing campaign: ${campaign.campaignId}`);
-    //       continue;
-    //     }
-    //   }
-    //   foundCampaigns.push(CampaignService.hashId(campaign));
-    // }
-    // if (missingCampaigns.length !== 0) {
-    //   log.warn(`createManyBreakdown - Missing ${missingCampaigns.length} campaigns: ${missingCampaigns.join(", ")}`);
-    // }
-    return (await apiDbClient.rewardBreakdown.createMany({
-        data: rewardBreakdowns.map(x => {
-            const campaignId = Bun.hash(`${process.env.CHAIN_ID}${x.campaignId}`).toString();
-            return { ...x, campaignId };
-        }),
-        // .filter(x => !missingCampaigns.includes(x.campaignId)),
+    const count = (await apiDbClient.rewardBreakdown.createMany({
+        data: rewardBreakdowns,
         skipDuplicates: true,
     })).count;
+    log.info(`Successfully inserted ${rewardBreakdowns.length} reward breakdown(s).`);
+    return count;
+};
+// ─── Retry ───────────────────────────────────────────────────────────────────
+// Using binary search
+const retry = async (batches) => {
+    log.info(`Retrying ${batches.length} batch(es)...`);
+    let inserted = 0;
+    while (batches.length > 0) {
+        const currentBatch = batches.shift();
+        const firstHalf = currentBatch.splice(0, Math.ceil(currentBatch.length / 2));
+        const secondHalf = currentBatch;
+        // Process first half
+        try {
+            inserted += await load(firstHalf);
+        }
+        catch (err) {
+            log.error(JSON.stringify(firstHalf), err);
+            if (firstHalf.length > 1)
+                batches.push(firstHalf);
+            else
+                missingCampaigns.push(firstHalf[0].merklCampaignId);
+        }
+        // Process second half
+        try {
+            inserted += await load(secondHalf);
+        }
+        catch (err) {
+            log.error(JSON.stringify(secondHalf), err);
+            if (secondHalf.length > 1)
+                batches.push(secondHalf);
+            else
+                missingCampaigns.push(secondHalf[0].merklCampaignId);
+        }
+    }
+    return inserted;
 };
 // ─── Main ────────────────────────────────────────────────────────────────────
 export const main = async () => {
     const start = moment().unix();
     const count = await extract();
-    log.info(`✅ Successfully created ${count} new records for ${process.env.CHAIN_ID}-${process.env.ROOT} in ${moment().unix() - start} sec`);
+    log.info(`✅ Successfully created ${count} new record(s) for ${process.env.CHAIN_ID}-${process.env.ROOT} in ${moment().unix() - start} sec.`);
     if (failedBatches.length !== 0) {
-        log.error("breakdowns", `${failedBatches.length} batches failed.`);
-        process.exit(1);
-    }
-    if (failedBatches.length === 0) {
-        // await file.delete();
+        const inserted = await retry(failedBatches);
+        log.info(`Successfully inserted ${inserted} record(s) after retrying.`);
+        if (inserted !== failedBatches.length) {
+            log.error("breakdowns", `${failedBatches.length} breakdown(s) failed to insert.`);
+            // ─── Push Missing Campaigns To Bucket ────────────────
+            const bucket = new BucketService(`merkl-campaigns-lake-${process.env.ENV}`, `merkl-data-${process.env.ENV}`);
+            await bucket.push(`${process.env.CHAIN_ID}_missing_campaigns`, JSON.stringify(missingCampaigns), {
+                type: "application/json",
+            });
+            process.exit(1);
+        }
     }
+    // if (failedBatches.length === 0) await file.delete();
 };
 main();

package/dist/src/modules/v4/bucket/bucket.service.d.ts

@@ -4,7 +4,7 @@ export declare class BucketService {
     defaultUploadOptions: UploadOptions<unknown>;
     constructor(bucket: string, projectId: string);
     pushArray<T>(filename: string, arr: T[], options?: UploadOptions<T>): Promise<number>;
-    push(filename: string, text: string, options?: Omit<UploadOptions<never>, "transform">): Promise<number>;
+    push(filename: string, text: string, options?: Omit<UploadOptions<never>, "transform" | "separator">): Promise<number>;
     pullArray<T>(filename: string, callback: (elem: string) => T, separator?: string): Promise<T[]>;
     pull(filename: string): Promise<string>;
 }

package/dist/src/modules/v4/chain/chain.controller.d.ts

@@ -121,9 +121,12 @@ export declare const ChainController: Elysia<"/chains", false, {
                 };
                 response: {
                     200: {
-                        name: string;
-                        id: number;
-                        icon: string;
+                        Explorer: {
+                            type: import("../../../../database/api/.generated").$Enums.ExplorerType;
+                            url: string;
+                            id: string;
+                            chainId: number;
+                        }[];
                     };
                 };
             };

package/dist/src/modules/v4/chain/chain.repository.d.ts

@@ -46,15 +46,26 @@ export declare abstract class ChainRepository {
      * @returns
      */
     static create(data: CreateChainModel): Promise<{
-        name: string;
-        id: number;
-        icon: string;
+        Explorer: {
+            type: import("../../../../database/api/.generated").$Enums.ExplorerType;
+            url: string;
+            id: string;
+            chainId: number;
+        }[];
     }>;
     static findMany(): Promise<{
         name: string;
         id: number;
         icon: string;
     }[]>;
+    static findUniqueOrThrow(id: number): Promise<{
+        Explorer: {
+            type: import("../../../../database/api/.generated").$Enums.ExplorerType;
+            url: string;
+            id: string;
+            chainId: number;
+        }[];
+    }>;
     /**
      * Fetches the campaign dynamic data for a v3 campaign onchain
      * @param chainId

package/dist/src/modules/v4/chain/chain.repository.js

@@ -46,19 +46,29 @@ export class ChainRepository {
      * @returns
      */
     static async create(data) {
-        const explorerId = await ExplorerRepository.create(data.id, data.explorerType, data.explorerUrl);
-        return apiDbClient.chain.create({
+        await apiDbClient.chain.create({
             data: {
                 id: data.id,
                 icon: data.icon,
                 name: data.name,
-                Explorer: { connect: { id: explorerId.id } },
             },
         });
+        await ExplorerRepository.create(data.id, data.explorerType, data.explorerUrl);
+        return await ChainRepository.findUniqueOrThrow(data.id);
     }
     static async findMany() {
         return apiDbClient.chain.findMany();
     }
+    static async findUniqueOrThrow(id) {
+        return apiDbClient.chain.findUniqueOrThrow({
+            where: {
+                id,
+            },
+            select: {
+                Explorer: true,
+            },
+        });
+    }
     /**
      * Fetches the campaign dynamic data for a v3 campaign onchain
      * @param chainId

package/dist/src/modules/v4/chain/chain.service.d.ts

@@ -40,9 +40,12 @@ export declare abstract class ChainService {
      */
     static getSupportedIds(): Promise<number[]>;
     static create(chain: CreateChainModel): Promise<{
-        name: string;
-        id: number;
-        icon: string;
+        Explorer: {
+            type: import("../../../../database/api/.generated").$Enums.ExplorerType;
+            url: string;
+            id: string;
+            chainId: number;
+        }[];
     }>;
     static update(id: number, data: UpdateChainModel): Promise<{
         name: string;

package/dist/src/modules/v4/protocol/protocol.repository.d.ts

@@ -25,14 +25,46 @@ export declare abstract class ProtocolRepository {
         tags: string[];
         icon: string;
     }[]>;
-    static findMany(query: GetProtocolsQueryModel): Promise<{
+    static findMany(query: GetProtocolsQueryModel): Promise<({
+        MainOpportunities: ({
+            Campaigns: {
+                type: import("../../../../database/api/.generated").$Enums.CampaignType;
+                id: string;
+                params: import("database/api/.generated/runtime/library").JsonValue;
+                subType: number | null;
+                startTimestamp: bigint;
+                endTimestamp: bigint;
+                computeChainId: number;
+                distributionChainId: number;
+                campaignId: string;
+                rewardTokenId: string;
+                amount: string;
+                opportunityId: string;
+                creatorAddress: string;
+            }[];
+        } & {
+            name: string;
+            type: import("../../../../database/api/.generated").$Enums.CampaignType;
+            id: string;
+            status: import("../../../../database/api/.generated").$Enums.Status;
+            tags: string[];
+            identifier: string;
+            chainId: number;
+            action: import("../../../../database/api/.generated").$Enums.OpportunityAction;
+            depositUrl: string | null;
+            mainProtocolId: string | null;
+            tvl: number;
+            apr: number;
+            dailyRewards: number;
+        })[];
+    } & {
         name: string;
         url: string;
         description: string;
         id: string;
         tags: string[];
         icon: string;
-    }[]>;
+    })[]>;
     static countMany(query: GetProtocolsQueryModel): Promise<number>;
     static update(id: string, data: UpdateProtocolModel): Promise<{
         name: string;

package/dist/src/modules/v4/protocol/protocol.repository.js

@@ -52,6 +52,19 @@ export class ProtocolRepository {
             take: items,
             skip: page * items,
             ...args,
+            include: {
+                MainOpportunities: {
+                    include: {
+                        Campaigns: {
+                            where: {
+                                endTimestamp: {
+                                    gt: BigInt(Math.floor(Date.now() / 1000)),
+                                },
+                            },
+                        },
+                    },
+                },
+            },
         });
     }
     static async countMany(query) {

package/dist/src/modules/v4/protocol/protocol.service.js

@@ -39,7 +39,13 @@ export class ProtocolService {
         }[amm];
     }
     static async findMany(query) {
-        return ProtocolRepository.findMany(query);
+        const protocols = await ProtocolRepository.findMany(query);
+        const enrichedProtocols = protocols.map(({ MainOpportunities, ...protocol }) => ({
+            ...protocol,
+            dailyReward: MainOpportunities.reduce((sum, opportunity) => sum + opportunity.dailyRewards, 0),
+            numberOfCampaigns: MainOpportunities.reduce((sum, opportunity) => sum + opportunity.Campaigns.length, 0),
+        }));
+        return enrichedProtocols;
     }
     static async countMany(query) {
         return ProtocolRepository.countMany(query);

package/dist/src/modules/v4/router.d.ts

@@ -1996,9 +1996,12 @@ export declare const v4: Elysia<"/v4", false, {
                     };
                     response: {
                         200: {
-                            name: string;
-                            id: number;
-                            icon: string;
+                            Explorer: {
+                                type: import("../../../database/api/.generated").$Enums.ExplorerType;
+                                url: string;
+                                id: string;
+                                chainId: number;
+                            }[];
                         };
                     };
                 };