@merkl/api 0.21.47 → 1.0.0

package/dist/src/jobs/pendings.js

@@ -1,238 +0,0 @@
-// ─── Pending Rewards Etl ─────────────────────────────────────────────────────
-if (!process.env.ENV || !process.env.FILENAME)
-    throw new Error("[ENV]: missing variable");
-import { RewardService } from "@/modules/v4/reward/reward.service";
-import { log } from "@/utils/logger";
-import { apiDbClient } from "@db";
-import { NETWORK_LABELS, withRetry } from "@sdk";
-import { S3Client } from "bun";
-import moment from "moment";
-// ─── Constants ───────────────────────────────────────────────
-const MAX_BATCH_SIZE = 1_000;
-// ─── Global Variables ────────────────────────────────────────
-const [chainIdString, root, campaignId] = process.env.FILENAME.replace(".gz", "").split("_");
-const chainId = Number.parseInt(chainIdString);
-const gcsClient = new S3Client({
-    accessKeyId: process.env.GCS_ACCESS_ID,
-    secretAccessKey: process.env.GCS_SECRET,
-    endpoint: process.env.GCS_ENDPOINT,
-    bucket: `merkl-rewards-lake-${process.env.ENV}`,
-});
-const file = gcsClient.file(`pendings/${process.env.FILENAME}`);
-// ─── Extract ─────────────────────────────────────────────────────────────────
-const extract = async () => {
-    if (!file.exists())
-        throw new Error("File does not exist.");
-    const textDecoder = new TextDecoder();
-    let buffer = "";
-    const stream = file.stream();
-    let data = [];
-    const loadPromises = [];
-    for await (const chunk of stream) {
-        buffer += textDecoder.decode(chunk);
-        const lines = buffer.split("\n");
-        buffer = lines.pop() || "";
-        for (const line of lines) {
-            data.push(JSON.parse(line));
-        }
-        if (data.length < MAX_BATCH_SIZE)
-            continue;
-        // Retry up to 5 times with a 10s delay
-        const promise = withRetry(load, [data], 5, 10_000);
-        loadPromises.push(promise);
-        await promise;
-        data = [];
-    }
-    // Final batch
-    // Retry up to 5 times with a 10s delay
-    const promise = withRetry(load, [data], 5, 10_000);
-    loadPromises.push(promise);
-    await promise;
-    let created = 0;
-    let updated = 0;
-    let failed = 0;
-    const promiseResults = await Promise.allSettled(loadPromises);
-    for (const x of promiseResults) {
-        if (x.status === "rejected")
-            failed += 1;
-        else {
-            updated += x.value.updated;
-            created += x.value.created;
-        }
-    }
-    return { created, updated, failed, batch: promiseResults.length };
-};
-// ─── Transform & Load ────────────────────────────────────────────────────────
-const load = async (pendings) => {
-    if (pendings.length === 0)
-        return { updated: 0, created: 0 };
-    log.info(`pushing ${pendings.length} pendings for token ${pendings[0].rewardToken} on ${NETWORK_LABELS[chainId]}`);
-    const { updated, created } = await updatePendings({
-        distributionChainId: chainId,
-        rewardToken: pendings[0].rewardToken, // sometimes undefined
-        campaignId,
-        root,
-        data: pendings,
-    });
-    log.info(`✅ updated ${updated} and created ${created} pendings`);
-    return { updated, created };
-};
-const updatePendings = async (data) => {
-    const rewardTokenId = Bun.hash(`${data.distributionChainId}${data.rewardToken}`).toString();
-    const campaignId = Bun.hash(`${data.distributionChainId}${data.campaignId}`).toString();
-    // List of identifiers of breakdowns that will be touched by this jobd
-    const breakdownUniques = await data.data.map(({ recipient, reason }) => {
-        const rewardId = Bun.hash(`${data.root}${recipient}${rewardTokenId}`).toString();
-        return { rewardId, reason, campaignId };
-    });
-    // Check if these breakdowns already exist or not
-    const breakdownExists = await apiDbClient.$transaction(breakdownUniques.map(x => apiDbClient.rewardBreakdown.findUnique({
-        select: {
-            pending: true,
-        },
-        where: {
-            rewardId_campaignId_reason: {
-                rewardId: x.rewardId,
-                campaignId: x.campaignId,
-                reason: x.reason,
-            },
-        },
-    })));
-    // We want to get all current pendings for the rewardIds that will be modified
-    // Should contain rewardId => { pendingIncrease, recipient }
-    const rewardIdToPendingIncrease = {};
-    const breakdownToUpdate = [];
-    const breakdownToCreate = [];
-    let totalBreakdownIncrease = 0n;
-    // For all point to update
-    //  - compute the delta with what's in database, add it to rewardIdToPendingIncrease
-    //  - add the point to breakdownToUpdate or breakdownToCreate
-    for (const [pointIndex, point] of data.data.entries()) {
-        const rewardId = Bun.hash(`${data.root}${point.recipient}${rewardTokenId}`).toString();
-        if (!rewardIdToPendingIncrease[rewardId]) {
-            rewardIdToPendingIncrease[rewardId] = { pendingIncrease: 0n, recipient: point.recipient };
-        }
-        const delta = BigInt(point.pending) - BigInt(breakdownExists[pointIndex]?.pending ?? "0");
-        totalBreakdownIncrease += delta;
-        if (delta > 0n) {
-            rewardIdToPendingIncrease[rewardId].pendingIncrease += delta; // Store the delta
-            if (!!breakdownExists[pointIndex]) {
-                breakdownToUpdate.push(point);
-            }
-            else {
-                breakdownToCreate.push(point);
-            }
-        }
-    }
-    const rewardToUpdate = [];
-    const rewardToCreate = [];
-    const RewardExists = await apiDbClient.$transaction(Object.keys(rewardIdToPendingIncrease).map(x => apiDbClient.reward.findUnique({
-        select: {
-            pending: true,
-        },
-        where: {
-            id: x,
-        },
-    })));
-    for (const [pointIndex, point] of Object.values(rewardIdToPendingIncrease).entries()) {
-        if (BigInt(point.pendingIncrease) > 0n) {
-            const rewardExists = RewardExists[pointIndex];
-            if (!!rewardExists) {
-                rewardToUpdate.push({
-                    recipient: point.recipient,
-                    pending: BigInt(rewardExists.pending) + BigInt(point.pendingIncrease),
-                });
-            }
-            else {
-                rewardToCreate.push({
-                    recipient: point.recipient,
-                    pending: BigInt(point.pendingIncrease),
-                });
-            }
-        }
-    }
-    const totalRewardIncrease = Object.values(rewardIdToPendingIncrease).reduce((prev, current) => prev + BigInt(current.pendingIncrease), 0n);
-    log.info(`total increase: ${totalRewardIncrease.toString()}, rewards to create:    ${rewardToCreate.length}, to update: ${rewardToUpdate.length}`);
-    log.info(`total increase: ${totalBreakdownIncrease.toString()}, breakdowns to create: ${breakdownToCreate.length}, to update: ${breakdownToUpdate.length}`);
-    if (rewardToCreate.length > 0) {
-        const users = rewardToCreate.map(x => x.recipient);
-        await apiDbClient.user.createMany({ data: users.map(x => ({ address: x, tags: [] })), skipDuplicates: true });
-        await apiDbClient.reward.createMany({
-            data: rewardToCreate.map(x => {
-                const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
-                return {
-                    id: rewardId,
-                    root,
-                    recipient: x.recipient,
-                    rewardTokenId,
-                    proofs: [],
-                    amount: "0",
-                    pending: x.pending.toString(),
-                    claimed: "0",
-                };
-            }),
-        });
-    }
-    if (breakdownToCreate.length > 0) {
-        await apiDbClient.rewardBreakdown.createMany({
-            data: breakdownToCreate.map(x => {
-                const rewardId = RewardService.hashId(data.root, x.recipient, rewardTokenId);
-                return {
-                    reason: x.reason,
-                    amount: "0",
-                    pending: x.pending,
-                    claimed: "0",
-                    rewardId,
-                    campaignId,
-                };
-            }),
-        });
-    }
-    if (breakdownToUpdate.length > 0) {
-        await apiDbClient.$transaction(breakdownToUpdate.map(x => {
-            return apiDbClient.rewardBreakdown.update({
-                where: {
-                    rewardId_campaignId_reason: {
-                        rewardId: Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString(),
-                        campaignId: campaignId,
-                        reason: x.reason,
-                    },
-                },
-                data: {
-                    pending: x.pending,
-                },
-            });
-        }));
-    }
-    if (rewardToUpdate.length > 0) {
-        await apiDbClient.$transaction(rewardToUpdate.map(x => {
-            const rewardId = Bun.hash(`${data.root}${x.recipient}${rewardTokenId}`).toString();
-            return apiDbClient.reward.update({
-                where: {
-                    id: rewardId,
-                },
-                data: {
-                    pending: x.pending.toString(),
-                },
-            });
-        }));
-    }
-    return { created: breakdownToCreate.length, updated: breakdownToUpdate.length };
-};
-// ─────────────────────────────────────────────────────────────────────────────
-export const main = async () => {
-    log.info(`✅ Running for ${process.env.FILENAME}`);
-    const start = moment().unix();
-    // ─── Start Rewards ETL ───────────────────────────────────────────────
-    const { created, updated, failed, batch } = await extract();
-    log.info(`✅ Successfully created ${created} and updated ${updated} pendings records in ${batch} batches for ${process.env.FILENAME} in ${moment().unix() - start} sec`);
-    if (failed !== 0) {
-        log.error("pendings", `${failed} batches failed.`);
-        process.exit(1);
-    }
-    if (failed === 0) {
-        await file.delete();
-    }
-    process.exit(0);
-};
-main();

package/dist/src/jobs/prices.js

@@ -1,10 +0,0 @@
-import { TokenService } from "@/modules/v4/token/token.service";
-import { log } from "@/utils/logger";
-import { Pricer } from "@/utils/pricer";
-const pricer = await Pricer.load();
-const prices = await pricer.update();
-log.info(`✅ ${Object.keys(prices).length} prices cache updated successfully`);
-log.info("⏳ Updating API database...");
-await TokenService.updatePrices(pricer);
-log.info("✅ API database updated successfully!");
-process.exit(0);

package/dist/src/jobs/reward-breakdowns.js

@@ -1,163 +0,0 @@
-// ─── Reward Breakdowns ETL ───────────────────────────────────────────────────
-if (!process.env.DATABASE_API_URL || !process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
-    throw new Error("[ENV]: missing variable");
-import { BucketService } from "@/modules/v4/bucket/bucket.service";
-import { log } from "@/utils/logger";
-import { apiDbClient } from "@db";
-import { S3Client } from "bun";
-import moment from "moment";
-// ─── Global Variables ────────────────────────────────────────
-const gcsClient = new S3Client({
-    accessKeyId: process.env.GCS_ACCESS_ID,
-    secretAccessKey: process.env.GCS_SECRET,
-    endpoint: process.env.GCS_ENDPOINT,
-    bucket: `merkl-rewards-lake-${process.env.ENV}`,
-});
-const file = gcsClient.file(`breakdowns/${process.env.CHAIN_ID}-${process.env.ROOT}`);
-const failedBatches = [];
-const missingCampaigns = [];
-// ─── Extract ─────────────────────────────────────────────────────────────────
-const extract = async () => {
-    if (!file.exists())
-        throw new Error("File does not exist.");
-    let data = [];
-    const textDecoder = new TextDecoder();
-    let buffer = "";
-    const stream = file.stream();
-    let count = 0;
-    for await (const chunk of stream) {
-        buffer += textDecoder.decode(chunk);
-        const lines = buffer.split("\n");
-        buffer = lines.pop() || "";
-        for (const line of lines) {
-            if (line.trim())
-                data.push(...(await transform(JSON.parse(line))));
-        }
-        if (data.length < 30_000)
-            continue;
-        try {
-            count += await load(data);
-            data = [];
-        }
-        catch (err) {
-            // log.error("Failed to insert a batch, adding it to the fail queue.", err);
-            failedBatches.push(data);
-            data = [];
-        }
-    }
-    try {
-        count += await load(data);
-        data = [];
-    }
-    catch (err) {
-        // log.error("Failed to insert a batch, adding it to the fail queue.", err);
-        failedBatches.push(data);
-        data = [];
-    }
-    return count;
-};
-// ─── Transform ───────────────────────────────────────────────────────────────
-const transform = (rewardBreakdowns) => {
-    const transformedBreakdowns = Array(rewardBreakdowns.length);
-    let i = 0;
-    for (i; i < rewardBreakdowns.length; i++) {
-        const rewardTokenId = Bun.hash(`${process.env.CHAIN_ID}${rewardBreakdowns[i].token}`).toString();
-        const rewardId = Bun.hash(`${process.env.ROOT}${rewardBreakdowns[i].recipient}${rewardTokenId}`).toString();
-        const campaignId = Bun.hash(`${process.env.CHAIN_ID}${rewardBreakdowns[i].campaignId}`).toString();
-        transformedBreakdowns[i] = {
-            rewardId,
-            protocolId: rewardBreakdowns[i].protocolId ? rewardBreakdowns[i].protocolId : undefined,
-            campaignId,
-            reason: rewardBreakdowns[i].reason ? rewardBreakdowns[i].reason : "",
-            amount: rewardBreakdowns[i].amount,
-            claimed: rewardBreakdowns[i].claimed,
-            pending: rewardBreakdowns[i].pending,
-            merklCampaignId: rewardBreakdowns[i].campaignId,
-        };
-    }
-    return transformedBreakdowns;
-};
-// ─── Load ────────────────────────────────────────────────────────────────────
-const load = async (rewardBreakdowns, skipSubCampaignReasons = false) => {
-    const count = (await apiDbClient.rewardBreakdown.createMany({
-        data: rewardBreakdowns.map(breakdown => {
-            // Trying to parse subcampaign reasons only when skipSubCampaignReasons is false because priority is having the breakdowns in the DB
-            if (!skipSubCampaignReasons && breakdown.reason.includes("~")) {
-                // subcampaign reason will have the form <campaignId>_<protocol>_<reason> outputted by the processor>
-                const subcampaignReason = breakdown.reason.split("~")[breakdown.reason.split("~").length - 1];
-                breakdown.reason = subcampaignReason.split("_")[2];
-                breakdown.subCampaignId = subcampaignReason.split("_")[0];
-            }
-            return {
-                id: breakdown.id,
-                protocolId: breakdown.protocolId,
-                reason: breakdown.reason,
-                amount: breakdown.amount,
-                claimed: breakdown.claimed,
-                pending: breakdown.pending,
-                rewardId: breakdown.rewardId,
-                campaignId: breakdown.campaignId,
-                subCampaignId: breakdown.subCampaignId ?? undefined,
-            };
-        }),
-        skipDuplicates: true,
-    })).count;
-    log.info(`Successfully inserted ${rewardBreakdowns.length} reward breakdown(s).`);
-    return count;
-};
-// ─── Retry ───────────────────────────────────────────────────────────────────
-// Using binary search
-const retry = async (batches) => {
-    log.info(`Retrying ${batches.length} batch(es)...`);
-    let inserted = 0;
-    while (batches.length > 0) {
-        const currentBatch = batches.shift();
-        const firstHalf = currentBatch.splice(0, Math.ceil(currentBatch.length / 2));
-        const secondHalf = currentBatch;
-        // Process first half
-        try {
-            inserted += await load(firstHalf, true);
-        }
-        catch (err) {
-            log.error(JSON.stringify(firstHalf), err);
-            if (firstHalf.length > 1)
-                batches.push(firstHalf);
-            else
-                missingCampaigns.push(firstHalf[0].merklCampaignId);
-        }
-        // Process second half
-        try {
-            inserted += await load(secondHalf, true);
-        }
-        catch (err) {
-            log.error(JSON.stringify(secondHalf), err);
-            if (secondHalf.length > 1)
-                batches.push(secondHalf);
-            else
-                missingCampaigns.push(secondHalf[0].merklCampaignId);
-        }
-    }
-    return inserted;
-};
-// ─── Main ────────────────────────────────────────────────────────────────────
-export const main = async () => {
-    const start = moment().unix();
-    const count = await extract();
-    log.info(`✅ Successfully created ${count} new record(s) for ${process.env.CHAIN_ID}-${process.env.ROOT} in ${moment().unix() - start} sec.`);
-    if (failedBatches.length !== 0) {
-        const inserted = await retry(failedBatches);
-        log.info(`Successfully inserted ${inserted} record(s) after retrying.`);
-        if (inserted !== failedBatches.length) {
-            log.error("breakdowns", `${failedBatches.length} breakdown(s) failed to insert.`);
-            // ─── Push Missing Campaigns To Bucket ────────────────
-            const bucket = new BucketService(`merkl-campaigns-lake-${process.env.ENV}`, `merkl-data-${process.env.ENV}`);
-            await bucket.push(`${process.env.CHAIN_ID}_missing_campaigns.json`, JSON.stringify(missingCampaigns), {
-                type: "application/json",
-            });
-            process.exit(1);
-        }
-    }
-    // if (failedBatches.length === 0) await file.delete();
-    process.exit(0);
-};
-await main();

package/dist/src/jobs/rewards.js

@@ -1,112 +0,0 @@
-// ─── Rewards ETL ─────────────────────────────────────────────────────────────
-if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
-    throw new Error("[ENV]: missing variable");
-import { log } from "@/utils/logger";
-import { apiDbClient } from "@db";
-import { withRetry } from "@sdk";
-import { S3Client } from "bun";
-import moment from "moment";
-// ─── Global Variables ────────────────────────────────────────
-const gcsClient = new S3Client({
-    accessKeyId: process.env.GCS_ACCESS_ID,
-    secretAccessKey: process.env.GCS_SECRET,
-    endpoint: process.env.GCS_ENDPOINT,
-    bucket: `merkl-rewards-lake-${process.env.ENV}`,
-});
-let file = gcsClient.file(`rewards/${process.env.CHAIN_ID}-${process.env.ROOT}`);
-if (!(await file.exists()))
-    file = gcsClient.file(`rewards/${process.env.CHAIN_ID}-${process.env.ROOT}.gz`);
-if (!(await file.exists())) {
-    log.error("rewards", "File does not exist.");
-    process.exit(0);
-}
-const failedBatches = [];
-// ─── Extract ─────────────────────────────────────────────────────────────────
-const extract = async () => {
-    if (!file.exists())
-        throw new Error("File does not exist.");
-    const data = [];
-    const textDecoder = new TextDecoder();
-    let buffer = "";
-    const stream = file.stream();
-    let count = 0;
-    for await (const chunk of stream) {
-        buffer += textDecoder.decode(chunk);
-        const lines = buffer.split("\n");
-        buffer = lines.pop() || "";
-        for (const line of lines) {
-            if (line.trim())
-                data.push(transform(JSON.parse(line)));
-        }
-        try {
-            count += await withRetry(load, [data], 5, 60_000);
-            data.length = 0;
-        }
-        catch (err) {
-            console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
-            failedBatches.push(data);
-            data.length = 0;
-        }
-    }
-    return count;
-};
-// ─── Transform ───────────────────────────────────────────────────────────────
-const transform = (reward) => {
-    const rewardTokenId = Bun.hash(`${process.env.CHAIN_ID}${reward.rewardToken}`).toString();
-    const id = Bun.hash(`${process.env.ROOT}${reward.recipient}${rewardTokenId}`).toString();
-    return {
-        id,
-        root: reward.root,
-        amount: reward.amount,
-        pending: reward.pending,
-        claimed: reward.claimed,
-        recipient: reward.recipient,
-        rewardTokenId,
-        proofs: reward.proofs,
-    };
-};
-// ─── Load ────────────────────────────────────────────────────────────────────
-const load = async (rewards) => {
-    // ─── Load Users ──────────────────────────────────────────────────────
-    await apiDbClient.user.createMany({
-        data: rewards.map(r => {
-            return {
-                address: r.recipient,
-            };
-        }),
-        skipDuplicates: true,
-    });
-    // ─── Load Rewards ────────────────────────────────────────────────────
-    return (await apiDbClient.reward.createMany({
-        data: rewards,
-        skipDuplicates: true,
-    })).count;
-};
-// ─────────────────────────────────────────────────────────────────────────────
-export const main = async () => {
-    const start = moment().unix();
-    // ─── Create Merkle Root If Not Exists ────────────────────────────────
-    await apiDbClient.merklRoot.upsert({
-        create: {
-            root: process.env.ROOT,
-            chainId: +process.env.CHAIN_ID,
-            epoch: Math.floor(start / 3_600),
-            timestamp: start,
-        },
-        update: {},
-        where: {
-            root: process.env.ROOT,
-        },
-    });
-    // ─── Start Rewards ETL ───────────────────────────────────────────────
-    const count = await extract();
-    log.info(`✅ Successfully created ${count} new records for ${process.env.CHAIN_ID}-${process.env.ROOT} in ${moment().unix() - start} sec`);
-    if (failedBatches.length !== 0) {
-        log.error("rewards", `${failedBatches.length} batches failed.`);
-        process.exit(1);
-    }
-    if (failedBatches.length === 0) {
-        // await file.delete();
-    }
-};
-main();

package/dist/src/jobs/set-dungeon-keeper.js

@@ -1,90 +0,0 @@
-import { Client } from "@notionhq/client";
-import { Client as DiscordClient, GatewayIntentBits } from "discord.js";
-import moment from "moment";
-const notion = new Client({ auth: process.env.NOTION_TOKEN });
-const discord = new DiscordClient({ intents: [GatewayIntentBits.GuildMembers] });
-const NOTION_EMAIL_TO_DISCORD_ID = {
-    picodes: "thepicodes",
-    hugo: "ugolxt",
-    alex: "wombomango",
-    nileco: ".nileco",
-    thibaudb: ".greedythib",
-    gnervo: "gs8nrv",
-    vincent: "vince_merkl",
-    hicham: "lamicham_93854",
-    pveyrat: "sogipec",
-    baptiste: "baptistg",
-    clement: "clmntngl",
-    "hicham.bouanani": "hichrptg",
-};
-async function fetchEntriesForDateWithTag(databaseId, date, tag) {
-    const response = await notion.databases.query({
-        database_id: databaseId,
-        filter: {
-            and: [
-                {
-                    property: "Date",
-                    date: {
-                        equals: date,
-                    },
-                },
-                {
-                    property: "Database",
-                    select: {
-                        equals: tag,
-                    },
-                },
-            ],
-        },
-    });
-    return response.results;
-}
-async function assignRole(notionId) {
-    const discordId = NOTION_EMAIL_TO_DISCORD_ID[notionId];
-    if (!discordId)
-        throw "Discord ID not found.";
-    await discord.login(process.env.DISCORD_TOKEN);
-    const guild = await discord.guilds.cache.get("862708408711643136");
-    if (!guild)
-        throw "Guild not found.";
-    const roles = await guild.roles.fetch();
-    const teamRole = roles.find(r => r.name === "Team");
-    const dkRole = roles.find(r => r.name === "Dungeon Keeper");
-    if (!teamRole || !dkRole)
-        throw "Roles not found.";
-    const users = await guild.members.fetch();
-    for (const [_id, dkUser] of users.filter(member => member.roles.cache.has(dkRole.id))) {
-        await dkUser.roles.remove(dkRole);
-    }
-    const user = users
-        .filter(member => member.roles.cache.has(teamRole.id))
-        .find(member => member.user.tag === discordId);
-    if (!user)
-        throw "User not found";
-    await user.roles.add(dkRole);
-    const channel = await discord.channels.fetch("1328383110957633630");
-    if (!channel)
-        throw "Channel not found";
-    if (channel.isSendable()) {
-        await channel.send(`Lucky you <@${user.id}>! You're the <@&${dkRole.id}> today!`);
-    }
-}
-async function main() {
-    const entries = await fetchEntriesForDateWithTag("8546f6a84641406bafb2358762281e81", moment().format("YYYY-MM-DD"), "Dungeon Keeper");
-    const user = entries?.[0];
-    if (!user) {
-        throw "No dk found for today";
-    }
-    const notionId = (user?.properties["Qui ?"]).people[0].person.email
-        .replace("@angle.money", "")
-        .replace("@merkl.xyz", "");
-    await assignRole(notionId);
-}
-try {
-    await main();
-    process.exit(0);
-}
-catch (err) {
-    console.error(err);
-    process.exit(1);
-}