@merkl/api 0.15.41 → 0.15.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1753,9 +1753,12 @@ declare const eden: {
1753
1753
  fetch?: RequestInit | undefined;
1754
1754
  }) => Promise<import("@elysiajs/eden").Treaty.TreatyResponse<{
1755
1755
  200: {
1756
- name: string;
1757
- id: number;
1758
- icon: string;
1756
+ Explorer: {
1757
+ type: import("../../database/api/.generated").$Enums.ExplorerType;
1758
+ url: string;
1759
+ id: string;
1760
+ chainId: number;
1761
+ }[];
1759
1762
  };
1760
1763
  }>>;
1761
1764
  };
@@ -4872,9 +4875,12 @@ declare const eden: {
4872
4875
  fetch?: RequestInit | undefined;
4873
4876
  }) => Promise<import("@elysiajs/eden").Treaty.TreatyResponse<{
4874
4877
  200: {
4875
- name: string;
4876
- id: number;
4877
- icon: string;
4878
+ Explorer: {
4879
+ type: import("../../database/api/.generated").$Enums.ExplorerType;
4880
+ url: string;
4881
+ id: string;
4882
+ chainId: number;
4883
+ }[];
4878
4884
  };
4879
4885
  }>>;
4880
4886
  };
@@ -9062,9 +9068,12 @@ export declare const MerklApi: (domain: string | import("elysia").default<"", fa
9062
9068
  };
9063
9069
  response: {
9064
9070
  200: {
9065
- name: string;
9066
- id: number;
9067
- icon: string;
9071
+ Explorer: {
9072
+ type: import("../../database/api/.generated").$Enums.ExplorerType;
9073
+ url: string;
9074
+ id: string;
9075
+ chainId: number;
9076
+ }[];
9068
9077
  };
9069
9078
  };
9070
9079
  };
@@ -13466,9 +13475,12 @@ export declare const MerklApi: (domain: string | import("elysia").default<"", fa
13466
13475
  fetch?: RequestInit | undefined;
13467
13476
  }) => Promise<import("@elysiajs/eden").Treaty.TreatyResponse<{
13468
13477
  200: {
13469
- name: string;
13470
- id: number;
13471
- icon: string;
13478
+ Explorer: {
13479
+ type: import("../../database/api/.generated").$Enums.ExplorerType;
13480
+ url: string;
13481
+ id: string;
13482
+ chainId: number;
13483
+ }[];
13472
13484
  };
13473
13485
  }>>;
13474
13486
  };
@@ -16585,9 +16597,12 @@ export declare const MerklApi: (domain: string | import("elysia").default<"", fa
16585
16597
  fetch?: RequestInit | undefined;
16586
16598
  }) => Promise<import("@elysiajs/eden").Treaty.TreatyResponse<{
16587
16599
  200: {
16588
- name: string;
16589
- id: number;
16590
- icon: string;
16600
+ Explorer: {
16601
+ type: import("../../database/api/.generated").$Enums.ExplorerType;
16602
+ url: string;
16603
+ id: string;
16604
+ chainId: number;
16605
+ }[];
16591
16606
  };
16592
16607
  }>>;
16593
16608
  };
@@ -2126,9 +2126,12 @@ declare const app: Elysia<"", false, {
2126
2126
  };
2127
2127
  response: {
2128
2128
  200: {
2129
- name: string;
2130
- id: number;
2131
- icon: string;
2129
+ Explorer: {
2130
+ type: import("../database/api/.generated").$Enums.ExplorerType;
2131
+ url: string;
2132
+ id: string;
2133
+ chainId: number;
2134
+ }[];
2132
2135
  };
2133
2136
  };
2134
2137
  };
@@ -1,9 +1,9 @@
1
1
  // ─── Reward Breakdowns ETL ───────────────────────────────────────────────────
2
- if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
2
+ if (!process.env.DATABASE_API_URL || !process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
3
3
  throw new Error("[ENV]: missing variable");
4
+ import { BucketService } from "../../modules/v4/bucket/bucket.service";
4
5
  import { log } from "../../utils/logger";
5
6
  import { apiDbClient } from "../../utils/prisma";
6
- import { withRetry } from "@sdk";
7
7
  import { S3Client } from "bun";
8
8
  import moment from "moment";
9
9
  // ─── Global Variables ────────────────────────────────────────
@@ -13,10 +13,9 @@ const gcsClient = new S3Client({
13
13
  endpoint: process.env.GCS_ENDPOINT,
14
14
  bucket: `merkl-rewards-lake-${process.env.ENV}`,
15
15
  });
16
- let file = gcsClient.file(`breakdowns/${process.env.CHAIN_ID}-${process.env.ROOT}`);
17
- if (!(await file.exists()))
18
- file = gcsClient.file(`breakdowns/${process.env.CHAIN_ID}-${process.env.ROOT}.gz`);
16
+ const file = gcsClient.file(`breakdowns/${process.env.CHAIN_ID}-${process.env.ROOT}`);
19
17
  const failedBatches = [];
18
+ const missingCampaigns = [];
20
19
  // ─── Extract ─────────────────────────────────────────────────────────────────
21
20
  const extract = async () => {
22
21
  if (!file.exists())
@@ -30,106 +29,115 @@ const extract = async () => {
30
29
  buffer += textDecoder.decode(chunk);
31
30
  const lines = buffer.split("\n");
32
31
  buffer = lines.pop() || "";
33
- for (const line of lines)
34
- if (line.trim()) {
32
+ for (const line of lines) {
33
+ if (line.trim())
35
34
  data.push(...(await transform(JSON.parse(line))));
36
- }
35
+ }
37
36
  if (data.length < 30_000)
38
37
  continue;
39
38
  try {
40
- count += await withRetry(load, [data], 5, 60_000);
39
+ count += await load(data);
41
40
  data = [];
42
41
  }
43
42
  catch (err) {
44
- console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
43
+ // log.error("Failed to insert a batch, adding it to the fail queue.", err);
45
44
  failedBatches.push(data);
46
45
  data = [];
47
46
  }
48
47
  }
49
48
  try {
50
- count += await withRetry(load, [data], 5, 60_000);
49
+ count += await load(data);
51
50
  data = [];
52
51
  }
53
52
  catch (err) {
54
- console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
53
+ // log.error("Failed to insert a batch, adding it to the fail queue.", err);
55
54
  failedBatches.push(data);
56
55
  data = [];
57
56
  }
58
57
  return count;
59
58
  };
60
59
  // ─── Transform ───────────────────────────────────────────────────────────────
61
- const transform = async (rewardBreakdowns) => {
62
- const transformedRewardBreakdowns = [];
63
- for (const rewardBreakdown of rewardBreakdowns) {
64
- const rewardTokenId = Bun.hash(`${process.env.CHAIN_ID}${rewardBreakdown.token}`).toString();
65
- const rewardId = Bun.hash(`${process.env.ROOT}${rewardBreakdown.recipient}${rewardTokenId}`).toString();
66
- transformedRewardBreakdowns.push({
60
+ const transform = (rewardBreakdowns) => {
61
+ const transformedBreakdowns = Array(rewardBreakdowns.length);
62
+ let i = 0;
63
+ for (i; i < rewardBreakdowns.length; i++) {
64
+ const rewardTokenId = Bun.hash(`${process.env.CHAIN_ID}${rewardBreakdowns[i].token}`).toString();
65
+ const rewardId = Bun.hash(`${process.env.ROOT}${rewardBreakdowns[i].recipient}${rewardTokenId}`).toString();
66
+ const campaignId = Bun.hash(`${process.env.CHAIN_ID}${rewardBreakdowns[i].campaignId}`).toString();
67
+ transformedBreakdowns[i] = {
67
68
  rewardId,
68
- protocolId: rewardBreakdown.protocolId ? rewardBreakdown.protocolId : undefined,
69
- campaignId: rewardBreakdown.campaignId,
70
- reason: rewardBreakdown.reason ? rewardBreakdown.reason : "",
71
- amount: rewardBreakdown.amount,
72
- claimed: rewardBreakdown.claimed,
73
- pending: rewardBreakdown.pending,
74
- });
69
+ protocolId: rewardBreakdowns[i].protocolId ? rewardBreakdowns[i].protocolId : undefined,
70
+ campaignId,
71
+ reason: rewardBreakdowns[i].reason ? rewardBreakdowns[i].reason : "",
72
+ amount: rewardBreakdowns[i].amount,
73
+ claimed: rewardBreakdowns[i].claimed,
74
+ pending: rewardBreakdowns[i].pending,
75
+ merklCampaignId: rewardBreakdowns[i].campaignId,
76
+ };
75
77
  }
76
- return transformedRewardBreakdowns;
78
+ return transformedBreakdowns;
77
79
  };
78
80
  // ─── Load ────────────────────────────────────────────────────────────────────
79
81
  const load = async (rewardBreakdowns) => {
80
- log.info(`pushing ${rewardBreakdowns.length} rewardBreakdowns`);
81
- // const missingCampaigns: string[] = [];
82
- // const foundCampaigns: string[] = [];
83
- // const campaigns = rewardBreakdowns.reduce(
84
- // (acc, x) => {
85
- // const campaignUnique = {
86
- // campaignId: x.campaignId,
87
- // distributionChain: Number.parseInt(process.env.CHAIN_ID!),
88
- // };
89
- // if (!Object.keys(acc).includes(CampaignService.hashId(campaignUnique)))
90
- // acc[CampaignService.hashId(campaignUnique)] = {
91
- // campaignId: x.campaignId,
92
- // distributionChain: Number.parseInt(process.env.CHAIN_ID!),
93
- // };
94
- // return acc;
95
- // },
96
- // {} as Record<string, CampaignUnique>
97
- // );
98
- // for (const campaign of Object.values(campaigns)) {
99
- // const campaignExists = await CampaignService.checkIfExist(campaign);
100
- // if (!campaignExists) {
101
- // const { success, fail } = await CampaignService.fill([campaign]);
102
- // if (fail === 1 || success !== 1) {
103
- // missingCampaigns.push(CampaignService.hashId(campaign));
104
- // log.warn(`createManyBreakdown - Missing campaign: ${campaign.campaignId}`);
105
- // continue;
106
- // }
107
- // }
108
- // foundCampaigns.push(CampaignService.hashId(campaign));
109
- // }
110
- // if (missingCampaigns.length !== 0) {
111
- // log.warn(`createManyBreakdown - Missing ${missingCampaigns.length} campaigns: ${missingCampaigns.join(", ")}`);
112
- // }
113
- return (await apiDbClient.rewardBreakdown.createMany({
114
- data: rewardBreakdowns.map(x => {
115
- const campaignId = Bun.hash(`${process.env.CHAIN_ID}${x.campaignId}`).toString();
116
- return { ...x, campaignId };
117
- }),
118
- // .filter(x => !missingCampaigns.includes(x.campaignId)),
82
+ const count = (await apiDbClient.rewardBreakdown.createMany({
83
+ data: rewardBreakdowns,
119
84
  skipDuplicates: true,
120
85
  })).count;
86
+ log.info(`Successfully inserted ${rewardBreakdowns.length} reward breakdown(s).`);
87
+ return count;
88
+ };
89
+ // ─── Retry ───────────────────────────────────────────────────────────────────
90
+ // Using binary search
91
+ const retry = async (batches) => {
92
+ log.info(`Retrying ${batches.length} batch(es)...`);
93
+ let inserted = 0;
94
+ while (batches.length > 0) {
95
+ const currentBatch = batches.shift();
96
+ const firstHalf = currentBatch.splice(0, Math.ceil(currentBatch.length / 2));
97
+ const secondHalf = currentBatch;
98
+ // Process first half
99
+ try {
100
+ inserted += await load(firstHalf);
101
+ }
102
+ catch (err) {
103
+ log.error(JSON.stringify(firstHalf), err);
104
+ if (firstHalf.length > 1)
105
+ batches.push(firstHalf);
106
+ else
107
+ missingCampaigns.push(firstHalf[0].merklCampaignId);
108
+ }
109
+ // Process second half
110
+ try {
111
+ inserted += await load(secondHalf);
112
+ }
113
+ catch (err) {
114
+ log.error(JSON.stringify(secondHalf), err);
115
+ if (secondHalf.length > 1)
116
+ batches.push(secondHalf);
117
+ else
118
+ missingCampaigns.push(secondHalf[0].merklCampaignId);
119
+ }
120
+ }
121
+ return inserted;
121
122
  };
122
123
  // ─── Main ────────────────────────────────────────────────────────────────────
123
124
  export const main = async () => {
124
125
  const start = moment().unix();
125
126
  const count = await extract();
126
- log.info(`✅ Successfully created ${count} new records for ${process.env.CHAIN_ID}-${process.env.ROOT} in ${moment().unix() - start} sec`);
127
+ log.info(`✅ Successfully created ${count} new record(s) for ${process.env.CHAIN_ID}-${process.env.ROOT} in ${moment().unix() - start} sec.`);
127
128
  if (failedBatches.length !== 0) {
128
- log.error("breakdowns", `${failedBatches.length} batches failed.`);
129
- process.exit(1);
130
- }
131
- if (failedBatches.length === 0) {
132
- // await file.delete();
129
+ const inserted = await retry(failedBatches);
130
+ log.info(`Successfully inserted ${inserted} record(s) after retrying.`);
131
+ if (inserted !== failedBatches.length) {
132
+ log.error("breakdowns", `${failedBatches.length} breakdown(s) failed to insert.`);
133
+ // ─── Push Missing Campaigns To Bucket ────────────────
134
+ const bucket = new BucketService(`merkl-campaigns-lake-${process.env.ENV}`, `merkl-data-${process.env.ENV}`);
135
+ await bucket.push(`${process.env.CHAIN_ID}_missing_campaigns`, JSON.stringify(missingCampaigns), {
136
+ type: "application/json",
137
+ });
138
+ process.exit(1);
139
+ }
133
140
  }
141
+ // if (failedBatches.length === 0) await file.delete();
134
142
  };
135
143
  main();
@@ -4,7 +4,7 @@ export declare class BucketService {
4
4
  defaultUploadOptions: UploadOptions<unknown>;
5
5
  constructor(bucket: string, projectId: string);
6
6
  pushArray<T>(filename: string, arr: T[], options?: UploadOptions<T>): Promise<number>;
7
- push(filename: string, text: string, options?: Omit<UploadOptions<never>, "transform">): Promise<number>;
7
+ push(filename: string, text: string, options?: Omit<UploadOptions<never>, "transform" | "separator">): Promise<number>;
8
8
  pullArray<T>(filename: string, callback: (elem: string) => T, separator?: string): Promise<T[]>;
9
9
  pull(filename: string): Promise<string>;
10
10
  }
@@ -121,9 +121,12 @@ export declare const ChainController: Elysia<"/chains", false, {
121
121
  };
122
122
  response: {
123
123
  200: {
124
- name: string;
125
- id: number;
126
- icon: string;
124
+ Explorer: {
125
+ type: import("../../../../database/api/.generated").$Enums.ExplorerType;
126
+ url: string;
127
+ id: string;
128
+ chainId: number;
129
+ }[];
127
130
  };
128
131
  };
129
132
  };
@@ -46,15 +46,26 @@ export declare abstract class ChainRepository {
46
46
  * @returns
47
47
  */
48
48
  static create(data: CreateChainModel): Promise<{
49
- name: string;
50
- id: number;
51
- icon: string;
49
+ Explorer: {
50
+ type: import("../../../../database/api/.generated").$Enums.ExplorerType;
51
+ url: string;
52
+ id: string;
53
+ chainId: number;
54
+ }[];
52
55
  }>;
53
56
  static findMany(): Promise<{
54
57
  name: string;
55
58
  id: number;
56
59
  icon: string;
57
60
  }[]>;
61
+ static findUniqueOrThrow(id: number): Promise<{
62
+ Explorer: {
63
+ type: import("../../../../database/api/.generated").$Enums.ExplorerType;
64
+ url: string;
65
+ id: string;
66
+ chainId: number;
67
+ }[];
68
+ }>;
58
69
  /**
59
70
  * Fetches the campaign dynamic data for a v3 campaign onchain
60
71
  * @param chainId
@@ -46,19 +46,29 @@ export class ChainRepository {
46
46
  * @returns
47
47
  */
48
48
  static async create(data) {
49
- const explorerId = await ExplorerRepository.create(data.id, data.explorerType, data.explorerUrl);
50
- return apiDbClient.chain.create({
49
+ await apiDbClient.chain.create({
51
50
  data: {
52
51
  id: data.id,
53
52
  icon: data.icon,
54
53
  name: data.name,
55
- Explorer: { connect: { id: explorerId.id } },
56
54
  },
57
55
  });
56
+ await ExplorerRepository.create(data.id, data.explorerType, data.explorerUrl);
57
+ return await ChainRepository.findUniqueOrThrow(data.id);
58
58
  }
59
59
  static async findMany() {
60
60
  return apiDbClient.chain.findMany();
61
61
  }
62
+ static async findUniqueOrThrow(id) {
63
+ return apiDbClient.chain.findUniqueOrThrow({
64
+ where: {
65
+ id,
66
+ },
67
+ select: {
68
+ Explorer: true,
69
+ },
70
+ });
71
+ }
62
72
  /**
63
73
  * Fetches the campaign dynamic data for a v3 campaign onchain
64
74
  * @param chainId
@@ -40,9 +40,12 @@ export declare abstract class ChainService {
40
40
  */
41
41
  static getSupportedIds(): Promise<number[]>;
42
42
  static create(chain: CreateChainModel): Promise<{
43
- name: string;
44
- id: number;
45
- icon: string;
43
+ Explorer: {
44
+ type: import("../../../../database/api/.generated").$Enums.ExplorerType;
45
+ url: string;
46
+ id: string;
47
+ chainId: number;
48
+ }[];
46
49
  }>;
47
50
  static update(id: number, data: UpdateChainModel): Promise<{
48
51
  name: string;
@@ -25,14 +25,46 @@ export declare abstract class ProtocolRepository {
25
25
  tags: string[];
26
26
  icon: string;
27
27
  }[]>;
28
- static findMany(query: GetProtocolsQueryModel): Promise<{
28
+ static findMany(query: GetProtocolsQueryModel): Promise<({
29
+ MainOpportunities: ({
30
+ Campaigns: {
31
+ type: import("../../../../database/api/.generated").$Enums.CampaignType;
32
+ id: string;
33
+ params: import("database/api/.generated/runtime/library").JsonValue;
34
+ subType: number | null;
35
+ startTimestamp: bigint;
36
+ endTimestamp: bigint;
37
+ computeChainId: number;
38
+ distributionChainId: number;
39
+ campaignId: string;
40
+ rewardTokenId: string;
41
+ amount: string;
42
+ opportunityId: string;
43
+ creatorAddress: string;
44
+ }[];
45
+ } & {
46
+ name: string;
47
+ type: import("../../../../database/api/.generated").$Enums.CampaignType;
48
+ id: string;
49
+ status: import("../../../../database/api/.generated").$Enums.Status;
50
+ tags: string[];
51
+ identifier: string;
52
+ chainId: number;
53
+ action: import("../../../../database/api/.generated").$Enums.OpportunityAction;
54
+ depositUrl: string | null;
55
+ mainProtocolId: string | null;
56
+ tvl: number;
57
+ apr: number;
58
+ dailyRewards: number;
59
+ })[];
60
+ } & {
29
61
  name: string;
30
62
  url: string;
31
63
  description: string;
32
64
  id: string;
33
65
  tags: string[];
34
66
  icon: string;
35
- }[]>;
67
+ })[]>;
36
68
  static countMany(query: GetProtocolsQueryModel): Promise<number>;
37
69
  static update(id: string, data: UpdateProtocolModel): Promise<{
38
70
  name: string;
@@ -52,6 +52,19 @@ export class ProtocolRepository {
52
52
  take: items,
53
53
  skip: page * items,
54
54
  ...args,
55
+ include: {
56
+ MainOpportunities: {
57
+ include: {
58
+ Campaigns: {
59
+ where: {
60
+ endTimestamp: {
61
+ gt: BigInt(Math.floor(Date.now() / 1000)),
62
+ },
63
+ },
64
+ },
65
+ },
66
+ },
67
+ },
55
68
  });
56
69
  }
57
70
  static async countMany(query) {
@@ -39,7 +39,13 @@ export class ProtocolService {
39
39
  }[amm];
40
40
  }
41
41
  static async findMany(query) {
42
- return ProtocolRepository.findMany(query);
42
+ const protocols = await ProtocolRepository.findMany(query);
43
+ const enrichedProtocols = protocols.map(({ MainOpportunities, ...protocol }) => ({
44
+ ...protocol,
45
+ dailyReward: MainOpportunities.reduce((sum, opportunity) => sum + opportunity.dailyRewards, 0),
46
+ numberOfCampaigns: MainOpportunities.reduce((sum, opportunity) => sum + opportunity.Campaigns.length, 0),
47
+ }));
48
+ return enrichedProtocols;
43
49
  }
44
50
  static async countMany(query) {
45
51
  return ProtocolRepository.countMany(query);
@@ -1996,9 +1996,12 @@ export declare const v4: Elysia<"/v4", false, {
1996
1996
  };
1997
1997
  response: {
1998
1998
  200: {
1999
- name: string;
2000
- id: number;
2001
- icon: string;
1999
+ Explorer: {
2000
+ type: import("../../../database/api/.generated").$Enums.ExplorerType;
2001
+ url: string;
2002
+ id: string;
2003
+ chainId: number;
2004
+ }[];
2002
2005
  };
2003
2006
  };
2004
2007
  };