@merkl/api 0.10.416 → 0.10.417

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,7 @@ import { withRetry } from "@sdk";
7
7
  import { S3Client } from "bun";
8
8
  import moment from "moment";
9
9
  // ─── Constants ───────────────────────────────────────────────
10
- const MAX_BATCH_SIZE = 10_000;
10
+ const MAX_BATCH_SIZE = 1_000;
11
11
  // ─── Global Variables ────────────────────────────────────────
12
12
  const [chainIdString, root, campaignId] = process.env.FILENAME.replace(".gz", "").split("_");
13
13
  const chainId = Number.parseInt(chainIdString);
@@ -36,24 +36,32 @@ const extract = async () => {
36
36
  }
37
37
  if (data.length < MAX_BATCH_SIZE)
38
38
  continue;
39
- loadPromises.push(withRetry(load, [data], 5, 10_000));
39
+ const promise = withRetry(load, [data], 5, 10_000);
40
+ loadPromises.push(promise);
41
+ await promise;
40
42
  data = [];
41
43
  }
42
44
  // Final batch
43
- loadPromises.push(withRetry(load, [data], 5, 10_000));
44
- let count = 0;
45
+ const promise = withRetry(load, [data], 5, 10_000);
46
+ loadPromises.push(promise);
47
+ await promise;
48
+ let created = 0;
49
+ let updated = 0;
45
50
  let failed = 0;
46
51
  const promiseResults = await Promise.allSettled(loadPromises);
47
52
  for (const x of promiseResults) {
48
53
  if (x.status === "rejected")
49
54
  failed += 1;
50
- else
51
- count += x.value;
55
+ else {
56
+ updated += x.value.updated;
57
+ created += x.value.created;
58
+ }
52
59
  }
53
- return { count, failed, batch: promiseResults.length };
60
+ return { created, updated, failed, batch: promiseResults.length };
54
61
  };
55
62
  // ─── Transform & Load ────────────────────────────────────────────────────────
56
63
  const load = async (pendings) => {
64
+ log.info(`pushing ${pendings.length} pendings`);
57
65
  const { updated, created } = await updatePendings({
58
66
  distributionChainId: chainId,
59
67
  rewardToken: pendings[0].rewardToken,
@@ -61,7 +69,8 @@ const load = async (pendings) => {
61
69
  root,
62
70
  data: pendings,
63
71
  });
64
- return updated + created;
72
+ log.info(`✅ updated ${updated} and created ${created} pendings`);
73
+ return { updated, created };
65
74
  };
66
75
  const updatePendings = async (data) => {
67
76
  const rewardTokenId = Bun.hash(`${data.distributionChainId}${data.rewardToken}`).toString();
@@ -100,7 +109,9 @@ const updatePendings = async (data) => {
100
109
  for (const [pointIndex, point] of data.data.entries()) {
101
110
  updateRewardUniques(point.recipient, point.pending, breakdownExists[pointIndex]?.pending);
102
111
  if (!!breakdownExists[pointIndex]) {
103
- breakdownToUpdate.push(point);
112
+ if (point.pending !== breakdownExists[pointIndex].pending) {
113
+ breakdownToUpdate.push(point);
114
+ }
104
115
  }
105
116
  else {
106
117
  breakdownToCreate.push(point);
@@ -118,72 +129,82 @@ const updatePendings = async (data) => {
118
129
  })));
119
130
  for (const [pointIndex, point] of Object.values(rewardUniques).entries()) {
120
131
  if (!!exists[pointIndex]) {
121
- toUpdate.push({
122
- ...point,
123
- pending: (BigInt(exists[pointIndex].pending) + BigInt(point.pending)).toString(), // Store the delta
124
- });
132
+ if (point.pending !== "0") {
133
+ toUpdate.push({
134
+ ...point,
135
+ pending: (BigInt(exists[pointIndex].pending) + BigInt(point.pending)).toString(), // Store the delta
136
+ });
137
+ }
125
138
  }
126
139
  else {
127
140
  toCreate.push(point);
128
141
  }
129
142
  }
130
- await apiDbClient.$transaction(toUpdate.map(x => {
131
- const rewardId = Bun.hash(`${data.root}${x.recipient}${rewardTokenId}`).toString();
132
- return apiDbClient.reward.update({
133
- where: {
134
- id: rewardId,
135
- },
136
- data: {
137
- pending: x.pending,
138
- },
143
+ console.log(`rewards to create: ${toCreate.length}, to update: ${toUpdate.length}`);
144
+ if (toCreate.length > 0) {
145
+ const users = toCreate.map(x => x.recipient);
146
+ await apiDbClient.user.createMany({ data: users.map(x => ({ address: x, tags: [] })), skipDuplicates: true });
147
+ await apiDbClient.reward.createMany({
148
+ data: toCreate.map(x => {
149
+ const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
150
+ return {
151
+ id: rewardId,
152
+ root,
153
+ recipient: x.recipient,
154
+ rewardTokenId,
155
+ proofs: [],
156
+ amount: "0",
157
+ pending: x.pending,
158
+ claimed: "0",
159
+ };
160
+ }),
139
161
  });
140
- }));
141
- let users = toCreate.map(x => x.recipient);
142
- await apiDbClient.user.createMany({ data: users.map(x => ({ address: x, tags: [] })), skipDuplicates: true });
143
- await apiDbClient.reward.createMany({
144
- data: toCreate.map(x => {
145
- const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
146
- return {
147
- id: rewardId,
148
- root,
149
- recipient: x.recipient,
150
- rewardTokenId,
151
- proofs: [],
152
- amount: "0",
153
- pending: x.pending,
154
- claimed: "0",
155
- };
156
- }),
157
- });
158
- await apiDbClient.$transaction(breakdownToUpdate.map(x => {
159
- return apiDbClient.rewardBreakdown.update({
160
- where: {
161
- rewardId_campaignId_reason: {
162
- rewardId: Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString(),
163
- campaignId: campaignId,
164
- reason: x.reason,
162
+ }
163
+ if (toUpdate.length > 0) {
164
+ await apiDbClient.$transaction(toUpdate.map(x => {
165
+ const rewardId = Bun.hash(`${data.root}${x.recipient}${rewardTokenId}`).toString();
166
+ return apiDbClient.reward.update({
167
+ where: {
168
+ id: rewardId,
165
169
  },
166
- },
167
- data: {
168
- pending: x.pending,
169
- },
170
+ data: {
171
+ pending: x.pending,
172
+ },
173
+ });
174
+ }));
175
+ }
176
+ console.log(`breakdowns to create: ${breakdownToCreate.length}, to update: ${breakdownToUpdate.length}`);
177
+ if (breakdownToUpdate.length > 0) {
178
+ await apiDbClient.$transaction(breakdownToUpdate.map(x => {
179
+ return apiDbClient.rewardBreakdown.update({
180
+ where: {
181
+ rewardId_campaignId_reason: {
182
+ rewardId: Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString(),
183
+ campaignId: campaignId,
184
+ reason: x.reason,
185
+ },
186
+ },
187
+ data: {
188
+ pending: x.pending,
189
+ },
190
+ });
191
+ }));
192
+ }
193
+ if (breakdownToCreate.length > 0) {
194
+ await apiDbClient.rewardBreakdown.createMany({
195
+ data: breakdownToCreate.map(x => {
196
+ const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
197
+ return {
198
+ reason: x.reason,
199
+ amount: "0",
200
+ pending: x.pending,
201
+ claimed: "0",
202
+ rewardId,
203
+ campaignId,
204
+ };
205
+ }),
170
206
  });
171
- }));
172
- users = toCreate.map(x => x.recipient);
173
- await apiDbClient.user.createMany({ data: users.map(x => ({ address: x, tags: [] })), skipDuplicates: true });
174
- await apiDbClient.rewardBreakdown.createMany({
175
- data: breakdownToCreate.map(x => {
176
- const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
177
- return {
178
- reason: x.reason,
179
- amount: "0",
180
- pending: x.pending,
181
- claimed: "0",
182
- rewardId,
183
- campaignId,
184
- };
185
- }),
186
- });
207
+ }
187
208
  return { created: breakdownToCreate.length, updated: breakdownToUpdate.length };
188
209
  };
189
210
  // ─────────────────────────────────────────────────────────────────────────────
@@ -191,8 +212,8 @@ export const main = async () => {
191
212
  log.info(`✅ Running for ${process.env.FILENAME}`);
192
213
  const start = moment().unix();
193
214
  // ─── Start Rewards ETL ───────────────────────────────────────────────
194
- const { count, failed, batch } = await extract();
195
- log.info(`✅ Successfully created ${count} new records in ${batch} batches for ${process.env.FILENAME} in ${moment().unix() - start} sec`);
215
+ const { created, updated, failed, batch } = await extract();
216
+ log.info(`✅ Successfully created ${created} and updated ${updated} pendings records in ${batch} batches for ${process.env.FILENAME} in ${moment().unix() - start} sec`);
196
217
  if (failed !== 0) {
197
218
  log.error("pendings", `${failed} batches failed.`);
198
219
  process.exit(1);