@merkl/api 0.10.411 → 0.10.413

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/src/jobs/etl/pendings.js +34 -24
  2. package/dist/src/jobs/etl/reward-breakdowns.js +3 -3
  3. package/dist/src/jobs/etl/rewards.js +2 -1
  4. package/dist/tsconfig.package.tsbuildinfo +1 -1
  5. package/package.json +1 -1
package/dist/src/jobs/etl/pendings.js CHANGED
@@ -5,8 +5,10 @@ import { log } from "../../utils/logger";
5
5
  import { apiDbClient } from "../../utils/prisma";
6
6
  import { S3Client } from "bun";
7
7
  import moment from "moment";
8
+ // ─── Constants ───────────────────────────────────────────────
9
+ const MAX_BATCH_SIZE = 10_000;
8
10
  // ─── Global Variables ────────────────────────────────────────
9
- const [chainIdString, root, campaignId] = process.env.FILENAME.split("_");
11
+ const [chainIdString, root, campaignId] = process.env.FILENAME.replace(".gz", "").split("_");
10
12
  const chainId = Number.parseInt(chainIdString);
11
13
  const gcsClient = new S3Client({
12
14
  accessKeyId: process.env.GCS_ACCESS_ID,
@@ -32,6 +34,8 @@ const extract = async () => {
32
34
  for (const line of lines) {
33
35
  data.push(JSON.parse(line));
34
36
  }
37
+ if (data.length < MAX_BATCH_SIZE)
38
+ continue;
35
39
  try {
36
40
  count += await load(data);
37
41
  }
@@ -41,6 +45,16 @@ const extract = async () => {
41
45
  data.length = 0;
42
46
  }
43
47
  }
48
+ if (data.length === 0)
49
+ return count;
50
+ try {
51
+ count += await load(data);
52
+ }
53
+ catch (err) {
54
+ console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
55
+ failedBatches.push(data);
56
+ data.length = 0;
57
+ }
44
58
  return count;
45
59
  };
46
60
  // ─── Transform & Load ────────────────────────────────────────────────────────
@@ -131,21 +145,21 @@ const updatePendings = async (data) => {
131
145
  }));
132
146
  let users = toCreate.map(x => x.recipient);
133
147
  await apiDbClient.user.createMany({ data: users.map(x => ({ address: x, tags: [] })), skipDuplicates: true });
134
- await apiDbClient.$transaction(toCreate.map(x => {
135
- const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
136
- return apiDbClient.reward.create({
137
- data: {
148
+ await apiDbClient.reward.createMany({
149
+ data: toCreate.map(x => {
150
+ const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
151
+ return {
138
152
  id: rewardId,
139
- MerklRoot: { connect: { root } },
140
- User: { connect: { address: x.recipient } },
141
- RewardToken: { connect: { id: rewardTokenId } },
153
+ root,
154
+ recipient: x.recipient,
155
+ rewardTokenId,
142
156
  proofs: [],
143
157
  amount: "0",
144
158
  pending: x.pending,
145
159
  claimed: "0",
146
- },
147
- });
148
- }));
160
+ };
161
+ }),
162
+ });
149
163
  await apiDbClient.$transaction(breakdownToUpdate.map(x => {
150
164
  return apiDbClient.rewardBreakdown.update({
151
165
  where: {
@@ -162,23 +176,19 @@ const updatePendings = async (data) => {
162
176
  }));
163
177
  users = toCreate.map(x => x.recipient);
164
178
  await apiDbClient.user.createMany({ data: users.map(x => ({ address: x, tags: [] })), skipDuplicates: true });
165
- await apiDbClient.$transaction(breakdownToCreate.map(x => {
166
- const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
167
- return apiDbClient.rewardBreakdown.create({
168
- data: {
179
+ await apiDbClient.rewardBreakdown.createMany({
180
+ data: breakdownToCreate.map(x => {
181
+ const rewardId = Bun.hash(`${root}${x.recipient}${rewardTokenId}`).toString();
182
+ return {
169
183
  reason: x.reason,
170
184
  amount: "0",
171
185
  pending: x.pending,
172
186
  claimed: "0",
173
- Reward: {
174
- connect: {
175
- id: rewardId,
176
- },
177
- },
178
- Campaign: { connect: { id: campaignId } },
179
- },
180
- });
181
- }));
187
+ rewardId,
188
+ campaignId,
189
+ };
190
+ }),
191
+ });
182
192
  return { created: breakdownToCreate.length, updated: breakdownToUpdate.length };
183
193
  };
184
194
  // ─────────────────────────────────────────────────────────────────────────────
package/dist/src/jobs/etl/reward-breakdowns.js CHANGED
@@ -3,6 +3,7 @@ if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
3
3
  throw new Error("[ENV]: missing variable");
4
4
  import { log } from "../../utils/logger";
5
5
  import { apiDbClient } from "../../utils/prisma";
6
+ import { withRetry } from "@sdk";
6
7
  import { S3Client } from "bun";
7
8
  import moment from "moment";
8
9
  // ─── Global Variables ────────────────────────────────────────
@@ -32,7 +33,7 @@ const extract = async () => {
32
33
  data.push(...transform(JSON.parse(line)));
33
34
  }
34
35
  try {
35
- count += await load(data);
36
+ count += await withRetry(load, [data], 5, 60_000);
36
37
  data.length = 0;
37
38
  }
38
39
  catch (err) {
@@ -79,8 +80,7 @@ export const main = async () => {
79
80
  process.exit(1);
80
81
  }
81
82
  if (failedBatches.length === 0) {
82
- // await file.delete();
83
- // log.info("Object deleted");
83
+ await file.delete();
84
84
  }
85
85
  };
86
86
  main();
package/dist/src/jobs/etl/rewards.js CHANGED
@@ -3,6 +3,7 @@ if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
3
3
  throw new Error("[ENV]: missing variable");
4
4
  import { log } from "../../utils/logger";
5
5
  import { apiDbClient } from "../../utils/prisma";
6
+ import { withRetry } from "@sdk";
6
7
  import { S3Client } from "bun";
7
8
  import moment from "moment";
8
9
  // ─── Global Variables ────────────────────────────────────────
@@ -32,7 +33,7 @@ const extract = async () => {
32
33
  data.push(transform(JSON.parse(line)));
33
34
  }
34
35
  try {
35
- count += await load(data);
36
+ count += await withRetry(load, [data], 5, 60_000);
36
37
  data.length = 0;
37
38
  }
38
39
  catch (err) {