@merkl/api 0.10.412 → 0.10.413

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/src/jobs/etl/pendings.js +14 -0
  2. package/dist/src/jobs/etl/reward-breakdowns.js +3 -3
  3. package/dist/src/jobs/etl/rewards.js +2 -1
  4. package/dist/tsconfig.package.tsbuildinfo +1 -1
  5. package/package.json +1 -1
package/dist/src/jobs/etl/pendings.js CHANGED
@@ -5,6 +5,8 @@ import { log } from "../../utils/logger";
5
5
  import { apiDbClient } from "../../utils/prisma";
6
6
  import { S3Client } from "bun";
7
7
  import moment from "moment";
8
+ // ─── Constants ───────────────────────────────────────────────
9
+ const MAX_BATCH_SIZE = 10_000;
8
10
  // ─── Global Variables ────────────────────────────────────────
9
11
  const [chainIdString, root, campaignId] = process.env.FILENAME.replace(".gz", "").split("_");
10
12
  const chainId = Number.parseInt(chainIdString);
@@ -32,6 +34,8 @@ const extract = async () => {
32
34
  for (const line of lines) {
33
35
  data.push(JSON.parse(line));
34
36
  }
37
+ if (data.length < MAX_BATCH_SIZE)
38
+ continue;
35
39
  try {
36
40
  count += await load(data);
37
41
  }
@@ -41,6 +45,16 @@ const extract = async () => {
41
45
  data.length = 0;
42
46
  }
43
47
  }
48
+ if (data.length === 0)
49
+ return count;
50
+ try {
51
+ count += await load(data);
52
+ }
53
+ catch (err) {
54
+ console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
55
+ failedBatches.push(data);
56
+ data.length = 0;
57
+ }
44
58
  return count;
45
59
  };
46
60
  // ─── Transform & Load ────────────────────────────────────────────────────────
package/dist/src/jobs/etl/reward-breakdowns.js CHANGED
@@ -3,6 +3,7 @@ if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
3
3
  throw new Error("[ENV]: missing variable");
4
4
  import { log } from "../../utils/logger";
5
5
  import { apiDbClient } from "../../utils/prisma";
6
+ import { withRetry } from "@sdk";
6
7
  import { S3Client } from "bun";
7
8
  import moment from "moment";
8
9
  // ─── Global Variables ────────────────────────────────────────
@@ -32,7 +33,7 @@ const extract = async () => {
32
33
  data.push(...transform(JSON.parse(line)));
33
34
  }
34
35
  try {
35
- count += await load(data);
36
+ count += await withRetry(load, [data], 5, 60_000);
36
37
  data.length = 0;
37
38
  }
38
39
  catch (err) {
@@ -79,8 +80,7 @@ export const main = async () => {
79
80
  process.exit(1);
80
81
  }
81
82
  if (failedBatches.length === 0) {
82
- // await file.delete();
83
- // log.info("Object deleted");
83
+ await file.delete();
84
84
  }
85
85
  };
86
86
  main();
package/dist/src/jobs/etl/rewards.js CHANGED
@@ -3,6 +3,7 @@ if (!process.env.ENV || !process.env.CHAIN_ID || !process.env.ROOT)
3
3
  throw new Error("[ENV]: missing variable");
4
4
  import { log } from "../../utils/logger";
5
5
  import { apiDbClient } from "../../utils/prisma";
6
+ import { withRetry } from "@sdk";
6
7
  import { S3Client } from "bun";
7
8
  import moment from "moment";
8
9
  // ─── Global Variables ────────────────────────────────────────
@@ -32,7 +33,7 @@ const extract = async () => {
32
33
  data.push(transform(JSON.parse(line)));
33
34
  }
34
35
  try {
35
- count += await load(data);
36
+ count += await withRetry(load, [data], 5, 60_000);
36
37
  data.length = 0;
37
38
  }
38
39
  catch (err) {