@merkl/api 0.10.392 → 0.10.393

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,55 +1,52 @@
1
1
  // ─── Pending Rewards Etl ─────────────────────────────────────────────────────
2
2
  if (!process.env.ENV || !process.env.FILENAME)
3
3
  throw new Error("[ENV]: missing variable");
4
- import { BucketService } from "../../modules/v4/bucket/bucket.service";
5
4
  import { RewardService } from "../../modules/v4/reward";
6
- import { withRetry } from "@sdk";
5
+ import { S3Client } from "bun";
7
6
  import moment from "moment";
8
- import { log } from "../../utils/logger";
9
7
  // ─── Constants ───────────────────────────────────────────────
10
8
  const BATCH_SIZE = 20_000;
11
9
  // ─── Global Variables ────────────────────────────────────────
12
10
  const [chainIdString, root, campaignId] = process.env.FILENAME.split("_");
13
11
  const chainId = Number.parseInt(chainIdString);
14
- const pendingsToCreate = [[]];
12
+ const gcsClient = new S3Client({
13
+ accessKeyId: process.env.GCS_ACCESS_ID,
14
+ secretAccessKey: process.env.GCS_SECRET,
15
+ endpoint: process.env.GCS_ENDPOINT,
16
+ bucket: `merkl-rewards-lake-${process.env.ENV}`,
17
+ });
18
+ const file = gcsClient.file(`pendings/${process.env.FILENAME}`);
15
19
  const failedBatches = [];
16
20
  // ─── Extract ─────────────────────────────────────────────────────────────────
17
21
  const extract = async () => {
22
+ if (!file.exists())
23
+ throw new Error("File does not exist.");
24
+ const textDecoder = new TextDecoder();
25
+ let buffer = "";
26
+ const stream = file.stream();
18
27
  let count = 0;
19
- let currentBatchIndex = 0;
20
- await BucketService.readStreamFromBucket(`pendings/${process.env.FILENAME}.gz`, `merkl-rewards-lake-${process.env.ENV}`, `merkl-data-${process.env.ENV}`, async (x) => {
21
- pendingsToCreate[currentBatchIndex].push(transform(JSON.parse(x)));
22
- if (pendingsToCreate[currentBatchIndex].length >= BATCH_SIZE) {
23
- try {
24
- count += await withRetry(load, [pendingsToCreate[currentBatchIndex]], 5, 10_000);
25
- log.info(`Successfully inserted a batch of ${count} rewards`);
28
+ const data = [];
29
+ for await (const chunk of stream) {
30
+ buffer += textDecoder.decode(chunk);
31
+ const lines = buffer.split("\n");
32
+ buffer = lines.pop() || "";
33
+ for (const line of lines) {
34
+ data.push(JSON.parse(line));
35
+ if (data.length >= BATCH_SIZE) {
36
+ try {
37
+ count += await load(data);
38
+ }
39
+ catch (err) {
40
+ console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
41
+ failedBatches.push(data);
42
+ data.length = 0;
43
+ }
26
44
  }
27
- catch (err) {
28
- console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
29
- failedBatches.push(currentBatchIndex);
30
- }
31
- currentBatchIndex++;
32
- pendingsToCreate.push([]);
33
45
  }
34
- return;
35
- });
36
- // ─── Current Batch Not In DB Yet ─────────────────────────────────────
37
- try {
38
- const count = await load(pendingsToCreate[currentBatchIndex]);
39
- if (count !== 0)
40
- log.info(`Successfully inserted a batch of ${count} rewards`);
41
- }
42
- catch (err) {
43
- console.error(`Failed to insert a batch, adding it to the fail queue.\n${err}`);
44
- failedBatches.push(currentBatchIndex);
45
46
  }
46
47
  return count;
47
48
  };
48
- // ─── Transform ───────────────────────────────────────────────────────────────
49
- const transform = (pending) => {
50
- return pending;
51
- };
52
- // ─── Load ────────────────────────────────────────────────────────────────────
49
+ // ─── Transform & Load ────────────────────────────────────────────────────────
53
50
  const load = async (pendings) => {
54
51
  const { updated, created } = await RewardService.updatePendings({
55
52
  distributionChainId: chainId,
@@ -65,11 +62,14 @@ export const main = async () => {
65
62
  const start = moment().unix();
66
63
  // ─── Start Rewards ETL ───────────────────────────────────────────────
67
64
  const count = await extract();
68
- log.info(`✅ Successfully created ${count} new records in ${moment().unix() - start} sec`);
69
- if (failedBatches.length !== 0)
70
- log.info(`${failedBatches.length}/${pendingsToCreate.length} batches failed.`);
65
+ console.log(`✅ Successfully created ${count} new records in ${moment().unix() - start} sec`);
66
+ if (failedBatches.length !== 0) {
67
+ console.log(`${failedBatches.length} batches failed.`);
68
+ process.exit(1);
69
+ }
71
70
  if (failedBatches.length === 0) {
72
- await BucketService.deleteFile(`rewards/${process.env.FILENAME}.gz`, `merkl-rewards-lake-${process.env.ENV}`, `merkl-data-${process.env.ENV}`);
71
+ await file.delete();
72
+ console.log("Object deleted");
73
73
  }
74
74
  };
75
75
  main();