@defillama/sdk 5.0.236 → 5.0.240

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/package.json +3 -3
  2. package/build/Balances.d.ts +0 -139
  3. package/build/Balances.js +0 -363
  4. package/build/ChainApi.d.ts +0 -142
  5. package/build/ChainApi.js +0 -297
  6. package/build/abi/abi2.d.ts +0 -5
  7. package/build/abi/abi2.js +0 -215
  8. package/build/abi/cachedABIs.d.ts +0 -4
  9. package/build/abi/cachedABIs.js +0 -8
  10. package/build/abi/convertResults.d.ts +0 -5
  11. package/build/abi/convertResults.js +0 -60
  12. package/build/abi/index.d.ts +0 -31
  13. package/build/abi/index.js +0 -327
  14. package/build/abi/multicall.d.ts +0 -23
  15. package/build/abi/multicall.js +0 -382
  16. package/build/abi/multicall3.d.ts +0 -7
  17. package/build/abi/multicall3.js +0 -397
  18. package/build/abi/rpcCall.d.ts +0 -4
  19. package/build/abi/rpcCall.js +0 -62
  20. package/build/abi/tron.d.ts +0 -33
  21. package/build/abi/tron.js +0 -110
  22. package/build/api.d.ts +0 -10
  23. package/build/api.js +0 -45
  24. package/build/api2.d.ts +0 -5
  25. package/build/api2.js +0 -41
  26. package/build/chains/algorand.d.ts +0 -277
  27. package/build/chains/algorand.js +0 -618
  28. package/build/chains/aptos.d.ts +0 -230
  29. package/build/chains/aptos.js +0 -612
  30. package/build/chains/cardano.d.ts +0 -242
  31. package/build/chains/cardano.js +0 -407
  32. package/build/chains/cosmos.d.ts +0 -252
  33. package/build/chains/cosmos.js +0 -618
  34. package/build/chains/icp.d.ts +0 -235
  35. package/build/chains/icp.js +0 -1040
  36. package/build/chains/index.d.ts +0 -23
  37. package/build/chains/index.js +0 -59
  38. package/build/chains/near.d.ts +0 -192
  39. package/build/chains/near.js +0 -446
  40. package/build/chains/rpc.d.ts +0 -125
  41. package/build/chains/rpc.js +0 -326
  42. package/build/chains/starknet.d.ts +0 -205
  43. package/build/chains/starknet.js +0 -884
  44. package/build/chains/stellar.d.ts +0 -410
  45. package/build/chains/stellar.js +0 -1082
  46. package/build/chains/substrate.d.ts +0 -333
  47. package/build/chains/substrate.js +0 -1042
  48. package/build/chains/sui.d.ts +0 -365
  49. package/build/chains/sui.js +0 -1135
  50. package/build/chains/svm.d.ts +0 -340
  51. package/build/chains/svm.js +0 -760
  52. package/build/chains/tezos.d.ts +0 -225
  53. package/build/chains/tezos.js +0 -412
  54. package/build/chains/ton.d.ts +0 -353
  55. package/build/chains/ton.js +0 -788
  56. package/build/chains/tron.d.ts +0 -45
  57. package/build/chains/tron.js +0 -110
  58. package/build/chains/utxo.d.ts +0 -202
  59. package/build/chains/utxo.js +0 -592
  60. package/build/chains/xrpl.d.ts +0 -179
  61. package/build/chains/xrpl.js +0 -467
  62. package/build/computeTVL/blocks.d.ts +0 -1
  63. package/build/computeTVL/blocks.js +0 -12
  64. package/build/computeTVL/humanizeNumber.d.ts +0 -1
  65. package/build/computeTVL/humanizeNumber.js +0 -19
  66. package/build/erc20/index.d.ts +0 -31
  67. package/build/erc20/index.js +0 -56
  68. package/build/eth/index.d.ts +0 -22
  69. package/build/eth/index.js +0 -118
  70. package/build/general.d.ts +0 -5
  71. package/build/general.js +0 -20
  72. package/build/generalUtil.d.ts +0 -8
  73. package/build/generalUtil.js +0 -73
  74. package/build/index.d.ts +0 -32
  75. package/build/index.js +0 -74
  76. package/build/providers.json +0 -1
  77. package/build/types.d.ts +0 -76
  78. package/build/types.js +0 -2
  79. package/build/util/LlamaProvider.d.ts +0 -46
  80. package/build/util/LlamaProvider.js +0 -365
  81. package/build/util/blocks.d.ts +0 -36
  82. package/build/util/blocks.js +0 -292
  83. package/build/util/cache.d.ts +0 -45
  84. package/build/util/cache.js +0 -309
  85. package/build/util/chainUtils/data.json +0 -1356
  86. package/build/util/chainUtils/index.d.ts +0 -11
  87. package/build/util/chainUtils/index.js +0 -136
  88. package/build/util/coins.d.ts +0 -19
  89. package/build/util/coins.js +0 -113
  90. package/build/util/common.d.ts +0 -34
  91. package/build/util/common.js +0 -373
  92. package/build/util/computeTVL.d.ts +0 -19
  93. package/build/util/computeTVL.js +0 -102
  94. package/build/util/cosmos.d.ts +0 -1
  95. package/build/util/cosmos.js +0 -8
  96. package/build/util/debugLog.d.ts +0 -4
  97. package/build/util/debugLog.js +0 -18
  98. package/build/util/discord.d.ts +0 -1
  99. package/build/util/discord.js +0 -33
  100. package/build/util/elastic.d.ts +0 -43
  101. package/build/util/elastic.js +0 -113
  102. package/build/util/env.d.ts +0 -19
  103. package/build/util/env.js +0 -158
  104. package/build/util/graph.d.ts +0 -8
  105. package/build/util/graph.js +0 -50
  106. package/build/util/index.d.ts +0 -32
  107. package/build/util/index.js +0 -227
  108. package/build/util/indexer.compatibility.d.ts +0 -6
  109. package/build/util/indexer.compatibility.js +0 -64
  110. package/build/util/indexer.d.ts +0 -92
  111. package/build/util/indexer.js +0 -744
  112. package/build/util/indexerSql.d.ts +0 -9
  113. package/build/util/indexerSql.js +0 -102
  114. package/build/util/internal-cache.d.ts +0 -11
  115. package/build/util/internal-cache.js +0 -77
  116. package/build/util/logs.d.ts +0 -55
  117. package/build/util/logs.decode.shared.d.ts +0 -8
  118. package/build/util/logs.decode.shared.js +0 -289
  119. package/build/util/logs.js +0 -401
  120. package/build/util/promisePool.d.ts +0 -8
  121. package/build/util/promisePool.js +0 -31
  122. package/build/util/r2.d.ts +0 -4
  123. package/build/util/r2.js +0 -52
  124. package/build/util/removeRedundantRPCs.d.ts +0 -1
  125. package/build/util/removeRedundantRPCs.js +0 -49
  126. package/build/util/transactions.d.ts +0 -35
  127. package/build/util/transactions.js +0 -22
  128. package/build/util/updateChainLabels.d.ts +0 -1
  129. package/build/util/updateChainLabels.js +0 -5
  130. package/build/util/updateProviderList.d.ts +0 -1
  131. package/build/util/updateProviderList.js +0 -274
@@ -1,292 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.getLatestBlock = exports.chainsForBlocks = void 0;
7
- exports.getChainBlocks = getChainBlocks;
8
- exports.getBlocks = getBlocks;
9
- exports.getCurrentBlocks = getCurrentBlocks;
10
- exports.getBlock = getBlock;
11
- exports.getBlockNumber = getBlockNumber;
12
- exports.getCurrentChainBlock = getCurrentChainBlock;
13
- exports.lookupBlock = lookupBlock;
14
- exports.getTimestamp = getTimestamp;
15
- const general_1 = require("../general");
16
- const common_1 = require("./common");
17
- const debugLog_1 = require("./debugLog");
18
- const cosmos_1 = require("./cosmos");
19
- const cache_1 = require("./cache");
20
- const p_limit_1 = __importDefault(require("p-limit"));
21
- const env_1 = require("./env");
22
- const defaultChains = ["avax", "bsc", "polygon", "arbitrum"];
23
- exports.chainsForBlocks = defaultChains;
24
- const blockRetries = 3;
25
- async function getChainBlocks(timestamp, chains = defaultChains) {
26
- const chainBlocks = {};
27
- const setBlock = async (chain) => chainBlocks[chain] = await getBlockNumber(chain, timestamp);
28
- await Promise.all(chains.map(setBlock));
29
- return chainBlocks;
30
- }
31
- async function getBlocks(timestamp, chains = undefined) {
32
- chains = chains === null || chains === void 0 ? void 0 : chains.filter(i => i !== 'ethereum');
33
- const [ethBlock, chainBlocks] = await Promise.all([getBlockNumber('ethereum', timestamp), getChainBlocks(timestamp, chains)]);
34
- chainBlocks['ethereum'] = ethBlock;
35
- return {
36
- ethereumBlock: ethBlock,
37
- chainBlocks,
38
- };
39
- }
40
- async function getCurrentBlocks(chains = undefined) {
41
- if (chains)
42
- chains = chains.filter(i => i !== "ethereum");
43
- const block = await getBlock('ethereum');
44
- const chainBlocks = await getChainBlocks(undefined, chains);
45
- chainBlocks['ethereum'] = block.block;
46
- return {
47
- timestamp: block.timestamp,
48
- ethereumBlock: block.block,
49
- chainBlocks,
50
- };
51
- }
52
- async function getBlock(chain, timestamp, options = {}) {
53
- if (!timestamp)
54
- return getCurrentChainBlock(chain);
55
- let error;
56
- for (let i = 0; i < blockRetries; i++) {
57
- try {
58
- let res = await lookupBlock(timestamp, { ...options, chain, });
59
- return res;
60
- }
61
- catch (e) {
62
- error = e;
63
- }
64
- }
65
- throw error;
66
- }
67
- async function getBlockNumber(chain, timestamp) {
68
- return (await getBlock(chain, timestamp)).block;
69
- }
70
- const refreshInterval = 1000 * 60 * 1; // 1 minute
71
- function getCurrentChainBlock(chain = 'ethereum') {
72
- const { timestamp, } = currentChainBlockCache[chain] || {};
73
- const currentTimestamp = Date.now();
74
- if (!timestamp || currentTimestamp - timestamp > refreshInterval) {
75
- currentChainBlockCache[chain] = {
76
- timestamp: currentTimestamp,
77
- promise: _getCurrentChainBlock()
78
- };
79
- }
80
- return currentChainBlockCache[chain].promise;
81
- async function _getCurrentChainBlock() {
82
- var _a, _b;
83
- try {
84
- const provider = getExtraProvider(chain);
85
- const { number, timestamp, } = await provider.getBlock('latest');
86
- const response = { block: number, timestamp, number };
87
- validateCurrentBlock(response, chain);
88
- currentChainBlockCache[chain] = { timestamp: currentTimestamp, promise: response };
89
- return response; // we are interested only in these fields
90
- }
91
- catch (e) {
92
- delete currentChainBlockCache[chain];
93
- if ((_b = (_a = e === null || e === void 0 ? void 0 : e.message) === null || _a === void 0 ? void 0 : _a.includes) === null || _b === void 0 ? void 0 : _b.call(_a, "Last block of chain"))
94
- throw e;
95
- throw (0, common_1.formError)(e);
96
- }
97
- }
98
- }
99
- const algorandBlockProvider = {
100
- getBlock: async (height) => {
101
- if (height !== 'latest')
102
- return (0, common_1.fetchJson)(`https://algoindexer.algoexplorerapi.io/v2/blocks/${height}`)
103
- .then((block) => ({
104
- number: block.round,
105
- timestamp: block.timestamp
106
- }));
107
- return (0, common_1.fetchJson)('https://algoindexer.algoexplorerapi.io/health')
108
- .then((block) => algorandBlockProvider.getBlock(block.round));
109
- }
110
- };
111
- async function fetchBlockFromProvider(height, chain) {
112
- const provider = getExtraProvider(chain !== null && chain !== void 0 ? chain : "ethereum");
113
- const block = await provider.getBlock(height);
114
- if (block === null) {
115
- throw new Error(`Can't get block of chain ${chain !== null && chain !== void 0 ? chain : "ethereum"}`);
116
- }
117
- return block;
118
- }
119
- function getExtraProvider(chain = "ethereum") {
120
- if ((0, cosmos_1.isCosmosChain)(chain)) // maybe check if it is also an evm chain?
121
- return (0, cosmos_1.getCosmosProvider)(chain);
122
- if (chain === "algorand")
123
- return algorandBlockProvider;
124
- return (0, general_1.getProvider)(chain);
125
- }
126
- exports.getLatestBlock = getCurrentChainBlock;
127
- const intialBlocks = {
128
- planq: 14000000,
129
- sei: 150023881,
130
- terra: 4724001,
131
- crab: 4969901
132
- };
133
- let lastBlockCacheSave = Date.now();
134
- const blockCacheSaveInterval = 1000 * 60 * 5; // 5 minutes
135
- const { data: blockTimeCache, saveCacheFile: saveBlockCacheFile } = (0, cache_1.getTempLocalCache)({ file: 'BlockCache-v1.json', defaultData: {}, clearAfter: cache_1.ONE_WEEK, returnWithSaveFunction: true });
136
- function validateCurrentBlock(block, chain = 'ethereum') {
137
- // sometimes the chain is halted or facing issues, so we skip the validation for those chains
138
- if ((0, env_1.shouldSkipBlockValidation)(chain))
139
- return;
140
- const provider = getExtraProvider(chain);
141
- const now = Math.floor(Date.now() / 1000);
142
- const minutesDiff = Math.floor((now - block.timestamp) / 60);
143
- if (minutesDiff > 60)
144
- throw new Error(`Last block for ${chain} is ${minutesDiff} minutes behind (${new Date(block.timestamp * 1000)}). Provider is "${(0, common_1.getProviderUrl)(provider)}"`);
145
- }
146
- const chainLimiters = {};
147
- function getChainLimiter(chain) {
148
- if (!chainLimiters[chain])
149
- chainLimiters[chain] = createLimiter(chain);
150
- return chainLimiters[chain];
151
- function createLimiter(chain) {
152
- return (0, p_limit_1.default)((0, env_1.getParallelGetBlocksLimit)(chain));
153
- }
154
- }
155
- async function lookupBlock(timestamp, extraParams) {
156
- const { chain = "ethereum" } = extraParams !== null && extraParams !== void 0 ? extraParams : {};
157
- const limiter = getChainLimiter(chain);
158
- return limiter(() => _lookupBlock(timestamp, extraParams));
159
- }
160
- const earliestAvailableBlockRange = {
161
- nibiru: 4 * 1e5, // nibiru hold only the last 400k block data
162
- evmos: 2 * 1e5,
163
- };
164
- async function _lookupBlock(timestamp, extraParams) {
165
- var _a, _b;
166
- const { allowedTimeRange = 5 * 60, acceptableBlockImprecision = 10 } = extraParams !== null && extraParams !== void 0 ? extraParams : {};
167
- const chain = (_a = extraParams === null || extraParams === void 0 ? void 0 : extraParams.chain) !== null && _a !== void 0 ? _a : "ethereum";
168
- let time = Date.now();
169
- if (timestamp > time / 1000) {
170
- throw new Error(`Requesting for block in the future ${timestamp} is in the future, current time is ${Math.floor(time / 1000)}`);
171
- }
172
- if (chain === 'waves') {
173
- const api = `https://nodes.wavesnodes.com/blocks/heightByTimestamp/${timestamp}`;
174
- const data = await (0, common_1.fetchJson)(api);
175
- return {
176
- timestamp,
177
- block: +data.height,
178
- number: +data.height,
179
- };
180
- }
181
- let low = (_b = intialBlocks[chain]) !== null && _b !== void 0 ? _b : 100;
182
- let envLowValue = process.env[`${chain.toUpperCase()}_BLOCK_LOW`];
183
- if (envLowValue)
184
- low = parseInt(envLowValue);
185
- let lowBlock = getLowBlock();
186
- let highBlock = getHighBlock();
187
- if ((lowBlock === null || lowBlock === void 0 ? void 0 : lowBlock.number) < low)
188
- lowBlock = undefined;
189
- let block;
190
- let i = 0;
191
- let blockImprecision;
192
- let imprecision;
193
- try {
194
- let firstBlock, lastBlock;
195
- if (!!earliestAvailableBlockRange[chain]) {
196
- lastBlock = await (0, exports.getLatestBlock)(chain);
197
- let firstBlockNum = lastBlock.number - earliestAvailableBlockRange[chain];
198
- firstBlock = await fetchBlockFromProvider(firstBlockNum, chain);
199
- }
200
- else {
201
- [lastBlock, firstBlock] = await Promise.all([
202
- highBlock ? highBlock : (0, exports.getLatestBlock)(chain),
203
- lowBlock ? lowBlock : fetchBlockFromProvider(low, chain),
204
- ]);
205
- }
206
- lowBlock = firstBlock;
207
- if (!highBlock)
208
- highBlock = lastBlock;
209
- if (Math.abs(highBlock.timestamp - timestamp) < 60 * 5) {
210
- // Short-circuit in case we are trying to get the current block
211
- return {
212
- block: highBlock.number,
213
- number: highBlock.number,
214
- timestamp: highBlock.timestamp
215
- };
216
- }
217
- updateBlock();
218
- while (imprecision > allowedTimeRange && blockImprecision > acceptableBlockImprecision && i < 50) { // We lose some precision (max ~15 minutes) but reduce #calls needed
219
- ++i;
220
- const blockDiff = highBlock.number - lowBlock.number;
221
- const timeDiff = highBlock.timestamp - lowBlock.timestamp;
222
- const avgBlockTime = timeDiff / blockDiff;
223
- let closeBlock = Math.floor(lowBlock.number + (timestamp - lowBlock.timestamp) / avgBlockTime);
224
- if (closeBlock > highBlock.number)
225
- closeBlock = highBlock.number;
226
- const midBlock = Math.floor((lowBlock.number + highBlock.number) / 2);
227
- const blocks = await Promise.all([
228
- fetchBlockFromProvider(closeBlock, chain),
229
- fetchBlockFromProvider(midBlock, chain),
230
- ]);
231
- updateBlock(blocks);
232
- }
233
- if (i > 2 && env_1.ENV_CONSTANTS.GET_BLOCK_STATS)
234
- (0, debugLog_1.debugLog)(`chain: ${chain} block: ${block.number} #calls: ${i} imprecision: ${Number((imprecision) / 60).toFixed(2)} (min) Time Taken: ${Number((Date.now() - time) / 1000).toFixed(2)} (in sec)`);
235
- if (Math.abs(block.timestamp - timestamp) > 3600
236
- && chain !== "bsc" // this check is there because bsc halted the chain for few days
237
- && !(0, env_1.shouldSkipBlockValidation)(chain)) {
238
- throw new Error("Block selected is more than 1 hour away from the requested timestamp");
239
- }
240
- return {
241
- number: block.number,
242
- block: block.number,
243
- timestamp: block.timestamp
244
- };
245
- }
246
- catch (e) {
247
- throw (0, common_1.formError)(e);
248
- }
249
- function updateBlock(blocks = []) {
250
- blocks.forEach(addBlockToCache);
251
- const getPrecision = (block) => block.timestamp - timestamp > 0 ? block.timestamp - timestamp : timestamp - block.timestamp;
252
- blocks.push(highBlock, lowBlock);
253
- blocks.sort((a, b) => getPrecision(a) - getPrecision(b));
254
- block = blocks[0];
255
- // find the closest upper and lower bound between 4 points
256
- lowBlock = blocks.filter(i => i.timestamp < timestamp).reduce((lowestBlock, block) => (timestamp - lowestBlock.timestamp) < (timestamp - block.timestamp) ? lowestBlock : block, lowBlock);
257
- highBlock = blocks.filter(i => i.timestamp > timestamp).reduce((highestBlock, block) => (highestBlock.timestamp - timestamp) < (block.timestamp - timestamp) ? highestBlock : block, highBlock);
258
- imprecision = getPrecision(block);
259
- blockImprecision = highBlock.number - lowBlock.number;
260
- // debugLog(`chain: ${chain} block: ${block.number} #calls: ${i} imprecision: ${Number((imprecision)/60).toFixed(2)} (min) block diff: ${blockImprecision} Time Taken: ${Number((Date.now()-time)/1000).toFixed(2)} (in sec)`)
261
- }
262
- function getChainBlockTimeCache(chain) {
263
- if (!blockTimeCache[chain])
264
- blockTimeCache[chain] = {};
265
- return blockTimeCache[chain];
266
- }
267
- function addBlockToCache(block) {
268
- const { number, timestamp } = block;
269
- getChainBlockTimeCache(chain)[block.number] = { number, timestamp, };
270
- // save the block cache every 5 minutes
271
- if (Date.now() - lastBlockCacheSave > blockCacheSaveInterval) {
272
- lastBlockCacheSave = Date.now();
273
- saveBlockCacheFile();
274
- }
275
- }
276
- function getLowBlock() {
277
- return Object.values(getChainBlockTimeCache(chain))
278
- .filter(i => i.timestamp < timestamp)
279
- .sort((a, b) => b.timestamp - a.timestamp)[0];
280
- }
281
- function getHighBlock() {
282
- return Object.values(getChainBlockTimeCache(chain))
283
- .filter(i => i.timestamp > timestamp)
284
- .sort((a, b) => a.timestamp - b.timestamp)[0];
285
- }
286
- }
287
- async function getTimestamp(height, chain) {
288
- const provider = getExtraProvider(chain);
289
- const block = await provider.getBlock(height);
290
- return block.timestamp;
291
- }
292
- const currentChainBlockCache = {};
@@ -1,45 +0,0 @@
1
- export declare const currentVersion = "zlib-1.0";
2
- export declare function readCache(file: string, options?: ReadCacheOptions): Promise<any>;
3
- interface WriteCacheOptions {
4
- skipR2CacheWrite?: boolean;
5
- alreadyCompressed?: boolean;
6
- skipCompression?: boolean;
7
- }
8
- interface ReadCacheOptions {
9
- checkIfRecent?: boolean;
10
- readFromR2Cache?: boolean;
11
- skipR2Cache?: boolean;
12
- skipCompression?: boolean;
13
- }
14
- export declare function writeCache(file: string, data: any, options?: WriteCacheOptions): Promise<string | undefined>;
15
- export declare function parseCache(dataString: string | Buffer, options?: ReadCacheOptions): Promise<any>;
16
- export declare function compressCache(data: any, options?: WriteCacheOptions): Promise<string | Buffer<ArrayBufferLike>>;
17
- export declare const ONE_WEEK: number;
18
- export declare function getTempLocalCache({ file, defaultData, clearAfter, returnWithSaveFunction }: {
19
- file: string;
20
- defaultData?: any;
21
- clearAfter?: number;
22
- returnWithSaveFunction?: boolean;
23
- }): any | ({
24
- data: any;
25
- saveCacheFile: () => void;
26
- });
27
- export declare function deleteCache(file: string): Promise<void>;
28
- export declare function readExpiringJsonCache(file: string): Promise<any>;
29
- export declare function writeExpiringJsonCache(file: string, data: any, { expireAfter, // cache for 1 day by default
30
- expiryTimestamp, }?: {
31
- expireAfter?: number;
32
- expiryTimestamp?: number;
33
- }): Promise<void>;
34
- export declare function cachedFetch({ key, endpoint, postData, fetcher, fetchOptions, errorOnNoCache, cacheMaxAge, writeCacheOptions, readCacheOptions, }: {
35
- key: string;
36
- endpoint?: string;
37
- postData?: any;
38
- fetcher?: () => Promise<any>;
39
- fetchOptions?: any;
40
- errorOnNoCache?: boolean;
41
- cacheMaxAge?: number;
42
- writeCacheOptions?: WriteCacheOptions;
43
- readCacheOptions?: ReadCacheOptions;
44
- }): Promise<any>;
45
- export {};
@@ -1,309 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ONE_WEEK = exports.currentVersion = void 0;
4
- exports.readCache = readCache;
5
- exports.writeCache = writeCache;
6
- exports.parseCache = parseCache;
7
- exports.compressCache = compressCache;
8
- exports.getTempLocalCache = getTempLocalCache;
9
- exports.deleteCache = deleteCache;
10
- exports.readExpiringJsonCache = readExpiringJsonCache;
11
- exports.writeExpiringJsonCache = writeExpiringJsonCache;
12
- exports.cachedFetch = cachedFetch;
13
- const debugLog_1 = require("./debugLog");
14
- const r2_1 = require("./r2");
15
- const zlib_1 = require("zlib");
16
- const util_1 = require("util");
17
- const env_1 = require("./env");
18
- const common_1 = require("./common");
19
- const brotliOptions = {
20
- [zlib_1.constants.BROTLI_PARAM_MODE]: zlib_1.constants.BROTLI_MODE_TEXT,
21
- [zlib_1.constants.BROTLI_PARAM_QUALITY]: zlib_1.constants.BROTLI_MAX_QUALITY,
22
- };
23
- function zipAsync(data) {
24
- return (0, util_1.promisify)(zlib_1.brotliCompress)(data, brotliOptions);
25
- }
26
- function unzipAsync(data) {
27
- return (0, util_1.promisify)(zlib_1.brotliDecompress)(data, brotliOptions).then(r => r.toString());
28
- }
29
- const _fs = require('fs');
30
- const fs = _fs.promises;
31
- const path = require('path');
32
- const foldersCreated = {};
33
- exports.currentVersion = 'zlib-1.0';
34
- function getCacheRootFolder() {
35
- const defaultCacheRootFolder = path.join(__dirname, 'local_cache');
36
- return path.join((0, env_1.getEnvCacheFolder)(defaultCacheRootFolder), exports.currentVersion);
37
- }
38
- function getFilePath(file) {
39
- return path.join(getCacheRootFolder(), file);
40
- }
41
- async function createSubPath(folderPath) {
42
- const datRoot = getCacheRootFolder();
43
- folderPath = folderPath.replace(datRoot, '');
44
- if (foldersCreated[folderPath])
45
- return;
46
- try {
47
- await fs.mkdir(path.join(datRoot, folderPath), { recursive: true });
48
- }
49
- catch (error) {
50
- // sdk.log('Error creating folder path:', error)
51
- }
52
- foldersCreated[folderPath] = true;
53
- }
54
- const ONE_DAY_IN_MS = 1000 * 60 * 60 * 24;
55
- async function readCache(file, options = {}) {
56
- if (options.skipCompression)
57
- file = `${file}-uncompressed`;
58
- try {
59
- const data = await readFile();
60
- return await parseCache(data, options);
61
- }
62
- catch (error) {
63
- // debugLog('Error reading cache:', error)
64
- return {};
65
- }
66
- async function readFile() {
67
- var _a;
68
- try {
69
- const data = await readDataFromLocal((_a = options.readFromR2Cache) !== null && _a !== void 0 ? _a : false);
70
- return data;
71
- }
72
- catch (error) {
73
- // debugLog('Error reading cache:', error)
74
- return readDataFromR2();
75
- }
76
- }
77
- async function readDataFromLocal(readFromR2Cache) {
78
- const filePath = getFilePath(file);
79
- if (options.checkIfRecent) {
80
- const stats = await fs.stat(filePath);
81
- if (Date.now() - stats.mtimeMs > ONE_DAY_IN_MS)
82
- throw new Error('File too old, read from R2 instead');
83
- }
84
- if (readFromR2Cache)
85
- throw new Error('Read from R2 cache');
86
- const data = await fs.readFile(filePath);
87
- return data;
88
- }
89
- async function readDataFromR2() {
90
- var _a;
91
- if (options.skipR2Cache)
92
- return;
93
- try {
94
- const r2Data = await (0, r2_1.getR2JSONString)(exports.currentVersion + '/' + file, { throwErrorOnFail: (_a = options.readFromR2Cache) !== null && _a !== void 0 ? _a : false });
95
- if (r2Data) {
96
- // `file` already carries the `-uncompressed` suffix (added above), so omit skipCompression here to avoid writeCache appending it again
97
- await writeCache(file, r2Data, { alreadyCompressed: true, skipR2CacheWrite: true });
98
- return r2Data;
99
- }
100
- }
101
- catch (error) {
102
- if (options.readFromR2Cache) {
103
- // if we were forced to read from R2 cache, try reading from local cache if R2 fails
104
- try {
105
- const data = await readDataFromLocal(false);
106
- return data;
107
- }
108
- catch (_e) { }
109
- }
110
- return {};
111
- }
112
- }
113
- }
114
- async function writeCache(file, data, options = {}) {
115
- try {
116
- const fileData = options.alreadyCompressed ? data : await compressCache(data, options);
117
- if (!data || (typeof data === 'string' && data.length < 20) || fileData.length < 20) {
118
- (0, debugLog_1.debugLog)('Data too small to cache: ', file);
119
- return;
120
- }
121
- if (!options.skipCompression && isSameData(data, await readCache(file, { skipR2Cache: true, skipCompression: options.skipCompression, })))
122
- return;
123
- if (options.skipCompression)
124
- file = `${file}-uncompressed`;
125
- const filePath = getFilePath(file);
126
- await createSubPath(path.dirname(filePath));
127
- await fs.writeFile(filePath, fileData);
128
- if (!options.skipR2CacheWrite) {
129
- await (0, r2_1.storeR2JSONString)(exports.currentVersion + '/' + file, fileData);
130
- }
131
- return fileData;
132
- }
133
- catch (error) {
134
- if (typeof data === 'object' && data !== null && Object.keys(data).length === 0)
135
- return;
136
- (0, debugLog_1.debugLog)('Error writing cache:', error);
137
- }
138
- }
139
- async function parseCache(dataString, options = {}) {
140
- var _a;
141
- let dataBuffer;
142
- if (typeof dataString === 'string')
143
- dataBuffer = Buffer.from(dataString, 'base64');
144
- else {
145
- dataBuffer = dataString;
146
- dataString = dataBuffer.toString('utf8');
147
- }
148
- let decompressed;
149
- let _unzipAsync = (str) => options.skipCompression ? dataString : unzipAsync(str); // if skipCompression is true, we don't decompress the data
150
- try {
151
- decompressed = await _unzipAsync((_a = dataBuffer) !== null && _a !== void 0 ? _a : dataString);
152
- }
153
- catch (e) {
154
- // dataString = dataString.toString('utf8')
155
- const convertedDataString = Buffer.from(dataString, 'base64');
156
- decompressed = await _unzipAsync(convertedDataString);
157
- }
158
- return JSON.parse(decompressed).llamaWrapped;
159
- }
160
- async function compressCache(data, options = {}) {
161
- const dataString = JSON.stringify({ version: exports.currentVersion, llamaWrapped: data });
162
- if (options.skipCompression)
163
- return dataString;
164
- const comressedCache = await zipAsync(dataString);
165
- return comressedCache;
166
- }
167
- function isSameData(data1, data2) {
168
- return JSON.stringify(data1) === JSON.stringify(data2);
169
- }
170
- exports.ONE_WEEK = 1000 * 60 * 60 * 24 * 7;
171
- function getTempLocalCache({ file, defaultData = {}, clearAfter = exports.ONE_WEEK, returnWithSaveFunction = false }) {
172
- const now = Math.floor(Date.now() / 1e3);
173
- let emptyFile = { created: now, data: defaultData };
174
- let fileData = emptyFile;
175
- const filePath = getFilePath(file);
176
- let saveCacheFlag = false; // to ensure that we save only once on exit
177
- createSubPath(path.dirname(filePath)); // create folder if not exists, this is async but we don't care
178
- try {
179
- const data = _fs.readFileSync(filePath);
180
- fileData = JSON.parse(data.toString());
181
- if (now - fileData.created > clearAfter)
182
- fileData = emptyFile; // clear cache
183
- }
184
- catch (error) { }
185
- // save cache on exit
186
- process.on('exit', _save);
187
- process.on('SIGTERM', _save);
188
- process.on('SIGINT', _save);
189
- if (returnWithSaveFunction) {
190
- return {
191
- data: fileData.data,
192
- saveCacheFile: () => saveCache(true)
193
- };
194
- }
195
- return fileData.data;
196
- function _save() {
197
- saveCache(false);
198
- }
199
- function saveCache(skipCacheFlagCheck = false) {
200
- if (!skipCacheFlagCheck) {
201
- if (saveCacheFlag)
202
- return;
203
- saveCacheFlag = true;
204
- }
205
- try {
206
- // debugLog('Saving cache to file:', filePath)
207
- removePromisesAndFunctions(fileData);
208
- _fs.writeFileSync(filePath, JSON.stringify(fileData));
209
- }
210
- catch (error) {
211
- try {
212
- (0, debugLog_1.debugLog)('Error saving cache:', error === null || error === void 0 ? void 0 : error.toString());
213
- }
214
- catch (error) { }
215
- }
216
- }
217
- function removePromisesAndFunctions(data) {
218
- if (typeof data === 'object') {
219
- for (const key in data) {
220
- if (typeof data[key] === 'function')
221
- delete data[key];
222
- if (data[key] instanceof Promise)
223
- delete data[key];
224
- else
225
- removePromisesAndFunctions(data[key]);
226
- }
227
- }
228
- }
229
- }
230
- // this removes data only from the file cache, not from the R2 cache
231
- async function deleteCache(file) {
232
- const filePath = getFilePath(file);
233
- await fs.unlink(filePath).catch(() => {
234
- // ignore errors
235
- });
236
- }
237
- async function readExpiringJsonCache(file) {
238
- const options = { readFromR2Cache: false, skipR2Cache: true, skipCompression: true };
239
- file = 'expiring/' + file;
240
- const data = await readCache(file, options);
241
- if (!data || Object.keys(data).length === 0)
242
- return null;
243
- if (data.expiryTimestamp < (Date.now() / 1e3)) {
244
- await deleteCache(`${file}-uncompressed`);
245
- return null;
246
- }
247
- return data.data;
248
- }
249
- async function writeExpiringJsonCache(file, data, { expireAfter = 60 * 60 * 24, // cache for 1 day by default
250
- expiryTimestamp, } = {}) {
251
- file = 'expiring/' + file;
252
- if (!expiryTimestamp)
253
- expiryTimestamp = Math.floor(Date.now() / 1e3) + expireAfter;
254
- const options = { skipR2CacheWrite: true, skipCompression: true };
255
- await writeCache(file, { data, expiryTimestamp }, options);
256
- }
257
- async function cachedFetch({ key, endpoint, postData, fetcher, fetchOptions, errorOnNoCache = true, cacheMaxAge = 0, writeCacheOptions = {
258
- skipCompression: true,
259
- skipR2CacheWrite: true,
260
- }, readCacheOptions = {
261
- skipCompression: true,
262
- skipR2Cache: true,
263
- }, }) {
264
- if (!key) {
265
- if (!endpoint || postData || fetcher)
266
- throw new Error('Key is required for cachedFetch');
267
- key = (0, common_1.getHash)(endpoint);
268
- }
269
- const fileKey = 'cached-fetch/' + key;
270
- // check local cache first
271
- if (cacheMaxAge > 0) {
272
- try {
273
- const filePath = getFilePath(readCacheOptions.skipCompression ? `${fileKey}-uncompressed` : fileKey);
274
- const stats = await fs.stat(filePath);
275
- if (Date.now() - stats.mtimeMs < cacheMaxAge) {
276
- const cachedData = await readCache(fileKey, readCacheOptions);
277
- if (cachedData && Object.keys(cachedData).length > 0)
278
- return cachedData;
279
- }
280
- }
281
- catch (error) {
282
- // cache miss or stat error, proceed to fetch
283
- }
284
- }
285
- try {
286
- let data;
287
- if (fetcher)
288
- data = await fetcher();
289
- else {
290
- if (!endpoint)
291
- throw new Error('Either fetcher or endpoint is required for cachedFetch');
292
- if (postData)
293
- data = await (0, common_1.postJson)(endpoint, postData, fetchOptions);
294
- else
295
- data = await (0, common_1.fetchJson)(endpoint, fetchOptions);
296
- }
297
- await writeCache(fileKey, data, writeCacheOptions);
298
- return data;
299
- }
300
- catch (error) {
301
- const errorMessage = (0, common_1.formError)(error).message;
302
- (0, debugLog_1.debugLog)('Error in cachedFetch:', errorMessage);
303
- // if fetch fails, read from cache (regardless of age)
304
- const cachedData = await readCache(fileKey, readCacheOptions);
305
- if (!cachedData && errorOnNoCache)
306
- throw new Error('Fetch failed and no cache available');
307
- return cachedData;
308
- }
309
- }