@defisaver/positions-sdk 2.1.169-custom-fetching-3-dev → 2.1.169-custom-fetching-4-dev

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/cjs/aaveV3/merkl.js +1 -1
  2. package/cjs/services/merkl.d.ts +17 -9
  3. package/cjs/services/merkl.js +61 -22
  4. package/esm/aaveV3/merkl.js +1 -1
  5. package/esm/services/merkl.d.ts +17 -9
  6. package/esm/services/merkl.js +56 -21
  7. package/package.json +1 -1
  8. package/src/aaveV3/merkl.ts +1 -1
  9. package/src/services/merkl.ts +68 -20
package/cjs/aaveV3/merkl.js CHANGED
@@ -49,7 +49,7 @@ const getMerkleCampaigns = (chainId, asOfTimestamp) => __awaiter(void 0, void 0,
49
49
  mainProtocolId: 'aave',
50
50
  chainId: chainId.toString(),
51
51
  status: types_1.OpportunityStatus.LIVE,
52
- }, asOfTimestamp);
52
+ }, asOfTimestamp, (o) => { var _a; return !((_a = o.type) === null || _a === void 0 ? void 0 : _a.startsWith('AAVE_V4_')); });
53
53
  const relevantOpportunities = opportunities
54
54
  .filter((o) => o.chainId === chainId)
55
55
  .filter((o) => o.status === types_1.OpportunityStatus.LIVE);
package/cjs/services/merkl.d.ts CHANGED
@@ -14,17 +14,23 @@ export declare const fetchMerklOpportunitiesByIds: (ids: string[]) => Promise<Me
14
14
  /**
15
15
  * Returns the reward APR Merkl recorded for the opportunity at the given unix timestamp (seconds),
16
16
  * or `undefined` when no campaign was computing for it then. Records are hourly buckets keyed by
17
- * their start, newest first, so the first bucket at or before the timestamp carries the value in
18
- * force at that moment; the window is a day wide to tolerate computation gaps.
17
+ * their start, newest first, so the bucket of the timestamp's hour carries the value in force at
18
+ * that moment; the window is a day wide to tolerate computation gaps. The window is aligned to the
19
+ * hour, so every block within one hour asks for (and caches) the same records.
19
20
  */
20
21
  export declare const fetchMerklOpportunityAprAt: (opportunityId: string, timestamp: number) => Promise<number | undefined>;
22
+ /** Drops every cached historical response (campaign lists, opportunities, APR buckets). */
23
+ export declare const clearMerklHistoryCache: () => void;
21
24
  /**
22
25
  * Reconstructs the opportunities that were live at `timestamp` (unix seconds), shaped like the
23
26
  * live listing so the per-protocol reward mappers can consume either without knowing which they got:
24
- * 1. campaigns for the protocol (and chain, when the query names one) whose [start, end] covers
25
- * the timestamp identify the opportunities that were rewarded then — the live listing can't,
26
- * since campaigns live then are mostly PAST now and ones live now didn't exist then;
27
- * 2. those opportunities are loaded by id (works for PAST ones too) and narrowed to `query.type`;
27
+ * 1. every LIVE or PAST campaign for the protocol (and chain, when the query names one) is listed
28
+ * once per cache period — the list is block-independent, the exact window check happens here —
29
+ * and the ones whose [start, end] covers the timestamp identify the opportunities rewarded
30
+ * then (the live listing can't: campaigns live then are mostly PAST now and ones live now
31
+ * didn't exist then);
32
+ * 2. those opportunities are loaded by id (works for PAST ones too) and narrowed to `query.type`
33
+ * and `isRelevant`, so no APR history is fetched for opportunities the caller would discard;
28
34
  * 3. each one's `apr` is replaced with the value Merkl recorded at the timestamp, and ones with no
29
35
  * record then are dropped, since nothing was being computed for them.
30
36
  * `status` is forced to LIVE so status filters downstream keep working, `campaigns` carries the
@@ -32,9 +38,11 @@ export declare const fetchMerklOpportunityAprAt: (opportunityId: string, timesta
32
38
  * when Merkl no longer serves one for the opportunity (it drops them once an opportunity is PAST).
33
39
  * `query.status` is ignored: it describes the present, not the requested moment.
34
40
  */
35
- export declare const fetchMerklOpportunitiesAt: (query: Record<string, string>, timestamp: number) => Promise<MerklOpportunity[]>;
41
+ export declare const fetchMerklOpportunitiesAt: (query: Record<string, string>, timestamp: number, isRelevant?: (opportunity: MerklOpportunity) => boolean) => Promise<MerklOpportunity[]>;
36
42
  /**
37
43
  * Live opportunities when no timestamp is given, otherwise the opportunities as they were at that
38
- * unix timestamp (seconds) — see `fetchMerklOpportunitiesAt`.
44
+ * unix timestamp (seconds) — see `fetchMerklOpportunitiesAt`. `isRelevant` only applies to the
45
+ * historical path, where it saves fetching APR history for opportunities the caller would discard;
46
+ * callers still filter the live listing themselves.
39
47
  */
40
- export declare const fetchMerklOpportunities: (query: Record<string, string>, asOfTimestamp?: number) => Promise<MerklOpportunity[]>;
48
+ export declare const fetchMerklOpportunities: (query: Record<string, string>, asOfTimestamp?: number, isRelevant?: (opportunity: MerklOpportunity) => boolean) => Promise<MerklOpportunity[]>;
package/cjs/services/merkl.js CHANGED
@@ -8,8 +8,12 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
8
8
  step((generator = generator.apply(thisArg, _arguments || [])).next());
9
9
  });
10
10
  };
11
+ var __importDefault = (this && this.__importDefault) || function (mod) {
12
+ return (mod && mod.__esModule) ? mod : { "default": mod };
13
+ };
11
14
  Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.fetchMerklOpportunities = exports.fetchMerklOpportunitiesAt = exports.fetchMerklOpportunityAprAt = exports.fetchMerklOpportunitiesByIds = exports.fetchAllMerklCampaigns = exports.fetchAllMerklOpportunities = void 0;
15
+ exports.fetchMerklOpportunities = exports.fetchMerklOpportunitiesAt = exports.clearMerklHistoryCache = exports.fetchMerklOpportunityAprAt = exports.fetchMerklOpportunitiesByIds = exports.fetchAllMerklCampaigns = exports.fetchAllMerklOpportunities = void 0;
16
+ const memoizee_1 = __importDefault(require("memoizee"));
13
17
  const types_1 = require("../types");
14
18
  const utils_1 = require("./utils");
15
19
  const MERKL_API_URL = 'https://fe.defisaver.com/api/merkl';
@@ -22,11 +26,18 @@ const OPPORTUNITY_ID_BATCH_SIZE = 50;
22
26
  // APR history is fetched per opportunity, so historical lookups are throttled to keep a block with
23
27
  // many live campaigns from firing every request at once.
24
28
  const APR_RECORDS_CONCURRENCY = 10;
25
- const SECONDS_PER_DAY = 24 * 60 * 60;
29
+ const SECONDS_PER_HOUR = 60 * 60;
30
+ const SECONDS_PER_DAY = 24 * SECONDS_PER_HOUR;
31
+ // Historical lookups describe the past, so their responses are reused for a while: a job that
32
+ // walks many blocks (or fetches several protocols/chains at one block in parallel) would otherwise
33
+ // re-request the same campaign lists, opportunities and APR buckets for every block.
34
+ const HISTORY_CACHE_TTL = 5 * 60 * 1000;
26
35
  const merklGet = (url, query, label) => __awaiter(void 0, void 0, void 0, function* () {
27
36
  const res = yield fetch(`${url}?${new URLSearchParams(query)}`, { signal: AbortSignal.timeout(utils_1.LONGER_TIMEOUT) });
28
- if (!res.ok)
29
- throw new Error(`Failed to fetch Merkl ${label}`);
37
+ if (!res.ok) {
38
+ const body = yield res.text().catch(() => '');
39
+ throw new Error(`Failed to fetch Merkl ${label} (HTTP ${res.status}${body ? `: ${body.slice(0, 200)}` : ''})`);
40
+ }
30
41
  return res.json();
31
42
  });
32
43
  /**
@@ -61,6 +72,11 @@ const fetchAllMerklCampaigns = (query) => __awaiter(void 0, void 0, void 0, func
61
72
  return (fetchAllMerklPages(MERKL_CAMPAIGNS_URL, query, 'campaigns', MERKL_MAX_PAGE_ITEMS));
62
73
  });
63
74
  exports.fetchAllMerklCampaigns = fetchAllMerklCampaigns;
75
+ const fetchAllMerklCampaignsCached = (0, memoizee_1.default)(exports.fetchAllMerklCampaigns, {
76
+ promise: true,
77
+ maxAge: HISTORY_CACHE_TTL,
78
+ normalizer: ([query]) => JSON.stringify(Object.entries(query).sort()),
79
+ });
64
80
  /**
65
81
  * Fetches the opportunities with the given Merkl-internal ids, regardless of their current status.
66
82
  * Ids Merkl no longer serves (hidden or test campaigns) are simply absent from the result.
@@ -77,18 +93,25 @@ const fetchMerklOpportunitiesByIds = (ids) => __awaiter(void 0, void 0, void 0,
77
93
  });
78
94
  });
79
95
  exports.fetchMerklOpportunitiesByIds = fetchMerklOpportunitiesByIds;
96
+ const fetchMerklOpportunitiesByIdsCached = (0, memoizee_1.default)(exports.fetchMerklOpportunitiesByIds, {
97
+ promise: true,
98
+ maxAge: HISTORY_CACHE_TTL,
99
+ normalizer: ([ids]) => [...ids].sort().join(','),
100
+ });
80
101
  /**
81
102
  * Returns the reward APR Merkl recorded for the opportunity at the given unix timestamp (seconds),
82
103
  * or `undefined` when no campaign was computing for it then. Records are hourly buckets keyed by
83
- * their start, newest first, so the first bucket at or before the timestamp carries the value in
84
- * force at that moment; the window is a day wide to tolerate computation gaps.
104
+ * their start, newest first, so the bucket of the timestamp's hour carries the value in force at
105
+ * that moment; the window is a day wide to tolerate computation gaps. The window is aligned to the
106
+ * hour, so every block within one hour asks for (and caches) the same records.
85
107
  */
86
108
  const fetchMerklOpportunityAprAt = (opportunityId, timestamp) => __awaiter(void 0, void 0, void 0, function* () {
87
109
  var _a;
110
+ const to = Math.floor(timestamp / SECONDS_PER_HOUR) * SECONDS_PER_HOUR;
88
111
  const records = yield merklGet(`${MERKL_OPPORTUNITIES_URL}/${opportunityId}/apr-records`, {
89
112
  grain: 'HOUR',
90
- from: (timestamp - SECONDS_PER_DAY).toString(),
91
- to: timestamp.toString(),
113
+ from: (to - SECONDS_PER_DAY).toString(),
114
+ to: to.toString(),
92
115
  items: '1',
93
116
  }, `APR records for opportunity ${opportunityId}`);
94
117
  if (!Array.isArray(records))
@@ -96,6 +119,18 @@ const fetchMerklOpportunityAprAt = (opportunityId, timestamp) => __awaiter(void
96
119
  return (_a = records[0]) === null || _a === void 0 ? void 0 : _a.apr;
97
120
  });
98
121
  exports.fetchMerklOpportunityAprAt = fetchMerklOpportunityAprAt;
122
+ const fetchMerklOpportunityAprAtCached = (0, memoizee_1.default)(exports.fetchMerklOpportunityAprAt, {
123
+ promise: true,
124
+ maxAge: HISTORY_CACHE_TTL,
125
+ normalizer: ([opportunityId, timestamp]) => `${opportunityId}:${Math.floor(timestamp / SECONDS_PER_HOUR)}`,
126
+ });
127
+ /** Drops every cached historical response (campaign lists, opportunities, APR buckets). */
128
+ const clearMerklHistoryCache = () => {
129
+ fetchAllMerklCampaignsCached.clear();
130
+ fetchMerklOpportunitiesByIdsCached.clear();
131
+ fetchMerklOpportunityAprAtCached.clear();
132
+ };
133
+ exports.clearMerklHistoryCache = clearMerklHistoryCache;
99
134
  const mapWithConcurrency = (items, concurrency, fn) => __awaiter(void 0, void 0, void 0, function* () {
100
135
  const results = [];
101
136
  for (let i = 0; i < items.length; i += concurrency) {
@@ -106,10 +141,13 @@ const mapWithConcurrency = (items, concurrency, fn) => __awaiter(void 0, void 0,
106
141
  /**
107
142
  * Reconstructs the opportunities that were live at `timestamp` (unix seconds), shaped like the
108
143
  * live listing so the per-protocol reward mappers can consume either without knowing which they got:
109
- * 1. campaigns for the protocol (and chain, when the query names one) whose [start, end] covers
110
- * the timestamp identify the opportunities that were rewarded then — the live listing can't,
111
- * since campaigns live then are mostly PAST now and ones live now didn't exist then;
112
- * 2. those opportunities are loaded by id (works for PAST ones too) and narrowed to `query.type`;
144
+ * 1. every LIVE or PAST campaign for the protocol (and chain, when the query names one) is listed
145
+ * once per cache period — the list is block-independent, the exact window check happens here —
146
+ * and the ones whose [start, end] covers the timestamp identify the opportunities rewarded
147
+ * then (the live listing can't: campaigns live then are mostly PAST now and ones live now
148
+ * didn't exist then);
149
+ * 2. those opportunities are loaded by id (works for PAST ones too) and narrowed to `query.type`
150
+ * and `isRelevant`, so no APR history is fetched for opportunities the caller would discard;
113
151
  * 3. each one's `apr` is replaced with the value Merkl recorded at the timestamp, and ones with no
114
152
  * record then are dropped, since nothing was being computed for them.
115
153
  * `status` is forced to LIVE so status filters downstream keep working, `campaigns` carries the
@@ -117,15 +155,14 @@ const mapWithConcurrency = (items, concurrency, fn) => __awaiter(void 0, void 0,
117
155
  * when Merkl no longer serves one for the opportunity (it drops them once an opportunity is PAST).
118
156
  * `query.status` is ignored: it describes the present, not the requested moment.
119
157
  */
120
- const fetchMerklOpportunitiesAt = (query, timestamp) => __awaiter(void 0, void 0, void 0, function* () {
158
+ const fetchMerklOpportunitiesAt = (query_1, timestamp_1, ...args_1) => __awaiter(void 0, [query_1, timestamp_1, ...args_1], void 0, function* (query, timestamp, isRelevant = () => true) {
121
159
  var _a;
122
- const campaignQuery = { status: 'LIVE,PAST', endTimestamp: timestamp.toString() };
160
+ const campaignQuery = { status: 'LIVE,PAST' };
123
161
  if (query.mainProtocolId)
124
162
  campaignQuery.mainProtocolId = query.mainProtocolId;
125
163
  if (query.chainId)
126
164
  campaignQuery.chainId = query.chainId;
127
- // `endTimestamp` only narrows the listing upstream — the exact window check happens here
128
- const campaigns = (yield (0, exports.fetchAllMerklCampaigns)(campaignQuery))
165
+ const campaigns = (yield fetchAllMerklCampaignsCached(campaignQuery))
129
166
  .filter((c) => { var _a, _b; return c.opportunityId && +((_a = c.startTimestamp) !== null && _a !== void 0 ? _a : 0) <= timestamp && timestamp <= +((_b = c.endTimestamp) !== null && _b !== void 0 ? _b : 0); });
130
167
  const campaignsByOpportunity = {};
131
168
  campaigns.forEach((c) => {
@@ -137,9 +174,9 @@ const fetchMerklOpportunitiesAt = (query, timestamp) => __awaiter(void 0, void 0
137
174
  if (!opportunityIds.length)
138
175
  return [];
139
176
  const types = (_a = query.type) === null || _a === void 0 ? void 0 : _a.split(',').filter(Boolean);
140
- const opportunities = (yield (0, exports.fetchMerklOpportunitiesByIds)(opportunityIds))
141
- .filter((o) => !(types === null || types === void 0 ? void 0 : types.length) || types.includes(o.type));
142
- const aprs = yield mapWithConcurrency(opportunities, APR_RECORDS_CONCURRENCY, (o) => (0, exports.fetchMerklOpportunityAprAt)(o.id, timestamp));
177
+ const opportunities = (yield fetchMerklOpportunitiesByIdsCached(opportunityIds))
178
+ .filter((o) => (!(types === null || types === void 0 ? void 0 : types.length) || types.includes(o.type)) && isRelevant(o));
179
+ const aprs = yield mapWithConcurrency(opportunities, APR_RECORDS_CONCURRENCY, (o) => fetchMerklOpportunityAprAtCached(o.id, timestamp));
143
180
  return opportunities.flatMap((o, i) => {
144
181
  var _a, _b, _c;
145
182
  const apr = aprs[i];
@@ -154,9 +191,11 @@ const fetchMerklOpportunitiesAt = (query, timestamp) => __awaiter(void 0, void 0
154
191
  exports.fetchMerklOpportunitiesAt = fetchMerklOpportunitiesAt;
155
192
  /**
156
193
  * Live opportunities when no timestamp is given, otherwise the opportunities as they were at that
157
- * unix timestamp (seconds) — see `fetchMerklOpportunitiesAt`.
194
+ * unix timestamp (seconds) — see `fetchMerklOpportunitiesAt`. `isRelevant` only applies to the
195
+ * historical path, where it saves fetching APR history for opportunities the caller would discard;
196
+ * callers still filter the live listing themselves.
158
197
  */
159
- const fetchMerklOpportunities = (query, asOfTimestamp) => __awaiter(void 0, void 0, void 0, function* () {
160
- return (asOfTimestamp === undefined ? (0, exports.fetchAllMerklOpportunities)(query) : (0, exports.fetchMerklOpportunitiesAt)(query, asOfTimestamp));
198
+ const fetchMerklOpportunities = (query, asOfTimestamp, isRelevant) => __awaiter(void 0, void 0, void 0, function* () {
199
+ return (asOfTimestamp === undefined ? (0, exports.fetchAllMerklOpportunities)(query) : (0, exports.fetchMerklOpportunitiesAt)(query, asOfTimestamp, isRelevant));
161
200
  });
162
201
  exports.fetchMerklOpportunities = fetchMerklOpportunities;
package/esm/aaveV3/merkl.js CHANGED
@@ -44,7 +44,7 @@ export const getMerkleCampaigns = (chainId, asOfTimestamp) => __awaiter(void 0,
44
44
  mainProtocolId: 'aave',
45
45
  chainId: chainId.toString(),
46
46
  status: OpportunityStatus.LIVE,
47
- }, asOfTimestamp);
47
+ }, asOfTimestamp, (o) => { var _a; return !((_a = o.type) === null || _a === void 0 ? void 0 : _a.startsWith('AAVE_V4_')); });
48
48
  const relevantOpportunities = opportunities
49
49
  .filter((o) => o.chainId === chainId)
50
50
  .filter((o) => o.status === OpportunityStatus.LIVE);
package/esm/services/merkl.d.ts CHANGED
@@ -14,17 +14,23 @@ export declare const fetchMerklOpportunitiesByIds: (ids: string[]) => Promise<Me
14
14
  /**
15
15
  * Returns the reward APR Merkl recorded for the opportunity at the given unix timestamp (seconds),
16
16
  * or `undefined` when no campaign was computing for it then. Records are hourly buckets keyed by
17
- * their start, newest first, so the first bucket at or before the timestamp carries the value in
18
- * force at that moment; the window is a day wide to tolerate computation gaps.
17
+ * their start, newest first, so the bucket of the timestamp's hour carries the value in force at
18
+ * that moment; the window is a day wide to tolerate computation gaps. The window is aligned to the
19
+ * hour, so every block within one hour asks for (and caches) the same records.
19
20
  */
20
21
  export declare const fetchMerklOpportunityAprAt: (opportunityId: string, timestamp: number) => Promise<number | undefined>;
22
+ /** Drops every cached historical response (campaign lists, opportunities, APR buckets). */
23
+ export declare const clearMerklHistoryCache: () => void;
21
24
  /**
22
25
  * Reconstructs the opportunities that were live at `timestamp` (unix seconds), shaped like the
23
26
  * live listing so the per-protocol reward mappers can consume either without knowing which they got:
24
- * 1. campaigns for the protocol (and chain, when the query names one) whose [start, end] covers
25
- * the timestamp identify the opportunities that were rewarded then — the live listing can't,
26
- * since campaigns live then are mostly PAST now and ones live now didn't exist then;
27
- * 2. those opportunities are loaded by id (works for PAST ones too) and narrowed to `query.type`;
27
+ * 1. every LIVE or PAST campaign for the protocol (and chain, when the query names one) is listed
28
+ * once per cache period — the list is block-independent, the exact window check happens here —
29
+ * and the ones whose [start, end] covers the timestamp identify the opportunities rewarded
30
+ * then (the live listing can't: campaigns live then are mostly PAST now and ones live now
31
+ * didn't exist then);
32
+ * 2. those opportunities are loaded by id (works for PAST ones too) and narrowed to `query.type`
33
+ * and `isRelevant`, so no APR history is fetched for opportunities the caller would discard;
28
34
  * 3. each one's `apr` is replaced with the value Merkl recorded at the timestamp, and ones with no
29
35
  * record then are dropped, since nothing was being computed for them.
30
36
  * `status` is forced to LIVE so status filters downstream keep working, `campaigns` carries the
@@ -32,9 +38,11 @@ export declare const fetchMerklOpportunityAprAt: (opportunityId: string, timesta
32
38
  * when Merkl no longer serves one for the opportunity (it drops them once an opportunity is PAST).
33
39
  * `query.status` is ignored: it describes the present, not the requested moment.
34
40
  */
35
- export declare const fetchMerklOpportunitiesAt: (query: Record<string, string>, timestamp: number) => Promise<MerklOpportunity[]>;
41
+ export declare const fetchMerklOpportunitiesAt: (query: Record<string, string>, timestamp: number, isRelevant?: (opportunity: MerklOpportunity) => boolean) => Promise<MerklOpportunity[]>;
36
42
  /**
37
43
  * Live opportunities when no timestamp is given, otherwise the opportunities as they were at that
38
- * unix timestamp (seconds) — see `fetchMerklOpportunitiesAt`.
44
+ * unix timestamp (seconds) — see `fetchMerklOpportunitiesAt`. `isRelevant` only applies to the
45
+ * historical path, where it saves fetching APR history for opportunities the caller would discard;
46
+ * callers still filter the live listing themselves.
39
47
  */
40
- export declare const fetchMerklOpportunities: (query: Record<string, string>, asOfTimestamp?: number) => Promise<MerklOpportunity[]>;
48
+ export declare const fetchMerklOpportunities: (query: Record<string, string>, asOfTimestamp?: number, isRelevant?: (opportunity: MerklOpportunity) => boolean) => Promise<MerklOpportunity[]>;
package/esm/services/merkl.js CHANGED
@@ -7,6 +7,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
7
7
  step((generator = generator.apply(thisArg, _arguments || [])).next());
8
8
  });
9
9
  };
10
+ import memoize from 'memoizee';
10
11
  import { OpportunityStatus, } from '../types';
11
12
  import { LONGER_TIMEOUT } from './utils';
12
13
  const MERKL_API_URL = 'https://fe.defisaver.com/api/merkl';
@@ -19,11 +20,18 @@ const OPPORTUNITY_ID_BATCH_SIZE = 50;
19
20
  // APR history is fetched per opportunity, so historical lookups are throttled to keep a block with
20
21
  // many live campaigns from firing every request at once.
21
22
  const APR_RECORDS_CONCURRENCY = 10;
22
- const SECONDS_PER_DAY = 24 * 60 * 60;
23
+ const SECONDS_PER_HOUR = 60 * 60;
24
+ const SECONDS_PER_DAY = 24 * SECONDS_PER_HOUR;
25
+ // Historical lookups describe the past, so their responses are reused for a while: a job that
26
+ // walks many blocks (or fetches several protocols/chains at one block in parallel) would otherwise
27
+ // re-request the same campaign lists, opportunities and APR buckets for every block.
28
+ const HISTORY_CACHE_TTL = 5 * 60 * 1000;
23
29
  const merklGet = (url, query, label) => __awaiter(void 0, void 0, void 0, function* () {
24
30
  const res = yield fetch(`${url}?${new URLSearchParams(query)}`, { signal: AbortSignal.timeout(LONGER_TIMEOUT) });
25
- if (!res.ok)
26
- throw new Error(`Failed to fetch Merkl ${label}`);
31
+ if (!res.ok) {
32
+ const body = yield res.text().catch(() => '');
33
+ throw new Error(`Failed to fetch Merkl ${label} (HTTP ${res.status}${body ? `: ${body.slice(0, 200)}` : ''})`);
34
+ }
27
35
  return res.json();
28
36
  });
29
37
  /**
@@ -56,6 +64,11 @@ export const fetchAllMerklOpportunities = (query) => __awaiter(void 0, void 0, v
56
64
  export const fetchAllMerklCampaigns = (query) => __awaiter(void 0, void 0, void 0, function* () {
57
65
  return (fetchAllMerklPages(MERKL_CAMPAIGNS_URL, query, 'campaigns', MERKL_MAX_PAGE_ITEMS));
58
66
  });
67
+ const fetchAllMerklCampaignsCached = memoize(fetchAllMerklCampaigns, {
68
+ promise: true,
69
+ maxAge: HISTORY_CACHE_TTL,
70
+ normalizer: ([query]) => JSON.stringify(Object.entries(query).sort()),
71
+ });
59
72
  /**
60
73
  * Fetches the opportunities with the given Merkl-internal ids, regardless of their current status.
61
74
  * Ids Merkl no longer serves (hidden or test campaigns) are simply absent from the result.
@@ -71,24 +84,42 @@ export const fetchMerklOpportunitiesByIds = (ids) => __awaiter(void 0, void 0, v
71
84
  return data;
72
85
  });
73
86
  });
87
+ const fetchMerklOpportunitiesByIdsCached = memoize(fetchMerklOpportunitiesByIds, {
88
+ promise: true,
89
+ maxAge: HISTORY_CACHE_TTL,
90
+ normalizer: ([ids]) => [...ids].sort().join(','),
91
+ });
74
92
  /**
75
93
  * Returns the reward APR Merkl recorded for the opportunity at the given unix timestamp (seconds),
76
94
  * or `undefined` when no campaign was computing for it then. Records are hourly buckets keyed by
77
- * their start, newest first, so the first bucket at or before the timestamp carries the value in
78
- * force at that moment; the window is a day wide to tolerate computation gaps.
95
+ * their start, newest first, so the bucket of the timestamp's hour carries the value in force at
96
+ * that moment; the window is a day wide to tolerate computation gaps. The window is aligned to the
97
+ * hour, so every block within one hour asks for (and caches) the same records.
79
98
  */
80
99
  export const fetchMerklOpportunityAprAt = (opportunityId, timestamp) => __awaiter(void 0, void 0, void 0, function* () {
81
100
  var _a;
101
+ const to = Math.floor(timestamp / SECONDS_PER_HOUR) * SECONDS_PER_HOUR;
82
102
  const records = yield merklGet(`${MERKL_OPPORTUNITIES_URL}/${opportunityId}/apr-records`, {
83
103
  grain: 'HOUR',
84
- from: (timestamp - SECONDS_PER_DAY).toString(),
85
- to: timestamp.toString(),
104
+ from: (to - SECONDS_PER_DAY).toString(),
105
+ to: to.toString(),
86
106
  items: '1',
87
107
  }, `APR records for opportunity ${opportunityId}`);
88
108
  if (!Array.isArray(records))
89
109
  throw new Error(`Invalid Merkl APR records response for opportunity ${opportunityId}`);
90
110
  return (_a = records[0]) === null || _a === void 0 ? void 0 : _a.apr;
91
111
  });
112
+ const fetchMerklOpportunityAprAtCached = memoize(fetchMerklOpportunityAprAt, {
113
+ promise: true,
114
+ maxAge: HISTORY_CACHE_TTL,
115
+ normalizer: ([opportunityId, timestamp]) => `${opportunityId}:${Math.floor(timestamp / SECONDS_PER_HOUR)}`,
116
+ });
117
+ /** Drops every cached historical response (campaign lists, opportunities, APR buckets). */
118
+ export const clearMerklHistoryCache = () => {
119
+ fetchAllMerklCampaignsCached.clear();
120
+ fetchMerklOpportunitiesByIdsCached.clear();
121
+ fetchMerklOpportunityAprAtCached.clear();
122
+ };
92
123
  const mapWithConcurrency = (items, concurrency, fn) => __awaiter(void 0, void 0, void 0, function* () {
93
124
  const results = [];
94
125
  for (let i = 0; i < items.length; i += concurrency) {
@@ -99,10 +130,13 @@ const mapWithConcurrency = (items, concurrency, fn) => __awaiter(void 0, void 0,
99
130
  /**
100
131
  * Reconstructs the opportunities that were live at `timestamp` (unix seconds), shaped like the
101
132
  * live listing so the per-protocol reward mappers can consume either without knowing which they got:
102
- * 1. campaigns for the protocol (and chain, when the query names one) whose [start, end] covers
103
- * the timestamp identify the opportunities that were rewarded then — the live listing can't,
104
- * since campaigns live then are mostly PAST now and ones live now didn't exist then;
105
- * 2. those opportunities are loaded by id (works for PAST ones too) and narrowed to `query.type`;
133
+ * 1. every LIVE or PAST campaign for the protocol (and chain, when the query names one) is listed
134
+ * once per cache period — the list is block-independent, the exact window check happens here —
135
+ * and the ones whose [start, end] covers the timestamp identify the opportunities rewarded
136
+ * then (the live listing can't: campaigns live then are mostly PAST now and ones live now
137
+ * didn't exist then);
138
+ * 2. those opportunities are loaded by id (works for PAST ones too) and narrowed to `query.type`
139
+ * and `isRelevant`, so no APR history is fetched for opportunities the caller would discard;
106
140
  * 3. each one's `apr` is replaced with the value Merkl recorded at the timestamp, and ones with no
107
141
  * record then are dropped, since nothing was being computed for them.
108
142
  * `status` is forced to LIVE so status filters downstream keep working, `campaigns` carries the
@@ -110,15 +144,14 @@ const mapWithConcurrency = (items, concurrency, fn) => __awaiter(void 0, void 0,
110
144
  * when Merkl no longer serves one for the opportunity (it drops them once an opportunity is PAST).
111
145
  * `query.status` is ignored: it describes the present, not the requested moment.
112
146
  */
113
- export const fetchMerklOpportunitiesAt = (query, timestamp) => __awaiter(void 0, void 0, void 0, function* () {
147
+ export const fetchMerklOpportunitiesAt = (query_1, timestamp_1, ...args_1) => __awaiter(void 0, [query_1, timestamp_1, ...args_1], void 0, function* (query, timestamp, isRelevant = () => true) {
114
148
  var _a;
115
- const campaignQuery = { status: 'LIVE,PAST', endTimestamp: timestamp.toString() };
149
+ const campaignQuery = { status: 'LIVE,PAST' };
116
150
  if (query.mainProtocolId)
117
151
  campaignQuery.mainProtocolId = query.mainProtocolId;
118
152
  if (query.chainId)
119
153
  campaignQuery.chainId = query.chainId;
120
- // `endTimestamp` only narrows the listing upstream — the exact window check happens here
121
- const campaigns = (yield fetchAllMerklCampaigns(campaignQuery))
154
+ const campaigns = (yield fetchAllMerklCampaignsCached(campaignQuery))
122
155
  .filter((c) => { var _a, _b; return c.opportunityId && +((_a = c.startTimestamp) !== null && _a !== void 0 ? _a : 0) <= timestamp && timestamp <= +((_b = c.endTimestamp) !== null && _b !== void 0 ? _b : 0); });
123
156
  const campaignsByOpportunity = {};
124
157
  campaigns.forEach((c) => {
@@ -130,9 +163,9 @@ export const fetchMerklOpportunitiesAt = (query, timestamp) => __awaiter(void 0,
130
163
  if (!opportunityIds.length)
131
164
  return [];
132
165
  const types = (_a = query.type) === null || _a === void 0 ? void 0 : _a.split(',').filter(Boolean);
133
- const opportunities = (yield fetchMerklOpportunitiesByIds(opportunityIds))
134
- .filter((o) => !(types === null || types === void 0 ? void 0 : types.length) || types.includes(o.type));
135
- const aprs = yield mapWithConcurrency(opportunities, APR_RECORDS_CONCURRENCY, (o) => fetchMerklOpportunityAprAt(o.id, timestamp));
166
+ const opportunities = (yield fetchMerklOpportunitiesByIdsCached(opportunityIds))
167
+ .filter((o) => (!(types === null || types === void 0 ? void 0 : types.length) || types.includes(o.type)) && isRelevant(o));
168
+ const aprs = yield mapWithConcurrency(opportunities, APR_RECORDS_CONCURRENCY, (o) => fetchMerklOpportunityAprAtCached(o.id, timestamp));
136
169
  return opportunities.flatMap((o, i) => {
137
170
  var _a, _b, _c;
138
171
  const apr = aprs[i];
@@ -146,8 +179,10 @@ export const fetchMerklOpportunitiesAt = (query, timestamp) => __awaiter(void 0,
146
179
  });
147
180
  /**
148
181
  * Live opportunities when no timestamp is given, otherwise the opportunities as they were at that
149
- * unix timestamp (seconds) — see `fetchMerklOpportunitiesAt`.
182
+ * unix timestamp (seconds) — see `fetchMerklOpportunitiesAt`. `isRelevant` only applies to the
183
+ * historical path, where it saves fetching APR history for opportunities the caller would discard;
184
+ * callers still filter the live listing themselves.
150
185
  */
151
- export const fetchMerklOpportunities = (query, asOfTimestamp) => __awaiter(void 0, void 0, void 0, function* () {
152
- return (asOfTimestamp === undefined ? fetchAllMerklOpportunities(query) : fetchMerklOpportunitiesAt(query, asOfTimestamp));
186
+ export const fetchMerklOpportunities = (query, asOfTimestamp, isRelevant) => __awaiter(void 0, void 0, void 0, function* () {
187
+ return (asOfTimestamp === undefined ? fetchAllMerklOpportunities(query) : fetchMerklOpportunitiesAt(query, asOfTimestamp, isRelevant));
153
188
  });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@defisaver/positions-sdk",
3
- "version": "2.1.169-custom-fetching-3-dev",
3
+ "version": "2.1.169-custom-fetching-4-dev",
4
4
  "description": "",
5
5
  "main": "./cjs/index.js",
6
6
  "module": "./esm/index.js",
package/src/aaveV3/merkl.ts CHANGED
@@ -40,7 +40,7 @@ export const getMerkleCampaigns = async (chainId: NetworkNumber, asOfTimestamp?:
40
40
  mainProtocolId: 'aave',
41
41
  chainId: chainId.toString(),
42
42
  status: OpportunityStatus.LIVE,
43
- }, asOfTimestamp);
43
+ }, asOfTimestamp, (o) => !o.type?.startsWith('AAVE_V4_'));
44
44
  const relevantOpportunities = opportunities
45
45
  .filter((o: MerklOpportunity) => o.chainId === chainId)
46
46
  .filter((o: MerklOpportunity) => o.status === OpportunityStatus.LIVE);
package/src/services/merkl.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import memoize from 'memoizee';
1
2
  import {
2
3
  MerklAprRecord, MerklCampaignListItem, MerklOpportunity, OpportunityStatus,
3
4
  } from '../types';
@@ -14,11 +15,19 @@ const OPPORTUNITY_ID_BATCH_SIZE = 50;
14
15
  // APR history is fetched per opportunity, so historical lookups are throttled to keep a block with
15
16
  // many live campaigns from firing every request at once.
16
17
  const APR_RECORDS_CONCURRENCY = 10;
17
- const SECONDS_PER_DAY = 24 * 60 * 60;
18
+ const SECONDS_PER_HOUR = 60 * 60;
19
+ const SECONDS_PER_DAY = 24 * SECONDS_PER_HOUR;
20
+ // Historical lookups describe the past, so their responses are reused for a while: a job that
21
+ // walks many blocks (or fetches several protocols/chains at one block in parallel) would otherwise
22
+ // re-request the same campaign lists, opportunities and APR buckets for every block.
23
+ const HISTORY_CACHE_TTL = 5 * 60 * 1000;
18
24
 
19
25
  const merklGet = async <T>(url: string, query: Record<string, string>, label: string): Promise<T> => {
20
26
  const res = await fetch(`${url}?${new URLSearchParams(query)}`, { signal: AbortSignal.timeout(LONGER_TIMEOUT) });
21
- if (!res.ok) throw new Error(`Failed to fetch Merkl ${label}`);
27
+ if (!res.ok) {
28
+ const body = await res.text().catch(() => '');
29
+ throw new Error(`Failed to fetch Merkl ${label} (HTTP ${res.status}${body ? `: ${body.slice(0, 200)}` : ''})`);
30
+ }
22
31
  return res.json();
23
32
  };
24
33
 
@@ -57,6 +66,12 @@ export const fetchAllMerklCampaigns = async (query: Record<string, string>): Pro
57
66
  fetchAllMerklPages<MerklCampaignListItem>(MERKL_CAMPAIGNS_URL, query, 'campaigns', MERKL_MAX_PAGE_ITEMS)
58
67
  );
59
68
 
69
+ const fetchAllMerklCampaignsCached = memoize(fetchAllMerklCampaigns, {
70
+ promise: true,
71
+ maxAge: HISTORY_CACHE_TTL,
72
+ normalizer: ([query]: [Record<string, string>]) => JSON.stringify(Object.entries(query).sort()),
73
+ });
74
+
60
75
  /**
61
76
  * Fetches the opportunities with the given Merkl-internal ids, regardless of their current status.
62
77
  * Ids Merkl no longer serves (hidden or test campaigns) are simply absent from the result.
@@ -75,23 +90,44 @@ export const fetchMerklOpportunitiesByIds = async (ids: string[]): Promise<Merkl
75
90
  });
76
91
  };
77
92
 
93
+ const fetchMerklOpportunitiesByIdsCached = memoize(fetchMerklOpportunitiesByIds, {
94
+ promise: true,
95
+ maxAge: HISTORY_CACHE_TTL,
96
+ normalizer: ([ids]: [string[]]) => [...ids].sort().join(','),
97
+ });
98
+
78
99
  /**
79
100
  * Returns the reward APR Merkl recorded for the opportunity at the given unix timestamp (seconds),
80
101
  * or `undefined` when no campaign was computing for it then. Records are hourly buckets keyed by
81
- * their start, newest first, so the first bucket at or before the timestamp carries the value in
82
- * force at that moment; the window is a day wide to tolerate computation gaps.
102
+ * their start, newest first, so the bucket of the timestamp's hour carries the value in force at
103
+ * that moment; the window is a day wide to tolerate computation gaps. The window is aligned to the
104
+ * hour, so every block within one hour asks for (and caches) the same records.
83
105
  */
84
106
  export const fetchMerklOpportunityAprAt = async (opportunityId: string, timestamp: number): Promise<number | undefined> => {
107
+ const to = Math.floor(timestamp / SECONDS_PER_HOUR) * SECONDS_PER_HOUR;
85
108
  const records = await merklGet<unknown>(`${MERKL_OPPORTUNITIES_URL}/${opportunityId}/apr-records`, {
86
109
  grain: 'HOUR',
87
- from: (timestamp - SECONDS_PER_DAY).toString(),
88
- to: timestamp.toString(),
110
+ from: (to - SECONDS_PER_DAY).toString(),
111
+ to: to.toString(),
89
112
  items: '1',
90
113
  }, `APR records for opportunity ${opportunityId}`);
91
114
  if (!Array.isArray(records)) throw new Error(`Invalid Merkl APR records response for opportunity ${opportunityId}`);
92
115
  return (records as MerklAprRecord[])[0]?.apr;
93
116
  };
94
117
 
118
+ const fetchMerklOpportunityAprAtCached = memoize(fetchMerklOpportunityAprAt, {
119
+ promise: true,
120
+ maxAge: HISTORY_CACHE_TTL,
121
+ normalizer: ([opportunityId, timestamp]: [string, number]) => `${opportunityId}:${Math.floor(timestamp / SECONDS_PER_HOUR)}`,
122
+ });
123
+
124
+ /** Drops every cached historical response (campaign lists, opportunities, APR buckets). */
125
+ export const clearMerklHistoryCache = () => {
126
+ fetchAllMerklCampaignsCached.clear();
127
+ fetchMerklOpportunitiesByIdsCached.clear();
128
+ fetchMerklOpportunityAprAtCached.clear();
129
+ };
130
+
95
131
  const mapWithConcurrency = async <T, R>(items: T[], concurrency: number, fn: (item: T) => Promise<R>): Promise<R[]> => {
96
132
  const results: R[] = [];
97
133
  for (let i = 0; i < items.length; i += concurrency) {
@@ -103,10 +139,13 @@ const mapWithConcurrency = async <T, R>(items: T[], concurrency: number, fn: (it
103
139
  /**
104
140
  * Reconstructs the opportunities that were live at `timestamp` (unix seconds), shaped like the
105
141
  * live listing so the per-protocol reward mappers can consume either without knowing which they got:
106
- * 1. campaigns for the protocol (and chain, when the query names one) whose [start, end] covers
107
- * the timestamp identify the opportunities that were rewarded then — the live listing can't,
108
- * since campaigns live then are mostly PAST now and ones live now didn't exist then;
109
- * 2. those opportunities are loaded by id (works for PAST ones too) and narrowed to `query.type`;
142
+ * 1. every LIVE or PAST campaign for the protocol (and chain, when the query names one) is listed
143
+ * once per cache period — the list is block-independent, the exact window check happens here —
144
+ * and the ones whose [start, end] covers the timestamp identify the opportunities rewarded
145
+ * then (the live listing can't: campaigns live then are mostly PAST now and ones live now
146
+ * didn't exist then);
147
+ * 2. those opportunities are loaded by id (works for PAST ones too) and narrowed to `query.type`
148
+ * and `isRelevant`, so no APR history is fetched for opportunities the caller would discard;
110
149
  * 3. each one's `apr` is replaced with the value Merkl recorded at the timestamp, and ones with no
111
150
  * record then are dropped, since nothing was being computed for them.
112
151
  * `status` is forced to LIVE so status filters downstream keep working, `campaigns` carries the
@@ -114,13 +153,16 @@ const mapWithConcurrency = async <T, R>(items: T[], concurrency: number, fn: (it
114
153
  * when Merkl no longer serves one for the opportunity (it drops them once an opportunity is PAST).
115
154
  * `query.status` is ignored: it describes the present, not the requested moment.
116
155
  */
117
- export const fetchMerklOpportunitiesAt = async (query: Record<string, string>, timestamp: number): Promise<MerklOpportunity[]> => {
118
- const campaignQuery: Record<string, string> = { status: 'LIVE,PAST', endTimestamp: timestamp.toString() };
156
+ export const fetchMerklOpportunitiesAt = async (
157
+ query: Record<string, string>,
158
+ timestamp: number,
159
+ isRelevant: (opportunity: MerklOpportunity) => boolean = () => true,
160
+ ): Promise<MerklOpportunity[]> => {
161
+ const campaignQuery: Record<string, string> = { status: 'LIVE,PAST' };
119
162
  if (query.mainProtocolId) campaignQuery.mainProtocolId = query.mainProtocolId;
120
163
  if (query.chainId) campaignQuery.chainId = query.chainId;
121
164
 
122
- // `endTimestamp` only narrows the listing upstream — the exact window check happens here
123
- const campaigns = (await fetchAllMerklCampaigns(campaignQuery))
165
+ const campaigns = (await fetchAllMerklCampaignsCached(campaignQuery))
124
166
  .filter((c) => c.opportunityId && +(c.startTimestamp ?? 0) <= timestamp && timestamp <= +(c.endTimestamp ?? 0));
125
167
  const campaignsByOpportunity: Record<string, MerklCampaignListItem[]> = {};
126
168
  campaigns.forEach((c) => {
@@ -131,10 +173,10 @@ export const fetchMerklOpportunitiesAt = async (query: Record<string, string>, t
131
173
  if (!opportunityIds.length) return [];
132
174
 
133
175
  const types = query.type?.split(',').filter(Boolean);
134
- const opportunities = (await fetchMerklOpportunitiesByIds(opportunityIds))
135
- .filter((o) => !types?.length || types.includes(o.type));
176
+ const opportunities = (await fetchMerklOpportunitiesByIdsCached(opportunityIds))
177
+ .filter((o) => (!types?.length || types.includes(o.type)) && isRelevant(o));
136
178
 
137
- const aprs = await mapWithConcurrency(opportunities, APR_RECORDS_CONCURRENCY, (o) => fetchMerklOpportunityAprAt(o.id, timestamp));
179
+ const aprs = await mapWithConcurrency(opportunities, APR_RECORDS_CONCURRENCY, (o) => fetchMerklOpportunityAprAtCached(o.id, timestamp));
138
180
 
139
181
  return opportunities.flatMap((o, i) => {
140
182
  const apr = aprs[i];
@@ -157,8 +199,14 @@ export const fetchMerklOpportunitiesAt = async (query: Record<string, string>, t
157
199
 
158
200
  /**
159
201
  * Live opportunities when no timestamp is given, otherwise the opportunities as they were at that
160
- * unix timestamp (seconds) — see `fetchMerklOpportunitiesAt`.
202
+ * unix timestamp (seconds) — see `fetchMerklOpportunitiesAt`. `isRelevant` only applies to the
203
+ * historical path, where it saves fetching APR history for opportunities the caller would discard;
204
+ * callers still filter the live listing themselves.
161
205
  */
162
- export const fetchMerklOpportunities = async (query: Record<string, string>, asOfTimestamp?: number): Promise<MerklOpportunity[]> => (
163
- asOfTimestamp === undefined ? fetchAllMerklOpportunities(query) : fetchMerklOpportunitiesAt(query, asOfTimestamp)
206
+ export const fetchMerklOpportunities = async (
207
+ query: Record<string, string>,
208
+ asOfTimestamp?: number,
209
+ isRelevant?: (opportunity: MerklOpportunity) => boolean,
210
+ ): Promise<MerklOpportunity[]> => (
211
+ asOfTimestamp === undefined ? fetchAllMerklOpportunities(query) : fetchMerklOpportunitiesAt(query, asOfTimestamp, isRelevant)
164
212
  );