@chainlink/external-adapter-framework 0.3.7 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/README.md +102 -0
  2. package/package.json +42 -50
  3. package/adapter/basic.d.ts +0 -86
  4. package/adapter/basic.js +0 -323
  5. package/adapter/basic.js.map +0 -1
  6. package/adapter/endpoint.d.ts +0 -17
  7. package/adapter/endpoint.js +0 -19
  8. package/adapter/endpoint.js.map +0 -1
  9. package/adapter/index.d.ts +0 -4
  10. package/adapter/index.js +0 -21
  11. package/adapter/index.js.map +0 -1
  12. package/adapter/price.d.ts +0 -75
  13. package/adapter/price.js +0 -88
  14. package/adapter/price.js.map +0 -1
  15. package/adapter/types.d.ts +0 -118
  16. package/adapter/types.js +0 -3
  17. package/adapter/types.js.map +0 -1
  18. package/background-executor.d.ts +0 -9
  19. package/background-executor.js +0 -97
  20. package/background-executor.js.map +0 -1
  21. package/cache/factory.d.ts +0 -6
  22. package/cache/factory.js +0 -24
  23. package/cache/factory.js.map +0 -1
  24. package/cache/index.d.ts +0 -87
  25. package/cache/index.js +0 -133
  26. package/cache/index.js.map +0 -1
  27. package/cache/local.d.ts +0 -23
  28. package/cache/local.js +0 -84
  29. package/cache/local.js.map +0 -1
  30. package/cache/metrics.d.ts +0 -27
  31. package/cache/metrics.js +0 -121
  32. package/cache/metrics.js.map +0 -1
  33. package/cache/redis.d.ts +0 -16
  34. package/cache/redis.js +0 -101
  35. package/cache/redis.js.map +0 -1
  36. package/config/index.d.ts +0 -297
  37. package/config/index.js +0 -353
  38. package/config/index.js.map +0 -1
  39. package/config/provider-limits.d.ts +0 -27
  40. package/config/provider-limits.js +0 -75
  41. package/config/provider-limits.js.map +0 -1
  42. package/examples/bank-frick/accounts.d.ts +0 -52
  43. package/examples/bank-frick/accounts.js +0 -190
  44. package/examples/bank-frick/accounts.js.map +0 -1
  45. package/examples/bank-frick/config/index.d.ts +0 -17
  46. package/examples/bank-frick/config/index.js +0 -55
  47. package/examples/bank-frick/config/index.js.map +0 -1
  48. package/examples/bank-frick/index.d.ts +0 -16
  49. package/examples/bank-frick/index.js +0 -16
  50. package/examples/bank-frick/index.js.map +0 -1
  51. package/examples/bank-frick/util.d.ts +0 -4
  52. package/examples/bank-frick/util.js +0 -40
  53. package/examples/bank-frick/util.js.map +0 -1
  54. package/examples/coingecko/src/config/index.d.ts +0 -2
  55. package/examples/coingecko/src/config/index.js +0 -6
  56. package/examples/coingecko/src/config/index.js.map +0 -1
  57. package/examples/coingecko/src/config/overrides.json +0 -10825
  58. package/examples/coingecko/src/crypto-utils.d.ts +0 -48
  59. package/examples/coingecko/src/crypto-utils.js +0 -60
  60. package/examples/coingecko/src/crypto-utils.js.map +0 -1
  61. package/examples/coingecko/src/endpoint/coins.d.ts +0 -9
  62. package/examples/coingecko/src/endpoint/coins.js +0 -37
  63. package/examples/coingecko/src/endpoint/coins.js.map +0 -1
  64. package/examples/coingecko/src/endpoint/crypto-marketcap.d.ts +0 -3
  65. package/examples/coingecko/src/endpoint/crypto-marketcap.js +0 -30
  66. package/examples/coingecko/src/endpoint/crypto-marketcap.js.map +0 -1
  67. package/examples/coingecko/src/endpoint/crypto-volume.d.ts +0 -3
  68. package/examples/coingecko/src/endpoint/crypto-volume.js +0 -30
  69. package/examples/coingecko/src/endpoint/crypto-volume.js.map +0 -1
  70. package/examples/coingecko/src/endpoint/crypto.d.ts +0 -2
  71. package/examples/coingecko/src/endpoint/crypto.js +0 -28
  72. package/examples/coingecko/src/endpoint/crypto.js.map +0 -1
  73. package/examples/coingecko/src/endpoint/dominance.d.ts +0 -3
  74. package/examples/coingecko/src/endpoint/dominance.js +0 -28
  75. package/examples/coingecko/src/endpoint/dominance.js.map +0 -1
  76. package/examples/coingecko/src/endpoint/global-marketcap.d.ts +0 -3
  77. package/examples/coingecko/src/endpoint/global-marketcap.js +0 -28
  78. package/examples/coingecko/src/endpoint/global-marketcap.js.map +0 -1
  79. package/examples/coingecko/src/endpoint/index.d.ts +0 -6
  80. package/examples/coingecko/src/endpoint/index.js +0 -16
  81. package/examples/coingecko/src/endpoint/index.js.map +0 -1
  82. package/examples/coingecko/src/global-utils.d.ts +0 -27
  83. package/examples/coingecko/src/global-utils.js +0 -47
  84. package/examples/coingecko/src/global-utils.js.map +0 -1
  85. package/examples/coingecko/src/index.d.ts +0 -4
  86. package/examples/coingecko/src/index.js +0 -19
  87. package/examples/coingecko/src/index.js.map +0 -1
  88. package/examples/coingecko-old/batch-warming.d.ts +0 -7
  89. package/examples/coingecko-old/batch-warming.js +0 -54
  90. package/examples/coingecko-old/batch-warming.js.map +0 -1
  91. package/examples/coingecko-old/index.d.ts +0 -2
  92. package/examples/coingecko-old/index.js +0 -12
  93. package/examples/coingecko-old/index.js.map +0 -1
  94. package/examples/coingecko-old/rest.d.ts +0 -12
  95. package/examples/coingecko-old/rest.js +0 -55
  96. package/examples/coingecko-old/rest.js.map +0 -1
  97. package/examples/cryptocompare/src/config/index.d.ts +0 -2
  98. package/examples/cryptocompare/src/config/index.js +0 -6
  99. package/examples/cryptocompare/src/config/index.js.map +0 -1
  100. package/examples/cryptocompare/src/endpoints/crypto.d.ts +0 -28
  101. package/examples/cryptocompare/src/endpoints/crypto.js +0 -54
  102. package/examples/cryptocompare/src/endpoints/crypto.js.map +0 -1
  103. package/examples/cryptocompare/src/endpoints/index.d.ts +0 -1
  104. package/examples/cryptocompare/src/endpoints/index.js +0 -6
  105. package/examples/cryptocompare/src/endpoints/index.js.map +0 -1
  106. package/examples/cryptocompare/src/index.d.ts +0 -4
  107. package/examples/cryptocompare/src/index.js +0 -14
  108. package/examples/cryptocompare/src/index.js.map +0 -1
  109. package/examples/genesis/config/index.d.ts +0 -7
  110. package/examples/genesis/config/index.js +0 -11
  111. package/examples/genesis/config/index.js.map +0 -1
  112. package/examples/genesis/index.d.ts +0 -2
  113. package/examples/genesis/index.js +0 -13
  114. package/examples/genesis/index.js.map +0 -1
  115. package/examples/genesis/sseStream.d.ts +0 -16
  116. package/examples/genesis/sseStream.js +0 -149
  117. package/examples/genesis/sseStream.js.map +0 -1
  118. package/examples/ncfx/config/index.d.ts +0 -12
  119. package/examples/ncfx/config/index.js +0 -16
  120. package/examples/ncfx/config/index.js.map +0 -1
  121. package/examples/ncfx/index.d.ts +0 -13
  122. package/examples/ncfx/index.js +0 -12
  123. package/examples/ncfx/index.js.map +0 -1
  124. package/examples/ncfx/websocket.d.ts +0 -47
  125. package/examples/ncfx/websocket.js +0 -74
  126. package/examples/ncfx/websocket.js.map +0 -1
  127. package/index.d.ts +0 -17
  128. package/index.js +0 -174
  129. package/index.js.map +0 -1
  130. package/metrics/constants.d.ts +0 -16
  131. package/metrics/constants.js +0 -26
  132. package/metrics/constants.js.map +0 -1
  133. package/metrics/index.d.ts +0 -19
  134. package/metrics/index.js +0 -139
  135. package/metrics/index.js.map +0 -1
  136. package/metrics/util.d.ts +0 -7
  137. package/metrics/util.js +0 -10
  138. package/metrics/util.js.map +0 -1
  139. package/rate-limiting/background/fixed-frequency.d.ts +0 -11
  140. package/rate-limiting/background/fixed-frequency.js +0 -36
  141. package/rate-limiting/background/fixed-frequency.js.map +0 -1
  142. package/rate-limiting/index.d.ts +0 -56
  143. package/rate-limiting/index.js +0 -86
  144. package/rate-limiting/index.js.map +0 -1
  145. package/rate-limiting/metrics.d.ts +0 -3
  146. package/rate-limiting/metrics.js +0 -45
  147. package/rate-limiting/metrics.js.map +0 -1
  148. package/rate-limiting/request/simple-counting.d.ts +0 -21
  149. package/rate-limiting/request/simple-counting.js +0 -63
  150. package/rate-limiting/request/simple-counting.js.map +0 -1
  151. package/transports/batch-warming.d.ts +0 -35
  152. package/transports/batch-warming.js +0 -138
  153. package/transports/batch-warming.js.map +0 -1
  154. package/transports/index.d.ts +0 -87
  155. package/transports/index.js +0 -56
  156. package/transports/index.js.map +0 -1
  157. package/transports/metrics.d.ts +0 -25
  158. package/transports/metrics.js +0 -122
  159. package/transports/metrics.js.map +0 -1
  160. package/transports/rest.d.ts +0 -51
  161. package/transports/rest.js +0 -133
  162. package/transports/rest.js.map +0 -1
  163. package/transports/routing.d.ts +0 -23
  164. package/transports/routing.js +0 -52
  165. package/transports/routing.js.map +0 -1
  166. package/transports/sse.d.ts +0 -40
  167. package/transports/sse.js +0 -92
  168. package/transports/sse.js.map +0 -1
  169. package/transports/util.d.ts +0 -9
  170. package/transports/util.js +0 -87
  171. package/transports/util.js.map +0 -1
  172. package/transports/websocket.d.ts +0 -81
  173. package/transports/websocket.js +0 -165
  174. package/transports/websocket.js.map +0 -1
  175. package/util/censor/censor-list.d.ts +0 -9
  176. package/util/censor/censor-list.js +0 -13
  177. package/util/censor/censor-list.js.map +0 -1
  178. package/util/index.d.ts +0 -13
  179. package/util/index.js +0 -38
  180. package/util/index.js.map +0 -1
  181. package/util/logger.d.ts +0 -49
  182. package/util/logger.js +0 -123
  183. package/util/logger.js.map +0 -1
  184. package/util/request.d.ts +0 -102
  185. package/util/request.js +0 -3
  186. package/util/request.js.map +0 -1
  187. package/util/subscription-set/expiring-sorted-set.d.ts +0 -21
  188. package/util/subscription-set/expiring-sorted-set.js +0 -36
  189. package/util/subscription-set/expiring-sorted-set.js.map +0 -1
  190. package/util/subscription-set/redis-sorted-set.d.ts +0 -9
  191. package/util/subscription-set/redis-sorted-set.js +0 -29
  192. package/util/subscription-set/redis-sorted-set.js.map +0 -1
  193. package/util/subscription-set/subscription-set.d.ts +0 -19
  194. package/util/subscription-set/subscription-set.js +0 -28
  195. package/util/subscription-set/subscription-set.js.map +0 -1
  196. package/util/test-payload-loader.d.ts +0 -26
  197. package/util/test-payload-loader.js +0 -85
  198. package/util/test-payload-loader.js.map +0 -1
  199. package/validation/error.d.ts +0 -48
  200. package/validation/error.js +0 -78
  201. package/validation/error.js.map +0 -1
  202. package/validation/index.d.ts +0 -5
  203. package/validation/index.js +0 -91
  204. package/validation/index.js.map +0 -1
  205. package/validation/input-params.d.ts +0 -14
  206. package/validation/input-params.js +0 -3
  207. package/validation/input-params.js.map +0 -1
  208. package/validation/input-validator.d.ts +0 -16
  209. package/validation/input-validator.js +0 -123
  210. package/validation/input-validator.js.map +0 -1
@@ -1,45 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.rateLimitCreditsSpentTotal = exports.retrieveCost = void 0;
27
- const client = __importStar(require("prom-client"));
28
- // Retrieve cost field from response if exists
29
- // If not return default cost of 1
30
- const retrieveCost = (data) => {
31
- const cost = data['cost'];
32
- if (typeof cost === 'number' || typeof cost === 'string') {
33
- return Number(cost);
34
- }
35
- else {
36
- return 1;
37
- }
38
- };
39
- exports.retrieveCost = retrieveCost;
40
- exports.rateLimitCreditsSpentTotal = new client.Counter({
41
- name: 'rate_limit_credits_spent_total',
42
- help: 'The number of data provider credits the adapter is consuming',
43
- labelNames: ['participant_id', 'feed_id'],
44
- });
45
- //# sourceMappingURL=metrics.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/rate-limiting/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AAErC,8CAA8C;AAC9C,kCAAkC;AAC3B,MAAM,YAAY,GAAG,CAAuB,IAA0B,EAAU,EAAE;IACvF,MAAM,IAAI,GAAI,IAAgC,CAAC,MAAM,CAAC,CAAA;IACtD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;QACxD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAA;KACpB;SAAM;QACL,OAAO,CAAC,CAAA;KACT;AACH,CAAC,CAAA;AAPY,QAAA,YAAY,gBAOxB;AAEY,QAAA,0BAA0B,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC3D,IAAI,EAAE,gCAAgC;IACtC,IAAI,EAAE,8DAA8D;IACpE,UAAU,EAAE,CAAC,gBAAgB,EAAE,SAAS,CAAU;CACnD,CAAC,CAAA"}
@@ -1,21 +0,0 @@
1
- import { AdapterRateLimitTier, RateLimiter } from '..';
2
- import { AdapterEndpoint } from '../../adapter';
3
- import { SettingsMap } from '../../config';
4
- /**
5
- * This rate limiter is the simplest stateful option.
6
- * On startup, it'll compare the different thresholds for each tier, calculate them all
7
- * in the finest window we'll use (seconds), and use the most restrictive one.
8
- * This is so if the EA were to restart, we don't need to worry about persisting state
9
- * for things like daily quotas. The downside is that this does not work well for bursty
10
- * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
11
- */
12
- export declare class SimpleCountingRateLimiter implements RateLimiter {
13
- latestSecondInterval: number;
14
- requestsThisSecond: number;
15
- latestMinuteInterval: number;
16
- requestsThisMinute: number;
17
- perSecondLimit: number;
18
- perMinuteLimit: number;
19
- initialize<Params, Result, CustomSettings extends SettingsMap>(endpoints: AdapterEndpoint<Params, Result, CustomSettings>[], limits?: AdapterRateLimitTier): this;
20
- isUnderLimits(): boolean;
21
- }
@@ -1,63 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SimpleCountingRateLimiter = void 0;
4
- const util_1 = require("../../util");
5
- const logger = (0, util_1.makeLogger)('SimpleCountingRateLimiter');
6
- /**
7
- * This rate limiter is the simplest stateful option.
8
- * On startup, it'll compare the different thresholds for each tier, calculate them all
9
- * in the finest window we'll use (seconds), and use the most restrictive one.
10
- * This is so if the EA were to restart, we don't need to worry about persisting state
11
- * for things like daily quotas. The downside is that this does not work well for bursty
12
- * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
13
- */
14
- class SimpleCountingRateLimiter {
15
- constructor() {
16
- this.latestSecondInterval = 0;
17
- this.requestsThisSecond = 0;
18
- this.latestMinuteInterval = 0;
19
- this.requestsThisMinute = 0;
20
- }
21
- initialize(endpoints, limits) {
22
- // Translate the hourly limit into reqs per minute
23
- const perHourLimit = (limits?.rateLimit1h || Infinity) / 60;
24
- this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit);
25
- this.perSecondLimit = limits?.rateLimit1s || Infinity;
26
- logger.debug(`Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`);
27
- return this;
28
- }
29
- isUnderLimits() {
30
- // If the limit is set to infinity, there was no tier limit specified
31
- if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
32
- return true;
33
- }
34
- const now = Date.now();
35
- const nearestSecondInterval = Math.floor(now / 1000);
36
- const nearestMinuteInterval = Math.floor(now / (1000 * 60));
37
- // This should always run to completion, even if it doesn't look atomic; therefore the
38
- // Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
39
- if (nearestSecondInterval !== this.latestSecondInterval) {
40
- logger.trace(`Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `);
41
- this.latestSecondInterval = nearestSecondInterval;
42
- this.requestsThisSecond = 0;
43
- }
44
- if (nearestMinuteInterval !== this.latestMinuteInterval) {
45
- logger.trace(`Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `);
46
- this.latestMinuteInterval = nearestMinuteInterval;
47
- this.requestsThisMinute = 0;
48
- }
49
- if (this.requestsThisSecond < this.perSecondLimit &&
50
- this.requestsThisMinute < this.perMinuteLimit) {
51
- logger.trace('Request under limits, counting +1');
52
- this.requestsThisSecond++;
53
- this.requestsThisMinute++;
54
- return true;
55
- }
56
- else {
57
- logger.trace('Requests seen this interval are above limits');
58
- return false;
59
- }
60
- }
61
- }
62
- exports.SimpleCountingRateLimiter = SimpleCountingRateLimiter;
63
- //# sourceMappingURL=simple-counting.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"simple-counting.js","sourceRoot":"","sources":["../../../../src/rate-limiting/request/simple-counting.ts"],"names":[],"mappings":";;;AAGA,qCAAuC;AAEvC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,2BAA2B,CAAC,CAAA;AAEtD;;;;;;;GAOG;AACH,MAAa,yBAAyB;IAAtC;QACE,yBAAoB,GAAG,CAAC,CAAA;QACxB,uBAAkB,GAAG,CAAC,CAAA;QACtB,yBAAoB,GAAG,CAAC,CAAA;QACxB,uBAAkB,GAAG,CAAC,CAAA;IA4DxB,CAAC;IAxDC,UAAU,CACR,SAA4D,EAC5D,MAA6B;QAE7B,kDAAkD;QAClD,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAA;QAC3D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,QAAQ,EAAE,YAAY,CAAC,CAAA;QAC7E,IAAI,CAAC,cAAc,GAAG,MAAM,EAAE,WAAW,IAAI,QAAQ,CAAA;QACrD,MAAM,CAAC,KAAK,CACV,6CAA6C,IAAI,CAAC,cAAc,mBAAmB,IAAI,CAAC,cAAc,EAAE,CACzG,CAAA;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED,aAAa;QACX,qEAAqE;QACrE,IAAI,IAAI,CAAC,cAAc,KAAK,QAAQ,IAAI,IAAI,CAAC,cAAc,KAAK,QAAQ,EAAE;YACxE,OAAO,IAAI,CAAA;SACZ;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAA;QACpD,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAA;QAE3D,sFAAsF;QACtF,mFAAmF;QACnF,IAAI,qBAAqB,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACvD,MAAM,CAAC,KAAK,CACV,8DAA8D,IAAI,CAAC,kBAAkB,GAAG,CACzF,CAAA;YACD,IAAI,CAAC,oBAAoB,GAAG,qBAAqB,CAAA;YACjD,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA;SAC5B;QAED,IAAI,qBAAqB,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACvD,MAAM,CAAC,KAAK,CACV,4DAA4D,IAAI,CAAC,kBAAkB,GAAG,CACvF,CAAA;YACD,IAAI,CAAC,oBAAoB,GAAG,qBAAqB,CAAA;YACjD,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAA;SAC5B;QAED,IACE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAc;YAC7C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAc,EAC7C;YACA,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAA;YACjD,IAAI,CAAC,kBAAkB,EAAE,CAAA;YACzB,IAAI,CAAC,kBAAkB,EAAE,CAAA;YACzB,OAAO,IAAI,CAAA;SACZ;aAAM;YACL,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAA;YAC5D,OAAO,KAAK,CAAA;SACb;IACH,CAAC;CACF;AAhED,8DAgEC"}
@@ -1,35 +0,0 @@
1
- import { AxiosRequestConfig, AxiosResponse } from 'axios';
2
- import { Cache } from '../cache';
3
- import { AdapterConfig, SettingsMap } from '../config';
4
- import { BackgroundExecuteRateLimiter } from '../rate-limiting';
5
- import { SubscriptionSet } from '../util';
6
- import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request';
7
- import { Transport, BackgroundStoredResultType } from './';
8
- import { AdapterContext, AdapterDependencies } from '../adapter';
9
- /**
10
- * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
11
- * Within the setup function, adapter params are added to an set that also keeps track and expires values.
12
- * In the background execute, the list of non-expired items in the set is fetched.
13
- * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
14
- * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
15
- * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
16
- *
17
- * @typeParam AdapterParams - interface for the adapter request body
18
- * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
19
- * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
20
- */
21
- export declare class BatchWarmingTransport<AdapterParams, ProviderRequestBody, ProviderResponseBody, CustomSettings extends SettingsMap> implements Transport<AdapterParams, BackgroundStoredResultType, CustomSettings> {
22
- private config;
23
- cache: Cache<AdapterResponse<BackgroundStoredResultType>>;
24
- rateLimiter: BackgroundExecuteRateLimiter;
25
- subscriptionSet: SubscriptionSet<AdapterParams>;
26
- WARMER_ACTIVE: boolean;
27
- constructor(config: {
28
- prepareRequest: (params: AdapterParams[], config: AdapterConfig<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody>;
29
- parseResponse: (params: AdapterParams[], res: AxiosResponse<ProviderResponseBody>, config: AdapterConfig<CustomSettings>) => ProviderResult<AdapterParams>[];
30
- });
31
- initialize(dependencies: AdapterDependencies, _: AdapterConfig<CustomSettings>, endpointName: string): Promise<void>;
32
- registerRequest(req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>): Promise<void>;
33
- backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number>;
34
- private makeRequest;
35
- }
@@ -1,138 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.BatchWarmingTransport = void 0;
27
- const util_1 = require("../util");
28
- const _1 = require("./");
29
- const util_2 = require("./util");
30
- const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
31
- const cacheMetrics = __importStar(require("../cache/metrics"));
32
- const transportMetrics = __importStar(require("./metrics"));
33
- const error_1 = require("../validation/error");
34
- const WARMUP_BATCH_REQUEST_ID = '9002';
35
- const logger = (0, util_1.makeLogger)('BatchWarmingTransport');
36
- /**
37
- * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
38
- * Within the setup function, adapter params are added to an set that also keeps track and expires values.
39
- * In the background execute, the list of non-expired items in the set is fetched.
40
- * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
41
- * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
42
- * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
43
- *
44
- * @typeParam AdapterParams - interface for the adapter request body
45
- * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
46
- * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
47
- */
48
- class BatchWarmingTransport {
49
- constructor(config) {
50
- this.config = config;
51
- // Flag used to track whether the warmer has moved from having no entries to having some and vice versa
52
- // Used for recording the cache warmer active metrics accurately
53
- this.WARMER_ACTIVE = false;
54
- }
55
- async initialize(dependencies, _, endpointName) {
56
- this.cache = dependencies.cache;
57
- this.rateLimiter = dependencies.backgroundExecuteRateLimiter;
58
- this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet(endpointName);
59
- }
60
- // This might need coalescing to avoid too frequent ttl updates
61
- async registerRequest(req, config) {
62
- logger.debug(`Adding entry to subscription set (ttl ${config.WARMUP_SUBSCRIPTION_TTL}): [${req.requestContext.cacheKey}] = ${req.requestContext.data}`);
63
- if (config.BATCH_TRANSPORT_SETUP_VALIDATION) {
64
- const response = await this.makeRequest([req.requestContext.data], config);
65
- if (!response.results.length) {
66
- throw new error_1.AdapterError({
67
- statusCode: 200,
68
- providerStatusCode: response.providerResponse.status,
69
- message: response.providerResponse.message ||
70
- 'There was an error while validating the incoming request before adding to the batch subscription set',
71
- });
72
- }
73
- }
74
- await this.subscriptionSet.add(req.requestContext.cacheKey, req.requestContext.data, config.WARMUP_SUBSCRIPTION_TTL);
75
- }
76
- async backgroundExecute(context) {
77
- logger.debug('Starting background execute');
78
- const entries = await this.subscriptionSet.getAll();
79
- // Keep track of active subscriptions for background execute
80
- transportMetrics.bgExecuteSubscriptionSetCount
81
- .labels({ endpoint: context.adapterEndpoint.name, transport_type: 'batch' })
82
- .set(entries.length);
83
- const interval = this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name);
84
- if (!entries.length) {
85
- logger.debug('No entries in batch warming set, skipping');
86
- if (this.WARMER_ACTIVE) {
87
- // Decrement count when warmer changed from having entries to having none
88
- cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec();
89
- this.WARMER_ACTIVE = false;
90
- }
91
- return interval;
92
- }
93
- else if (this.WARMER_ACTIVE === false) {
94
- // Increment count when warmer changed from having no entries to having some
95
- cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc();
96
- this.WARMER_ACTIVE = true;
97
- }
98
- const response = await this.makeRequest(entries, context.adapterConfig);
99
- if (!response.results?.length) {
100
- return interval;
101
- }
102
- const cacheEntries = (0, _1.buildCacheEntriesFromResults)(response.results, context);
103
- logger.debug('Setting adapter responses in cache');
104
- await this.cache.setMany(cacheEntries, context.adapterConfig.CACHE_MAX_AGE);
105
- // Record cost of data provider call
106
- const cost = rateLimitMetrics.retrieveCost(response.providerResponse.data);
107
- rateLimitMetrics.rateLimitCreditsSpentTotal
108
- .labels({
109
- feed_id: 'N/A',
110
- participant_id: WARMUP_BATCH_REQUEST_ID,
111
- })
112
- .inc(cost);
113
- return interval;
114
- }
115
- async makeRequest(entries, config) {
116
- logger.trace(`Have ${entries.length} entries in batch, preparing request...`);
117
- const request = this.config.prepareRequest(entries, config);
118
- logger.trace(`Sending request to data provider: ${JSON.stringify(request)}`);
119
- let providerResponse;
120
- try {
121
- providerResponse = await (0, util_2.axiosRequest)(request, config);
122
- }
123
- catch (e) {
124
- logger.warn(`There was an error while performing the batch request: ${e}`);
125
- return {
126
- results: [],
127
- providerResponse: e,
128
- };
129
- }
130
- logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`);
131
- return {
132
- results: this.config.parseResponse(entries, providerResponse, config),
133
- providerResponse,
134
- };
135
- }
136
- }
137
- exports.BatchWarmingTransport = BatchWarmingTransport;
138
- //# sourceMappingURL=batch-warming.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"batch-warming.js","sourceRoot":"","sources":["../../../src/transports/batch-warming.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,kCAAqD;AAErD,yBAAwF;AACxF,iCAAqC;AACrC,2EAA4D;AAC5D,+DAAgD;AAEhD,4DAA6C;AAC7C,+CAAkD;AAElD,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAEtC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,uBAAuB,CAAC,CAAA;AAElD;;;;;;;;;;;GAWG;AACH,MAAa,qBAAqB;IAehC,YACU,MAUP;QAVO,WAAM,GAAN,MAAM,CAUb;QAfH,uGAAuG;QACvG,gEAAgE;QAChE,kBAAa,GAAG,KAAK,CAAA;IAclB,CAAC;IAEJ,KAAK,CAAC,UAAU,CACd,YAAiC,EACjC,CAAgC,EAChC,YAAoB;QAEpB,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,KAA2D,CAAA;QACrF,IAAI,CAAC,WAAW,GAAG,YAAY,CAAC,4BAA4B,CAAA;QAC5D,IAAI,CAAC,eAAe,GAAG,YAAY,CAAC,sBAAsB,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;IACnF,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,eAAe,CACnB,GAAkC,EAClC,MAAqC;QAErC,MAAM,CAAC,KAAK,CACV,yCAAyC,MAAM,CAAC,uBAAuB,OAAO,GAAG,CAAC,cAAc,CAAC,QAAQ,OAAO,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAC1I,CAAA;QAED,IAAI,MAAM,CAAC,gCAAgC,EAAE;YAC3C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAA;YAE1E,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE;gBAC5B,MAAM,IAAI,oBAAY,CAAC;oBACrB,UAAU,EAAE,GAAG;oBACf,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB,CAAC,MAAM;oBACpD,OAAO,EACJ,QAAQ,CAAC,gBAAqC,CAAC,OAAO;wBACvD,sGAAsG;iBACzG,CAAC,CAAA;aACH;SACF;QAED,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAC5B,GAAG,CAAC,cAAc,CAAC,QAAQ,EAC3B,GAAG,CAAC,cAAc,CAAC,IAAI,EACvB,MAAM,CAAC,uBAAuB,CAC/B,CAAA;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAAuC;QAC7D,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;QAC3C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAA;QAEnD,4DAA4D;QAC5D,gBAAgB,CAAC,6BAA6B;aAC3C,MAAM,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;aAC3E,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAEtB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAEpF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACnB,MAAM,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAA;YACzD,IAAI,IAAI,CAAC,aAAa,EAAE;gBACtB,yEAAyE;gBACzE,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAA;gBACjE,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;aAC3B;YACD,OAAO,QAAQ,CAAA;SAChB;aAAM,IAAI,IAAI,CAAC,aAAa,KAAK,KAAK,EAAE;YACvC,4EAA4E;YAC5E,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAA;YACjE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;SAC1B;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;QAEvE,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE;YAC7B,OAAO,QAAQ,CAAA;SAChB;QAED,MAAM,YAAY,GAAG,IAAA,+BAA4B,EAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAE5E,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAClD,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,aAAa,CAAC,aAAa,CAAC,CAAA;QAE3E,oCAAoC;QACpC,MAAM,IAAI,GAAG,gBAAgB,CAAC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAC1E,gBAAgB,CAAC,0BAA0B;aACxC,MAAM,CAAC;YACN,OAAO,EAAE,KAAK;YACd,cAAc,EAAE,uBAAuB;SACxC,CAAC;aACD,GAAG,CAAC,IAAI,CAAC,CAAA;QAEZ,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,OAAwB,EACxB,MAAqC;QAKrC,MAAM,CAAC,KAAK,CAAC,QAAQ,OAAO,CAAC,MAAM,yCAAyC,CAAC,CAAA;QAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAE3D,MAAM,CAAC,KAAK,CAAC,qCAAqC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC5E,IAAI,gBAAgB,CAAA;QACpB,IAAI;YACF,gBAAgB,GAAG,MAAM,IAAA,mBAAY,EAInC,OAAO,EAAE,MAAM,CAAC,CAAA;SACnB;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,EAAE,CAAC,CAAA;YAC1E,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,gBAAgB,EAAE,CAAkB;aACrC,CAAA;SACF;QAED,MAAM,CAAC,KAAK,CAAC,kDAAkD,gBAAgB,CAAC,IAAI,GAAG,CAAC,CAAA;QACxF,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC;YACrE,gBAAgB;SACjB,CAAA;IACH,CAAC;CACF;AArJD,sDAqJC"}
@@ -1,87 +0,0 @@
1
- import { AdapterContext, AdapterDependencies } from '../adapter';
2
- import { Cache, CacheEntry } from '../cache';
3
- import { AdapterConfig, SettingsMap } from '../config';
4
- import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request';
5
- export * from './batch-warming';
6
- export * from './rest';
7
- export * from './sse';
8
- export * from './websocket';
9
- /**
10
- * Generic interface for a Transport.
11
- * A Transport defines the way in which an AdapterEndpoint will process incoming requests to
12
- * fetch data from a Data Provider. The setup phase will take care of the former, while the
13
- * backgroundExecute will be in charge of the latter.
14
- * This separation gives us the ability of splitting these concerns, and optionally parallelizing
15
- * the reading and writing of data to a centralized Cache.
16
- *
17
- * @typeParam Params - the structure of the AdapterRequest's body
18
- * @typeParam Result - the structure of the AdapterResponse's body
19
- */
20
- export interface Transport<Params, Result, CustomSettings extends SettingsMap> {
21
- cache?: Cache<AdapterResponse<Result>>;
22
- /**
23
- * Initializes the transport in the Adapter context.
24
- *
25
- * @param dependencies - Adapter dependencies (e.g. cache instance)
26
- * @param config - Adapter config containing env vars
27
- * @returns an empty Promise
28
- */
29
- initialize: (dependencies: AdapterDependencies, config: AdapterConfig<CustomSettings>, endpointName: string) => Promise<void>;
30
- /**
31
- * Registers a request within the context of the transport.
32
- * This means things like adding the request to a subscription set.
33
- *
34
- * @param req - the incoming AdapterRequest
35
- * @param config - common configuration for the Adapter as a whole
36
- * @returns an empty Promise
37
- */
38
- registerRequest?: (req: AdapterRequest<Params>, config: AdapterConfig<CustomSettings>) => Promise<void>;
39
- /**
40
- * Performs a synchronous fetch/processing of information within the lifecycle of an incoming request.
41
- * This is mostly intended to be used as a backwards compatibility mechanism, or for very specific cases where
42
- * processing is expected to be very simple. Things like rate limiting for example won't be able to be applied
43
- * to this method with more than one EA instance. It's heavily encouraged to instead rework adapter logic
44
- * to perform as much of the work as possible (or all of it) in the backgroundExecute method.
45
- *
46
- * @param req - the incoming AdapterRequest
47
- * @param config - common configuration for the Adapter as a whole
48
- * @returns a Promise that _optionally_ returns an AdapterResponse, if the Transport has the capability of
49
- * immediately fetching data and returning it without the background process.
50
- */
51
- foregroundExecute?: (req: AdapterRequest<Params>, config: AdapterConfig<CustomSettings>) => Promise<AdapterResponse<Result> | void>;
52
- /**
53
- * Main function that will be called by the main background loop of the writer EA.
54
- * All logic fetching from providers should live here.
55
- *
56
- * @param context - background context for the execution (e.g. endpoint name)
57
- * @returns an empty Promise
58
- */
59
- backgroundExecute?: (context: AdapterContext<CustomSettings>) => Promise<number>;
60
- }
61
- /**
62
- * Generic interface for a Transport that contains or manages other transports.
63
- * Some transports, such as the RoutingTransport, do not directly fetch data from a Data Provider, but
64
- * instead coordinate the execution of other transports for fetching and transforming data.
65
- * These transports have alternative initialization procedures, which typically involve
66
- * initializing each downstream transport.
67
- *
68
- * @typeParam Params - the structure of the AdapterRequest's body
69
- * @typeParam Result - the structure of the AdapterResponse's body
70
- * @typeParam CustomSettings - custom settings for the Adapter, or the default SettingsMap
71
- */
72
- export interface MetaTransport<Params, Result, CustomSettings extends SettingsMap> extends Transport<Params, Result, CustomSettings> {
73
- transports: {
74
- [key: string]: Transport<unknown, unknown, CustomSettings>;
75
- };
76
- }
77
- export declare type BackgroundStoredResultType = {
78
- result: unknown;
79
- };
80
- /**
81
- * Helper method to build cache entries to set after getting a bunch of responses from a DP.
82
- *
83
- * @param results - a list of results coming from a DataProvider
84
- * @param context - context for the Adapter
85
- * @returns a list of CacheEntries of AdapterResponses
86
- */
87
- export declare const buildCacheEntriesFromResults: <Params, CustomSettings extends SettingsMap>(results: ProviderResult<Params>[], context: AdapterContext<CustomSettings>) => CacheEntry<AdapterResponse<BackgroundStoredResultType>>[];
@@ -1,56 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
- for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
- };
16
- Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.buildCacheEntriesFromResults = void 0;
18
- const cache_1 = require("../cache");
19
- __exportStar(require("./batch-warming"), exports);
20
- __exportStar(require("./rest"), exports);
21
- __exportStar(require("./sse"), exports);
22
- __exportStar(require("./websocket"), exports);
23
- /**
24
- * Helper method to build cache entries to set after getting a bunch of responses from a DP.
25
- *
26
- * @param results - a list of results coming from a DataProvider
27
- * @param context - context for the Adapter
28
- * @returns a list of CacheEntries of AdapterResponses
29
- */
30
- const buildCacheEntriesFromResults = (results, context) => results.map((r) => {
31
- const cacheEntry = {
32
- key: (0, cache_1.calculateCacheKey)(context, r.params),
33
- value: {
34
- result: r.value,
35
- statusCode: 200,
36
- data: {
37
- result: r.value,
38
- },
39
- },
40
- };
41
- if (context.adapterConfig.METRICS_ENABLED &&
42
- context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED) {
43
- const metrics = {
44
- maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE,
45
- meta: {
46
- metrics: {
47
- feedId: (0, cache_1.calculateFeedId)(context, r.params),
48
- },
49
- },
50
- };
51
- cacheEntry.value = { ...cacheEntry.value, ...metrics };
52
- }
53
- return cacheEntry;
54
- });
55
- exports.buildCacheEntriesFromResults = buildCacheEntriesFromResults;
56
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transports/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AACA,oCAAgF;AAIhF,kDAA+B;AAC/B,yCAAsB;AACtB,wCAAqB;AACrB,8CAA2B;AA2F3B;;;;;;GAMG;AACI,MAAM,4BAA4B,GAAG,CAC1C,OAAiC,EACjC,OAAuC,EACoB,EAAE,CAC7D,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;IAChB,MAAM,UAAU,GAAG;QACjB,GAAG,EAAE,IAAA,yBAAiB,EAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QACzC,KAAK,EAAE;YACL,MAAM,EAAE,CAAC,CAAC,KAAK;YACf,UAAU,EAAE,GAAG;YACf,IAAI,EAAE;gBACJ,MAAM,EAAE,CAAC,CAAC,KAAK;aAChB;SACF;KACF,CAAA;IAED,IACE,OAAO,CAAC,aAAa,CAAC,eAAe;QACrC,OAAO,CAAC,aAAa,CAAC,4BAA4B,EAClD;QACA,MAAM,OAAO,GAAG;YACd,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,aAAa;YACxD,IAAI,EAAE;gBACJ,OAAO,EAAE;oBACP,MAAM,EAAE,IAAA,uBAAe,EAAC,OAAyB,EAAE,CAAC,CAAC,MAAM,CAAC;iBAC7D;aACF;SACF,CAAA;QACD,UAAU,CAAC,KAAK,GAAG,EAAE,GAAG,UAAU,CAAC,KAAK,EAAE,GAAG,OAAO,EAAE,CAAA;KACvD;IACD,OAAO,UAAU,CAAA;AACnB,CAAC,CAAC,CAAA;AA/BS,QAAA,4BAA4B,gCA+BrC"}
@@ -1,25 +0,0 @@
1
- import * as client from 'prom-client';
2
- import { AdapterContext } from '../adapter';
3
- import { SettingsMap } from '../config';
4
- export declare const dataProviderMetricsLabel: (providerStatusCode?: number, method?: string) => {
5
- provider_status_code: number | undefined;
6
- method: string;
7
- };
8
- export declare const dataProviderRequests: client.Counter<"method" | "provider_status_code">;
9
- export declare const dataProviderRequestDurationSeconds: client.Histogram<string>;
10
- export declare const connectionErrorLabels: (message: string) => {
11
- message: string;
12
- };
13
- export declare const messageSubsLabels: (feed_id: string, cache_key: string) => {
14
- feed_id: string;
15
- subscription_key: string;
16
- };
17
- export declare const recordWsMessageMetrics: <AdapterParams, CustomSettings extends SettingsMap>(context: AdapterContext<CustomSettings>, subscribes: AdapterParams[], unsubscrices: AdapterParams[]) => void;
18
- export declare const wsConnectionActive: client.Gauge<"url">;
19
- export declare const wsConnectionErrors: client.Counter<"message" | "url">;
20
- export declare const wsSubscriptionActive: client.Gauge<"feed_id" | "connection_url" | "subscription_key">;
21
- export declare const wsSubscriptionTotal: client.Counter<"feed_id" | "connection_url" | "subscription_key">;
22
- export declare const wsMessageTotal: client.Counter<"feed_id" | "subscription_key">;
23
- export declare const bgExecuteSubscriptionSetCount: client.Gauge<"endpoint" | "transport_type">;
24
- export declare const transportPollingFailureCount: client.Counter<"endpoint">;
25
- export declare const transportPollingDurationSeconds: client.Gauge<"succeeded" | "endpoint">;
@@ -1,122 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.transportPollingDurationSeconds = exports.transportPollingFailureCount = exports.bgExecuteSubscriptionSetCount = exports.wsMessageTotal = exports.wsSubscriptionTotal = exports.wsSubscriptionActive = exports.wsConnectionErrors = exports.wsConnectionActive = exports.recordWsMessageMetrics = exports.messageSubsLabels = exports.connectionErrorLabels = exports.dataProviderRequestDurationSeconds = exports.dataProviderRequests = exports.dataProviderMetricsLabel = void 0;
27
- const client = __importStar(require("prom-client"));
28
- const cache_1 = require("../cache");
29
- const constants_1 = require("../metrics/constants");
30
- // Data Provider Metrics
31
- const dataProviderMetricsLabel = (providerStatusCode, method = 'get') => ({
32
- provider_status_code: providerStatusCode,
33
- method: method.toUpperCase(),
34
- });
35
- exports.dataProviderMetricsLabel = dataProviderMetricsLabel;
36
- exports.dataProviderRequests = new client.Counter({
37
- name: 'data_provider_requests',
38
- help: 'The number of http requests that are made to a data provider',
39
- labelNames: ['method', 'provider_status_code'],
40
- });
41
- exports.dataProviderRequestDurationSeconds = new client.Histogram({
42
- name: 'data_provider_request_duration_seconds',
43
- help: 'A histogram bucket of the distribution of data provider request durations',
44
- buckets: constants_1.requestDurationBuckets,
45
- });
46
- // Websocket Metrics
47
- const connectionErrorLabels = (message) => ({
48
- // Key,
49
- message,
50
- });
51
- exports.connectionErrorLabels = connectionErrorLabels;
52
- const messageSubsLabels = (feed_id, cache_key) => ({
53
- feed_id,
54
- subscription_key: cache_key,
55
- });
56
- exports.messageSubsLabels = messageSubsLabels;
57
- // Record WS message and subscription metrics
58
- // Recalculate cacheKey and feedId for metrics
59
- // since avoiding storing extra info in expiring sorted set
60
- const recordWsMessageMetrics = (context, subscribes, unsubscrices) => {
61
- subscribes.forEach((param) => {
62
- const feedId = (0, cache_1.calculateFeedId)(context, param);
63
- const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
64
- // Record total number of ws messages sent
65
- exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
66
- // Record total number of subscriptions made
67
- exports.wsSubscriptionTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
68
- // Record number of active ws subscriptions
69
- exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
70
- });
71
- unsubscrices.forEach((param) => {
72
- const feedId = (0, cache_1.calculateFeedId)(context, param);
73
- const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
74
- // Record total number of ws messages sent
75
- exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
76
- // Record number of active ws subscriptions
77
- exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).dec();
78
- });
79
- };
80
- exports.recordWsMessageMetrics = recordWsMessageMetrics;
81
- exports.wsConnectionActive = new client.Gauge({
82
- name: 'ws_connection_active',
83
- help: 'The number of active connections',
84
- labelNames: ['url'],
85
- });
86
- exports.wsConnectionErrors = new client.Counter({
87
- name: 'ws_connection_errors',
88
- help: 'The number of connection errors',
89
- labelNames: ['url', 'message'],
90
- });
91
- exports.wsSubscriptionActive = new client.Gauge({
92
- name: 'ws_subscription_active',
93
- help: 'The number of currently active subscriptions',
94
- labelNames: ['connection_url', 'feed_id', 'subscription_key'],
95
- });
96
- exports.wsSubscriptionTotal = new client.Counter({
97
- name: 'ws_subscription_total',
98
- help: 'The number of subscriptions opened in total',
99
- labelNames: ['connection_url', 'feed_id', 'subscription_key'],
100
- });
101
- exports.wsMessageTotal = new client.Counter({
102
- name: 'ws_message_total',
103
- help: 'The number of messages received in total',
104
- labelNames: ['feed_id', 'subscription_key'],
105
- });
106
- // V3 specific metrics
107
- exports.bgExecuteSubscriptionSetCount = new client.Gauge({
108
- name: 'bg_execute_subscription_set_count',
109
- help: 'The number of active subscriptions in background execute',
110
- labelNames: ['endpoint', 'transport_type'],
111
- });
112
- exports.transportPollingFailureCount = new client.Counter({
113
- name: 'transport_polling_failure_count',
114
- help: 'The number of times the polling mechanism ran out of attempts and failed to return a response',
115
- labelNames: ['endpoint'],
116
- });
117
- exports.transportPollingDurationSeconds = new client.Gauge({
118
- name: 'transport_polling_duration_seconds',
119
- help: 'A histogram bucket of the distribution of transport polling idle time durations',
120
- labelNames: ['endpoint', 'succeeded'],
121
- });
122
- //# sourceMappingURL=metrics.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/transports/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AAErC,oCAA6D;AAE7D,oDAA6D;AAE7D,wBAAwB;AACjB,MAAM,wBAAwB,GAAG,CAAC,kBAA2B,EAAE,MAAM,GAAG,KAAK,EAAE,EAAE,CAAC,CAAC;IACxF,oBAAoB,EAAE,kBAAkB;IACxC,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;CAC7B,CAAC,CAAA;AAHW,QAAA,wBAAwB,4BAGnC;AAEW,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACrD,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,8DAA8D;IACpE,UAAU,EAAE,CAAC,QAAQ,EAAE,sBAAsB,CAAU;CACxD,CAAC,CAAA;AAEW,QAAA,kCAAkC,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC;IACrE,IAAI,EAAE,wCAAwC;IAC9C,IAAI,EAAE,2EAA2E;IACjF,OAAO,EAAE,kCAAsB;CAChC,CAAC,CAAA;AAEF,oBAAoB;AACb,MAAM,qBAAqB,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,CAAC;IACzD,OAAO;IACP,OAAO;CACR,CAAC,CAAA;AAHW,QAAA,qBAAqB,yBAGhC;AAEK,MAAM,iBAAiB,GAAG,CAAC,OAAe,EAAE,SAAiB,EAAE,EAAE,CAAC,CAAC;IACxE,OAAO;IACP,gBAAgB,EAAE,SAAS;CAC5B,CAAC,CAAA;AAHW,QAAA,iBAAiB,qBAG5B;AAEF,6CAA6C;AAC7C,8CAA8C;AAC9C,2DAA2D;AACpD,MAAM,sBAAsB,GAAG,CACpC,OAAuC,EACvC,UAA2B,EAC3B,YAA6B,EACvB,EAAE;IACR,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC3B,MAAM,MAAM,GAAG,IAAA,uBAAe,EAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAC9C,MAAM,QAAQ,GAAG,IAAA,yBAAiB,EAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAClD,0CAA0C;QAC1C,sBAAc,CAAC,MAAM,CAAC,IAAA,yBAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;QAEhE,4CAA4C;QAC5C,2BAAmB,CAAC,MAAM,CAAC,IAAA,yBAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;QAErE,2CAA2C;QAC3C,4BAAoB,CAAC,MAAM,CAAC,IAAA,yBAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;IACxE,CAAC,CAAC,CAAA;IACF,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,IAAA,uBAAe,EAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAC9C,MAAM,QAAQ,GAAG,IAAA,yBAAiB,EAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAElD,0CAA0C;QAC1C,sBAAc,CAAC,MAAM,CAAC,IAAA,yBAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;QAEhE,2CAA2C;QAC3C,4BAAoB,CAAC,MAAM,CAAC,IAAA,yBAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;IACxE,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AA3BY,QAAA,sBAAsB,0BA2BlC;AAEY,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACjD,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,kCAAkC;IACxC,UAAU,EAAE,CAAC,KAAK,CAAU;CAC7B,CAAC,CAAA;AAEW,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACnD,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,iCAAiC;IACvC,UAAU,EAAE,CAAC,KAAK,EAAE,SAAS,CAAU;CACxC,CAAC,CAAA;AAEW,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACnD,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,8CAA8C;IACpD,UAAU,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,kBAAkB,CAAU;CACvE,CAAC,CAAA;AAEW,QAAA,mBAAmB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACpD,IAAI,EAAE,uBAAuB;IAC7B,IAAI,EAAE,6CAA6C;IACnD,UAAU,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,kBAAkB,CAAU;CACvE,CAAC,CAAA;AAEW,QAAA,cAAc,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC/C,IAAI,EAAE,kBAAkB;IACxB,IAAI,EAAE,0CAA0C;IAChD,UAAU,EAAE,CAAC,SAAS,EAAE,kBAAkB,CAAU;CACrD,CAAC,CAAA;AAEF,sBAAsB;AACT,QAAA,6BAA6B,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IAC5D,IAAI,EAAE,mCAAmC;IACzC,IAAI,EAAE,0DAA0D;IAChE,UAAU,EAAE,CAAC,UAAU,EAAE,gBAAgB,CAAU;CACpD,CAAC,CAAA;AAEW,QAAA,4BAA4B,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC7D,IAAI,EAAE,iCAAiC;IACvC,IAAI,EAAE,+FAA+F;IACrG,UAAU,EAAE,CAAC,UAAU,CAAU;CAClC,CAAC,CAAA;AAEW,QAAA,+BAA+B,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IAC9D,IAAI,EAAE,oCAAoC;IAC1C,IAAI,EAAE,iFAAiF;IACvF,UAAU,EAAE,CAAC,UAAU,EAAE,WAAW,CAAU;CAC/C,CAAC,CAAA"}