@chainlink/external-adapter-framework 0.0.7 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/adapter.d.ts +88 -0
  2. package/{dist/src/adapter.js → adapter.js} +0 -0
  3. package/background-executor.d.ts +11 -0
  4. package/{dist/background-executor.js → background-executor.js} +0 -0
  5. package/cache/factory.d.ts +6 -0
  6. package/{dist/cache → cache}/factory.js +0 -0
  7. package/cache/index.d.ts +90 -0
  8. package/{dist/src/cache → cache}/index.js +6 -2
  9. package/cache/local.d.ts +23 -0
  10. package/{dist/cache → cache}/local.js +1 -1
  11. package/cache/metrics.d.ts +27 -0
  12. package/{dist/cache → cache}/metrics.js +7 -1
  13. package/cache/redis.d.ts +16 -0
  14. package/{dist/cache → cache}/redis.js +2 -2
  15. package/config/index.d.ts +195 -0
  16. package/{dist/src/config → config}/index.js +5 -6
  17. package/config/provider-limits.d.ts +31 -0
  18. package/{dist/src/config → config}/provider-limits.js +1 -1
  19. package/examples/coingecko/batch-warming.d.ts +2 -0
  20. package/{dist/examples → examples}/coingecko/batch-warming.js +0 -0
  21. package/examples/coingecko/index.d.ts +2 -0
  22. package/{dist/examples → examples}/coingecko/index.js +0 -0
  23. package/examples/coingecko/rest.d.ts +2 -0
  24. package/{dist/examples → examples}/coingecko/rest.js +0 -0
  25. package/examples/ncfx/config/index.d.ts +12 -0
  26. package/{dist/examples → examples}/ncfx/config/index.js +0 -0
  27. package/examples/ncfx/index.d.ts +2 -0
  28. package/{dist/examples → examples}/ncfx/index.js +0 -0
  29. package/examples/ncfx/websocket.d.ts +36 -0
  30. package/{dist/examples → examples}/ncfx/websocket.js +0 -0
  31. package/index.d.ts +12 -0
  32. package/{dist/index.js → index.js} +3 -0
  33. package/metrics/constants.d.ts +16 -0
  34. package/{dist/src/metrics → metrics}/constants.js +1 -1
  35. package/metrics/index.d.ts +15 -0
  36. package/{dist/metrics → metrics}/index.js +48 -1
  37. package/metrics/util.d.ts +3 -0
  38. package/{dist/metrics → metrics}/util.js +0 -0
  39. package/package.json +2 -6
  40. package/rate-limiting/background/fixed-frequency.d.ts +10 -0
  41. package/{dist/src/rate-limiting → rate-limiting}/background/fixed-frequency.js +0 -0
  42. package/rate-limiting/index.d.ts +54 -0
  43. package/{dist/src/rate-limiting → rate-limiting}/index.js +0 -0
  44. package/rate-limiting/metrics.d.ts +3 -0
  45. package/{dist/src/rate-limiting → rate-limiting}/metrics.js +14 -2
  46. package/rate-limiting/request/simple-counting.d.ts +20 -0
  47. package/{dist/src/rate-limiting → rate-limiting}/request/simple-counting.js +0 -0
  48. package/test.d.ts +1 -0
  49. package/{dist/test.js → test.js} +0 -0
  50. package/transports/batch-warming.d.ts +34 -0
  51. package/{dist/src/transports → transports}/batch-warming.js +47 -1
  52. package/transports/index.d.ts +87 -0
  53. package/{dist/src/transports → transports}/index.js +5 -3
  54. package/transports/metrics.d.ts +21 -0
  55. package/{dist/src/transports → transports}/metrics.js +28 -42
  56. package/transports/rest.d.ts +43 -0
  57. package/{dist/src/transports → transports}/rest.js +38 -2
  58. package/transports/util.d.ts +8 -0
  59. package/{dist/src/transports → transports}/util.js +16 -16
  60. package/transports/websocket.d.ts +80 -0
  61. package/{dist/src/transports → transports}/websocket.js +8 -14
  62. package/util/expiring-sorted-set.d.ts +21 -0
  63. package/{dist/src/util → util}/expiring-sorted-set.js +0 -0
  64. package/util/index.d.ts +11 -0
  65. package/{dist/src/util → util}/index.js +0 -0
  66. package/util/logger.d.ts +42 -0
  67. package/{dist/src/util → util}/logger.js +0 -0
  68. package/util/request.d.ts +55 -0
  69. package/{dist/src/util → util}/request.js +0 -0
  70. package/validation/error.d.ts +50 -0
  71. package/validation/error.js +79 -0
  72. package/validation/index.d.ts +5 -0
  73. package/{dist/src/validation → validation}/index.js +8 -6
  74. package/validation/input-params.d.ts +15 -0
  75. package/{dist/src/validation → validation}/input-params.js +0 -0
  76. package/validation/override-functions.d.ts +3 -0
  77. package/{dist/src/validation → validation}/override-functions.js +0 -0
  78. package/{dist/src/validation → validation}/preset-tokens.json +0 -0
  79. package/validation/validator.d.ts +47 -0
  80. package/{dist/src/validation → validation}/validator.js +0 -0
  81. package/README.md +0 -103
  82. package/dist/adapter.js +0 -60
  83. package/dist/cache/index.js +0 -163
  84. package/dist/config/index.js +0 -364
  85. package/dist/config/provider-limits.js +0 -75
  86. package/dist/metrics/constants.js +0 -25
  87. package/dist/rate-limiting/factory.js +0 -33
  88. package/dist/rate-limiting/index.js +0 -36
  89. package/dist/rate-limiting/metrics.js +0 -32
  90. package/dist/rate-limiting/nop-limiter.js +0 -15
  91. package/dist/rate-limiting/simple-counting.js +0 -61
  92. package/dist/src/background-executor.js +0 -45
  93. package/dist/src/cache/factory.js +0 -57
  94. package/dist/src/cache/local.js +0 -83
  95. package/dist/src/cache/metrics.js +0 -114
  96. package/dist/src/cache/redis.js +0 -100
  97. package/dist/src/examples/bank-frick/accounts.js +0 -191
  98. package/dist/src/examples/bank-frick/config/index.js +0 -45
  99. package/dist/src/examples/bank-frick/index.js +0 -14
  100. package/dist/src/examples/bank-frick/util.js +0 -39
  101. package/dist/src/examples/coingecko/batch-warming.js +0 -52
  102. package/dist/src/examples/coingecko/index.js +0 -10
  103. package/dist/src/examples/coingecko/rest.js +0 -50
  104. package/dist/src/examples/ncfx/config/index.js +0 -15
  105. package/dist/src/examples/ncfx/index.js +0 -10
  106. package/dist/src/examples/ncfx/websocket.js +0 -72
  107. package/dist/src/index.js +0 -89
  108. package/dist/src/metrics/index.js +0 -76
  109. package/dist/src/metrics/util.js +0 -9
  110. package/dist/src/test.js +0 -6
  111. package/dist/src/validation/error.js +0 -41
  112. package/dist/transports/batch-warming.js +0 -57
  113. package/dist/transports/index.js +0 -76
  114. package/dist/transports/metrics.js +0 -133
  115. package/dist/transports/rest.js +0 -91
  116. package/dist/transports/util.js +0 -85
  117. package/dist/transports/websocket.js +0 -171
  118. package/dist/util/expiring-sorted-set.js +0 -47
  119. package/dist/util/index.js +0 -35
  120. package/dist/util/logger.js +0 -62
  121. package/dist/util/request.js +0 -2
  122. package/dist/validation/error.js +0 -41
  123. package/dist/validation/index.js +0 -82
  124. package/dist/validation/input-params.js +0 -30
  125. package/dist/validation/overrideFunctions.js +0 -42
  126. package/dist/validation/presetTokens.json +0 -23
  127. package/dist/validation/validator.js +0 -303
@@ -23,8 +23,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
23
23
  return result;
24
24
  };
25
25
  Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.cacheWarmerCount = exports.wsMessageTotal = exports.wsSubscriptionErrors = exports.wsSubscriptionTotal = exports.wsSubscriptionActive = exports.wsConnectionRetries = exports.wsConnectionErrors = exports.wsConnectionActive = exports.messageLabels = exports.subscriptionErrorLabels = exports.subscriptionLabels = exports.connectionErrorLabels = exports.connectionLabels = exports.dataProviderRequestDurationSeconds = exports.dataProviderRequests = exports.dataProviderMetricsLabel = void 0;
26
+ exports.wsMessageTotal = exports.wsSubscriptionTotal = exports.wsSubscriptionActive = exports.wsConnectionErrors = exports.wsConnectionActive = exports.recordWsMessageMetrics = exports.messageSubsLabels = exports.connectionErrorLabels = exports.dataProviderRequestDurationSeconds = exports.dataProviderRequests = exports.dataProviderMetricsLabel = void 0;
27
27
  const client = __importStar(require("prom-client"));
28
+ const cache_1 = require("../cache");
28
29
  const constants_1 = require("../metrics/constants");
29
30
  // Data Provider Metrics
30
31
  const dataProviderMetricsLabel = (providerStatusCode, method = 'get') => ({
@@ -43,38 +44,40 @@ exports.dataProviderRequestDurationSeconds = new client.Histogram({
43
44
  buckets: constants_1.requestDurationBuckets,
44
45
  });
45
46
  // Websocket Metrics
46
- const connectionLabels = (key) => ({
47
- // Key,
48
- });
49
- exports.connectionLabels = connectionLabels;
50
47
  const connectionErrorLabels = (message) => ({
51
48
  // Key,
52
49
  message,
53
50
  });
54
51
  exports.connectionErrorLabels = connectionErrorLabels;
55
- const subscriptionLabels = (
56
- // Connection_key: string,
57
- feed_id, cache_key) => ({
58
- // Connection_key,
59
- feed_id,
60
- subscription_key: cache_key,
61
- });
62
- exports.subscriptionLabels = subscriptionLabels;
63
- const subscriptionErrorLabels = (
64
- // Connection_key: string,
65
- input, message, type) => ({
66
- // Connection_key,
67
- feed_id: input ? input.metricsMeta?.feedId : 'N/A',
68
- message,
69
- subscription_key: input.cacheKey ? input.cacheKey : 'N/A',
70
- type,
71
- });
72
- exports.subscriptionErrorLabels = subscriptionErrorLabels;
73
- const messageLabels = (feed_id, cache_key) => ({
52
+ const messageSubsLabels = (feed_id, cache_key) => ({
74
53
  feed_id,
75
54
  subscription_key: cache_key,
76
55
  });
77
- exports.messageLabels = messageLabels;
56
+ exports.messageSubsLabels = messageSubsLabels;
57
+ // Record WS message and subscription metrics
58
+ // Recalculate cacheKey and feedId for metrics
59
+ // since avoiding storing extra info in expiring sorted set
60
+ const recordWsMessageMetrics = (context, subscribes, unsubscrices) => {
61
+ subscribes.forEach((param) => {
62
+ const feedId = (0, cache_1.calculateFeedId)(context.adapterEndpoint, param);
63
+ const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
64
+ // Record total number of ws messages sent
65
+ exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
66
+ // Record total number of subscriptions made
67
+ exports.wsSubscriptionTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
68
+ // Record number of active ws subscriptions
69
+ exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
70
+ });
71
+ unsubscrices.forEach((param) => {
72
+ const feedId = (0, cache_1.calculateFeedId)(context.adapterEndpoint, param);
73
+ const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
74
+ // Record total number of ws messages sent
75
+ exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
76
+ // Record number of active ws subscriptions
77
+ exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).dec();
78
+ });
79
+ };
80
+ exports.recordWsMessageMetrics = recordWsMessageMetrics;
78
81
  exports.wsConnectionActive = new client.Gauge({
79
82
  name: 'ws_connection_active',
80
83
  help: 'The number of active connections',
@@ -85,12 +88,6 @@ exports.wsConnectionErrors = new client.Counter({
85
88
  help: 'The number of connection errors',
86
89
  labelNames: ['url', 'message'],
87
90
  });
88
- // Doesn't seem to be any retry connection functionality yet
89
- exports.wsConnectionRetries = new client.Counter({
90
- name: 'ws_connection_retries',
91
- help: 'The number of connection retries',
92
- labelNames: ['url'],
93
- });
94
91
  exports.wsSubscriptionActive = new client.Gauge({
95
92
  name: 'ws_subscription_active',
96
93
  help: 'The number of currently active subscriptions',
@@ -101,19 +98,8 @@ exports.wsSubscriptionTotal = new client.Counter({
101
98
  help: 'The number of subscriptions opened in total',
102
99
  labelNames: ['connection_url', 'feed_id', 'subscription_key'],
103
100
  });
104
- exports.wsSubscriptionErrors = new client.Counter({
105
- name: 'ws_subscription_errors',
106
- help: 'The number of subscriptions errors',
107
- labelNames: ['connection_url', 'feed_id', 'subscription_key', 'message', 'type'],
108
- });
109
101
  exports.wsMessageTotal = new client.Counter({
110
102
  name: 'ws_message_total',
111
103
  help: 'The number of messages received in total',
112
104
  labelNames: ['feed_id', 'subscription_key'],
113
105
  });
114
- // Cache Warmer Metrics
115
- exports.cacheWarmerCount = new client.Gauge({
116
- name: 'cache_warmer_get_count',
117
- help: 'The number of cache warmers running',
118
- labelNames: ['isBatched'],
119
- });
@@ -0,0 +1,43 @@
1
+ import { AdapterRequest, AdapterResponse } from '../util/request';
2
+ import { AdapterDependencies, Transport } from './';
3
+ import { Cache } from '../cache';
4
+ import { AxiosRequestConfig, AxiosResponse } from 'axios';
5
+ import { AdapterConfig, SettingsMap } from '../config';
6
+ import { RequestRateLimiter } from '../rate-limiting';
7
+ /**
8
+ * Transport implementation that takes incoming requests, transforms them into a DataProvider request,
9
+ * and executes that request returning the response immediately from the `setup` function.
10
+ * Optionally, setting the `coalescing` option to `true` will make it so once a request is in flight,
11
+ * new adapter requests for the same feed will not trigger a new one, but return an empty promise from
12
+ * the setup instead so the normal cache polling mechanism is used.
13
+ *
14
+ * @typeParam AdapterParams - interface for the adapter request body
15
+ * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
16
+ * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
17
+ */
18
+ export declare class RestTransport<AdapterParams, ProviderRequestBody, ProviderResponseBody, CustomSettings extends SettingsMap> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings> {
19
+ protected config: {
20
+ prepareRequest: (req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody> | Promise<AxiosRequestConfig<ProviderRequestBody>>;
21
+ parseResponse: (req: AdapterRequest<AdapterParams>, res: AxiosResponse<ProviderResponseBody>, config: AdapterConfig<CustomSettings>) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>;
22
+ options: {
23
+ coalescing: boolean;
24
+ };
25
+ };
26
+ inFlightPrefix: string;
27
+ cache: Cache;
28
+ rateLimiter: RequestRateLimiter;
29
+ constructor(config: {
30
+ prepareRequest: (req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>) => AxiosRequestConfig<ProviderRequestBody> | Promise<AxiosRequestConfig<ProviderRequestBody>>;
31
+ parseResponse: (req: AdapterRequest<AdapterParams>, res: AxiosResponse<ProviderResponseBody>, config: AdapterConfig<CustomSettings>) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>;
32
+ options: {
33
+ coalescing: boolean;
34
+ };
35
+ });
36
+ initialize(dependencies: AdapterDependencies): Promise<void>;
37
+ hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean>;
38
+ protected waitUntilUnderRateLimit(options: {
39
+ maxRetries: number;
40
+ msBetweenRetries: number;
41
+ }, retry?: number): Promise<void>;
42
+ setup(req: AdapterRequest<AdapterParams>, config: AdapterConfig<CustomSettings>): Promise<AdapterResponse<ProviderResponseBody> | undefined>;
43
+ }
@@ -1,9 +1,33 @@
1
1
  "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
2
25
  Object.defineProperty(exports, "__esModule", { value: true });
3
26
  exports.RestTransport = void 0;
4
27
  const util_1 = require("../util");
5
28
  const error_1 = require("../validation/error");
6
29
  const util_2 = require("./util");
30
+ const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
7
31
  const IN_FLIGHT_PREFIX = 'InFlight';
8
32
  const logger = (0, util_1.makeLogger)('RestTransport');
9
33
  /**
@@ -77,11 +101,23 @@ class RestTransport {
77
101
  if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
78
102
  // TODO: Potentially create function to add all telemetry data
79
103
  parsedResponse.maxAge = Date.now() + config.CACHE_MAX_AGE;
80
- parsedResponse.metricsMeta = { feedId: req.requestContext.metricsMeta?.feedId || 'N/A' };
104
+ parsedResponse.meta = { metrics: { feedId: req.requestContext.meta?.metrics?.feedId || 'N/A' } };
81
105
  }
82
106
  logger.debug('Setting provider response in cache');
83
107
  await this.cache.set(req.requestContext.cacheKey, parsedResponse, config.CACHE_MAX_AGE);
84
- // TODO: Record rate limit credit with cost, default 1
108
+ // Record cost of data provider call
109
+ const cost = rateLimitMetrics.retrieveCost(providerResponse.data);
110
+ rateLimitMetrics.rateLimitCreditsSpentTotal
111
+ .labels({
112
+ feed_id: req.requestContext.meta?.metrics?.feedId || 'N/A',
113
+ participant_id: req.requestContext.cacheKey,
114
+ })
115
+ .inc(cost);
116
+ // Update cacheHit flag in request meta for metrics use
117
+ req.requestContext.meta = {
118
+ ...req.requestContext.meta,
119
+ metrics: { ...req.requestContext.meta?.metrics, cacheHit: false },
120
+ };
85
121
  // TODO: move this to a try/catch/finally
86
122
  if (this.config.options.coalescing) {
87
123
  logger.debug('Set provider response in cache, removing in flight from cache');
@@ -0,0 +1,8 @@
1
+ import { AxiosRequestConfig, AxiosResponse } from 'axios';
2
+ /**
3
+ * Performs axios request along with metrics recording and error handling
4
+ *
5
+ * @param request - axios request config
6
+ * @returns axios response for the request
7
+ */
8
+ export declare function axiosRequest<ProviderRequestBody, ProviderResponseBody>(request: AxiosRequestConfig<ProviderRequestBody>): Promise<AxiosResponse<ProviderResponseBody>>;
@@ -46,31 +46,31 @@ async function axiosRequest(request) {
46
46
  const error = e;
47
47
  // Request error
48
48
  let providerStatusCode;
49
- let name;
50
- let statusCode;
49
+ let adapterError;
51
50
  if (error.code === 'ECONNABORTED') {
51
+ adapterError = new error_1.AdapterTimeoutError({});
52
52
  providerStatusCode = error?.response?.status ?? 504;
53
- name = 'Data Provider Request Timeout error';
54
- statusCode = 504;
53
+ adapterError.name = 'Data Provider Request Timeout error';
54
+ }
55
+ else if (error?.response?.status) {
56
+ adapterError = new error_1.AdapterDataProviderError({});
57
+ providerStatusCode = error?.response?.status;
55
58
  }
56
59
  else {
57
- providerStatusCode = error?.response?.status ?? 0; // 0 -> connection error
58
- statusCode = 200;
60
+ adapterError = new error_1.AdapterConnectionError({});
61
+ providerStatusCode = 0; // 0 -> connection error
59
62
  }
60
63
  // Record count of failed data provider request
61
64
  transportMetrics.dataProviderRequests
62
65
  .labels(transportMetrics.dataProviderMetricsLabel(providerStatusCode, request.method))
63
66
  .inc();
64
- // TODO: Use specific AdapterErrors once ported over (AdapterTimeoutError, AdapterConnectionError, AdapterDataProviderError)
65
- throw new error_1.AdapterError({
66
- statusCode,
67
- name,
68
- providerStatusCode,
69
- message: error?.message,
70
- cause: error,
71
- errorResponse: error?.response?.data,
72
- url: request.url,
73
- });
67
+ adapterError.statusCode = 200;
68
+ adapterError.providerStatusCode = providerStatusCode;
69
+ adapterError.message = error?.message;
70
+ adapterError.cause = error;
71
+ adapterError.errorResponse = error?.response?.data;
72
+ adapterError.url = request.url;
73
+ throw adapterError;
74
74
  }
75
75
  finally {
76
76
  // Record time taken for data provider request for success or failure
@@ -0,0 +1,80 @@
1
+ import WebSocket from 'ws';
2
+ import { Cache } from '../cache';
3
+ import { SettingsMap } from '../config';
4
+ import { BackgroundExecuteRateLimiter } from '../rate-limiting';
5
+ import { ExpiringSortedSet } from '../util';
6
+ import { AdapterRequest, ProviderResult } from '../util/request';
7
+ import { AdapterContext, AdapterDependencies, Transport } from './';
8
+ export declare const DEFAULT_WS_TTL = 10000;
9
+ declare type WebSocketClass = new (url: string, protocols?: string | string[] | undefined) => WebSocket;
10
+ export declare class WebSocketClassProvider {
11
+ static ctor: WebSocketClass;
12
+ static set(ctor: WebSocketClass): void;
13
+ static get(): WebSocketClass;
14
+ }
15
+ /**
16
+ * Config object that is provided to the WebSocketTransport constructor.
17
+ */
18
+ export interface WebSocketTransportConfig<AdapterParams, ProviderDataMessage, CustomSettings extends SettingsMap> {
19
+ /** Endpoint to which to open the WS connection*/
20
+ url: string;
21
+ /** Map of handlers for different WS lifecycle events */
22
+ handlers: {
23
+ /**
24
+ * Handles when the WS is successfully opened
25
+ *
26
+ * @param wsConnection - the WebSocket with an established connection
27
+ * @returns an empty Promise, or void
28
+ */
29
+ open: (wsConnection: WebSocket, context: AdapterContext<CustomSettings>) => Promise<void> | void;
30
+ /**
31
+ * Handles when the WS receives a message
32
+ *
33
+ * @param message - the message received by the WS
34
+ * @param context - the background context for the Adapter
35
+ * @returns a list of cache entries of adapter responses to set in the cache
36
+ */
37
+ message: (message: ProviderDataMessage, context: AdapterContext<CustomSettings>) => ProviderResult<AdapterParams>[];
38
+ };
39
+ /** Map of "builders", functions that will be used to prepare specific WS messages */
40
+ builders: {
41
+ /**
42
+ * Builds a WS message that will be sent to subscribe to a specific feed
43
+ *
44
+ * @param params - the body of the adapter request
45
+ * @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
46
+ */
47
+ subscribeMessage: (params: AdapterParams) => unknown;
48
+ /**
49
+ * Builds a WS message that will be sent to unsubscribe to a specific feed
50
+ *
51
+ * @param params - the body of the adapter request
52
+ * @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
53
+ */
54
+ unsubscribeMessage: (params: AdapterParams) => unknown;
55
+ };
56
+ }
57
+ /**
58
+ * Transport implementation that takes incoming requests, adds them to an [[ExpiringSortedSet]] and,
59
+ * through a WebSocket connection, subscribes to the relevant feeds to populate the cache.
60
+ *
61
+ * @typeParam AdapterParams - interface for the adapter request body
62
+ * @typeParam ProviderDataMessage - interface for a WS message containing processable data (i.e. not part of open/close/login/etc)
63
+ */
64
+ export declare class WebSocketTransport<AdapterParams, ProviderDataMessage, CustomSettings extends SettingsMap> implements Transport<AdapterParams, null, CustomSettings> {
65
+ private config;
66
+ cache: Cache;
67
+ rateLimiter: BackgroundExecuteRateLimiter;
68
+ expiringSortedSet: ExpiringSortedSet<AdapterParams>;
69
+ localSubscriptions: AdapterParams[];
70
+ wsConnection: WebSocket;
71
+ constructor(config: WebSocketTransportConfig<AdapterParams, ProviderDataMessage, CustomSettings>);
72
+ initialize(dependencies: AdapterDependencies): Promise<void>;
73
+ hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean>;
74
+ setup(req: AdapterRequest<AdapterParams>): Promise<void>;
75
+ serializeMessage(payload: unknown): string;
76
+ deserializeMessage(data: WebSocket.Data): ProviderDataMessage;
77
+ establishWsConnection(context: AdapterContext<CustomSettings>): Promise<unknown>;
78
+ backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number>;
79
+ }
80
+ export {};
@@ -90,7 +90,7 @@ class WebSocketTransport {
90
90
  logger.debug('Successfully executed connection opened handler');
91
91
  // Record active ws connections by incrementing count on open
92
92
  // Using URL in label since connection_key is removed from v3
93
- transportMetrics.wsConnectionActive.labels({ url: this.config.url }).inc();
93
+ transportMetrics.wsConnectionActive.inc();
94
94
  resolve(true);
95
95
  });
96
96
  this.wsConnection.addEventListener('message', async (event) => {
@@ -114,7 +114,7 @@ class WebSocketTransport {
114
114
  logger.debug(`Closed websocket connection. Code: ${event.code} ; reason: ${event.reason?.toString()}`);
115
115
  // Record active ws connections by decrementing count on close
116
116
  // Using URL in label since connection_key is removed from v3
117
- transportMetrics.wsConnectionActive.labels({ url: this.config.url }).dec();
117
+ transportMetrics.wsConnectionActive.dec();
118
118
  });
119
119
  });
120
120
  }
@@ -124,12 +124,12 @@ class WebSocketTransport {
124
124
  const desiredSubs = this.expiringSortedSet.getAll();
125
125
  logger.debug('Generating delta (subscribes & unsubscribes)');
126
126
  // TODO: More efficient algorithm, this is really easy to read, but high(er) time complexity
127
- const subscribes = desiredSubs
128
- .filter((s) => !this.localSubscriptions.includes(s))
127
+ const subscribeParams = desiredSubs.filter((s) => !this.localSubscriptions.includes(s));
128
+ const subscribes = subscribeParams
129
129
  .map(this.config.builders.subscribeMessage)
130
130
  .map(this.serializeMessage);
131
- const unsubscribes = this.localSubscriptions
132
- .filter((s) => !desiredSubs.includes(s))
131
+ const unsubscribeParams = this.localSubscriptions.filter((s) => !desiredSubs.includes(s));
132
+ const unsubscribes = unsubscribeParams
133
133
  .map(this.config.builders.unsubscribeMessage)
134
134
  .map(this.serializeMessage);
135
135
  logger.debug(`${subscribes.length} new subscriptions; ${unsubscribes.length} to unsubscribe`);
@@ -152,20 +152,14 @@ class WebSocketTransport {
152
152
  await this.establishWsConnection(context);
153
153
  }
154
154
  // TODO: Close connection at some point?
155
- // TODO: Finish ws metrics
156
155
  logger.debug('Sending subs/unsubs if there are any');
157
156
  const messages = unsubscribes.concat(subscribes);
158
157
  for (const message of messages) {
159
158
  logger.trace(`Sending message: ${JSON.stringify(message)}`);
160
159
  this.wsConnection.send(message);
161
- // Record total number of ws messages sent
162
- // ws_message_total.labels(messageLabels(this.config.key)).inc()
163
160
  }
164
- // Record number of total subscriptions made
165
- // ws_subscription_total.labels(subscriptionLabels(this.config.key)).inc(subscribes.length())
166
- // Record number of active ws subscriptions
167
- // const activeSubCount = desiredSubs ? desiredSubs.length : 0
168
- // ws_subscription_active.labels(subscriptionLabels(this.config.key)).set(activeSubCount)
161
+ // Record WS message and subscription metrics
162
+ transportMetrics.recordWsMessageMetrics(context, subscribeParams, unsubscribeParams);
169
163
  logger.debug('Setting local state to cache value');
170
164
  this.localSubscriptions = desiredSubs;
171
165
  logger.debug('Background execute complete');
@@ -0,0 +1,21 @@
1
+ /**
2
+ * An object describing an entry in the expiring sorted set.
3
+ * @typeParam T - the type of the entry's value
4
+ */
5
+ interface ExpiringSortedSetEntry<T> {
6
+ value: T;
7
+ expirationTimestamp: number;
8
+ }
9
+ /**
10
+ * This class implements a set of unique items, each of which has an expiration timestamp.
11
+ * On reads, items that have expired will be deleted from the set and not returned.
12
+ *
13
+ * @typeParam T - the type of the set entries' values
14
+ */
15
+ export declare class ExpiringSortedSet<T> {
16
+ map: Map<string, ExpiringSortedSetEntry<T>>;
17
+ add(key: string, value: T, ttl: number): void;
18
+ get(key: string): T | undefined;
19
+ getAll(): T[];
20
+ }
21
+ export {};
File without changes
@@ -0,0 +1,11 @@
1
+ export * from './request';
2
+ export * from './logger';
3
+ export * from './expiring-sorted-set';
4
+ /**
5
+ * Sleeps for the provided number of milliseconds
6
+ * @param ms - The number of milliseconds to sleep for
7
+ * @returns a Promise that resolves once the specified time passes
8
+ */
9
+ export declare const sleep: (ms: number) => Promise<void>;
10
+ export declare const isObject: (o: unknown) => boolean;
11
+ export declare const isArray: (o: unknown) => boolean;
File without changes
@@ -0,0 +1,42 @@
1
+ /// <reference types="node" />
2
+ import pino from 'pino';
3
+ import { AdapterRequest } from './request';
4
+ import { FastifyReply, HookHandlerDoneFunction } from 'fastify';
5
+ import { AsyncLocalStorage } from 'node:async_hooks';
6
+ export declare const asyncLocalStorage: AsyncLocalStorage<unknown>;
7
+ export declare type Store = {
8
+ correlationId: string;
9
+ };
10
+ /**
11
+ * Instead of using a global logger instance, we want to force using a child logger
12
+ * with a specific layer set in it, so that we can filter logs by where they're output from.
13
+ *
14
+ * Details on what each log level represents:
15
+ * "trace": Forensic debugging of issues on a local machine.
16
+ * "debug": Detailed logging level to get more context from users on their environments.
17
+ * "info": High-level informational messages, to describe at a glance the high level state of the system.
18
+ * "warn": A mild error occurred that might require non-urgent action.
19
+ * "error": An unexpected error occurred during the regular operation of a well-maintained EA.
20
+ * "fatal": The EA encountered an unrecoverable problem and had to exit.
21
+ *
22
+ * Full reference this is based on can be found at
23
+ * https://github.com/smartcontractkit/documentation/blob/main/docs/Node%20Operators/configuration-variables.md#log_level
24
+ *
25
+ * @param layer - the layer name to include in the logs (e.g. "SomeMiddleware", "RedisCache", etc.)
26
+ * @returns a layer specific logger
27
+ */
28
+ export declare const makeLogger: (layer: string) => pino.Logger<{
29
+ level: string;
30
+ mixin(): {};
31
+ transport: {
32
+ target: string;
33
+ options: {
34
+ levelFirst: boolean;
35
+ levelLabel: string;
36
+ ignore: string;
37
+ messageFormat: string;
38
+ translateTime: string;
39
+ };
40
+ } | undefined;
41
+ } & pino.ChildLoggerOptions>;
42
+ export declare const loggingContextMiddleware: (req: AdapterRequest, res: FastifyReply, done: HookHandlerDoneFunction) => void;
File without changes
@@ -0,0 +1,55 @@
1
+ import { FastifyReply, FastifyRequest, HookHandlerDoneFunction } from 'fastify';
2
+ import { InitializedAdapter } from '../adapter';
3
+ import { AdapterError } from '../validation/error';
4
+ declare module 'fastify' {
5
+ interface FastifyRequest {
6
+ requestContext: AdapterRequestContext;
7
+ }
8
+ }
9
+ export interface AdapterRequestBody<T = AdapterRequestData> {
10
+ endpoint?: string;
11
+ data: T;
12
+ id?: string;
13
+ }
14
+ export declare type AdapterRequestContext<T = AdapterRequestData> = {
15
+ id: string;
16
+ endpointName: string;
17
+ cacheKey: string;
18
+ data: T;
19
+ meta?: AdapterRequestMeta;
20
+ };
21
+ export declare type AdapterRouteGeneric<T = AdapterRequestData> = {
22
+ Body: AdapterRequestBody<T>;
23
+ };
24
+ export declare type AdapterRequest<T = AdapterRequestData> = FastifyRequest<AdapterRouteGeneric<T>> & {
25
+ requestContext: AdapterRequestContext<T>;
26
+ };
27
+ /**
28
+ * Meta info that pertains to exposing metrics
29
+ */
30
+ export interface AdapterRequestMeta {
31
+ metrics?: AdapterMetricsMeta;
32
+ error?: AdapterError | Error;
33
+ }
34
+ /**
35
+ * Meta info that pertains to exposing metrics
36
+ */
37
+ export interface AdapterMetricsMeta {
38
+ feedId?: string;
39
+ cacheHit?: boolean;
40
+ }
41
+ export declare type AdapterRequestData = Record<string, unknown>;
42
+ export interface ProviderResult<Params> {
43
+ params: Params;
44
+ value: unknown;
45
+ }
46
+ export declare type AdapterResponse<T = unknown> = {
47
+ statusCode: number;
48
+ data: T;
49
+ result: unknown;
50
+ maxAge?: number;
51
+ meta?: AdapterRequestMeta;
52
+ providerStatusCode?: number;
53
+ };
54
+ export declare type Middleware = ((req: AdapterRequest, reply: FastifyReply, done: HookHandlerDoneFunction) => FastifyReply | void) | ((req: AdapterRequest, reply: FastifyReply) => Promise<FastifyReply | void>);
55
+ export declare type AdapterMiddlewareBuilder = (adapter: InitializedAdapter) => Middleware;
File without changes
@@ -0,0 +1,50 @@
1
+ import { HttpRequestType } from "../metrics/constants";
2
+ declare type ErrorBasic = {
3
+ name: string;
4
+ message: string;
5
+ };
6
+ declare type ErrorFull = ErrorBasic & {
7
+ stack: string;
8
+ cause: string;
9
+ };
10
+ export declare type AdapterErrorResponse = {
11
+ jobRunID: string;
12
+ status: string;
13
+ statusCode: number;
14
+ providerStatusCode?: number;
15
+ error: ErrorBasic | ErrorFull;
16
+ };
17
+ export declare class AdapterError extends Error {
18
+ jobRunID: string;
19
+ status: string;
20
+ statusCode: number;
21
+ cause: any;
22
+ url?: string;
23
+ errorResponse: any;
24
+ feedID?: string;
25
+ providerStatusCode?: number;
26
+ metricsLabel?: HttpRequestType;
27
+ name: string;
28
+ message: string;
29
+ constructor({ jobRunID, status, statusCode, name, message, cause, url, errorResponse, feedID, providerStatusCode, metricsLabel }: Partial<AdapterError>);
30
+ toJSONResponse(): AdapterErrorResponse;
31
+ }
32
+ export declare class AdapterInputError extends AdapterError {
33
+ constructor(input: Partial<AdapterError>);
34
+ }
35
+ export declare class AdapterRateLimitError extends AdapterError {
36
+ constructor(input: Partial<AdapterError>);
37
+ }
38
+ export declare class AdapterTimeoutError extends AdapterError {
39
+ constructor(input: Partial<AdapterError>);
40
+ }
41
+ export declare class AdapterDataProviderError extends AdapterError {
42
+ constructor(input: Partial<AdapterError>);
43
+ }
44
+ export declare class AdapterConnectionError extends AdapterError {
45
+ constructor(input: Partial<AdapterError>);
46
+ }
47
+ export declare class AdapterCustomError extends AdapterError {
48
+ constructor(input: Partial<AdapterError>);
49
+ }
50
+ export {};