@chainlink/external-adapter-framework 0.14.4 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter/basic.js +7 -6
- package/adapter/basic.js.map +1 -1
- package/adapter/types.d.ts +5 -4
- package/background-executor.js +4 -4
- package/background-executor.js.map +1 -1
- package/cache/factory.d.ts +4 -1
- package/cache/factory.js +2 -2
- package/cache/factory.js.map +1 -1
- package/cache/local.d.ts +26 -4
- package/cache/local.js +113 -22
- package/cache/local.js.map +1 -1
- package/config/index.d.ts +30 -29
- package/config/index.js +37 -64
- package/config/index.js.map +1 -1
- package/package.json +4 -4
- package/rate-limiting/index.d.ts +2 -15
- package/rate-limiting/index.js +1 -2
- package/rate-limiting/index.js.map +1 -1
- package/rate-limiting/request/simple-counting.d.ts +1 -1
- package/rate-limiting/request/simple-counting.js +12 -8
- package/rate-limiting/request/simple-counting.js.map +1 -1
- package/rate-limiting/simple-counting.d.ts +20 -0
- package/rate-limiting/simple-counting.js +67 -0
- package/rate-limiting/simple-counting.js.map +1 -0
- package/transports/abstract/subscription.d.ts +1 -3
- package/transports/abstract/subscription.js +0 -1
- package/transports/abstract/subscription.js.map +1 -1
- package/transports/batch-warming.d.ts +20 -8
- package/transports/batch-warming.js +55 -68
- package/transports/batch-warming.js.map +1 -1
- package/transports/http.d.ts +90 -0
- package/transports/http.js +172 -0
- package/transports/http.js.map +1 -0
- package/transports/index.d.ts +2 -3
- package/transports/index.js +1 -2
- package/transports/index.js.map +1 -1
- package/transports/metrics.d.ts +0 -6
- package/transports/metrics.js +1 -18
- package/transports/metrics.js.map +1 -1
- package/transports/sse.d.ts +3 -3
- package/transports/sse.js +6 -3
- package/transports/sse.js.map +1 -1
- package/transports/websocket.d.ts +2 -2
- package/transports/websocket.js +3 -0
- package/transports/websocket.js.map +1 -1
- package/util/index.d.ts +1 -1
- package/util/index.js +1 -1
- package/util/index.js.map +1 -1
- package/util/logger.d.ts +1 -1
- package/util/metrics.d.ts +9 -0
- package/util/metrics.js +53 -0
- package/util/metrics.js.map +1 -0
- package/util/request.d.ts +25 -25
- package/util/requester.d.ts +42 -0
- package/util/requester.js +233 -0
- package/util/requester.js.map +1 -0
- package/util/test-payload-loader.d.ts +1 -1
- package/util/types.d.ts +217 -0
- package/util/types.js +3 -0
- package/util/types.js.map +1 -0
- package/validation/error.d.ts +5 -2
- package/validation/error.js +4 -2
- package/validation/error.js.map +1 -1
- package/validation/index.d.ts +1 -1
- package/validation/index.js.map +1 -1
- package/validation/input-validator.d.ts +3 -0
- package/validation/input-validator.js +31 -1
- package/validation/input-validator.js.map +1 -1
- package/validation/utils.d.ts +1 -0
- package/validation/utils.js +10 -0
- package/validation/utils.js.map +1 -1
|
@@ -23,13 +23,14 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.
|
|
26
|
+
exports.HttpTransport = void 0;
|
|
27
|
+
const axios_1 = require("axios");
|
|
27
28
|
const cacheMetrics = __importStar(require("../cache/metrics"));
|
|
28
29
|
const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
|
|
29
30
|
const util_1 = require("../util");
|
|
31
|
+
const requester_1 = require("../util/requester");
|
|
30
32
|
const error_1 = require("../validation/error");
|
|
31
33
|
const subscription_1 = require("./abstract/subscription");
|
|
32
|
-
const util_2 = require("./util");
|
|
33
34
|
const WARMUP_BATCH_REQUEST_ID = '9002';
|
|
34
35
|
const logger = (0, util_1.makeLogger)('BatchWarmingTransport');
|
|
35
36
|
/**
|
|
@@ -42,7 +43,7 @@ const logger = (0, util_1.makeLogger)('BatchWarmingTransport');
|
|
|
42
43
|
*
|
|
43
44
|
* @typeParam T - all types related to the [[Transport]]
|
|
44
45
|
*/
|
|
45
|
-
class
|
|
46
|
+
class HttpTransport extends subscription_1.SubscriptionTransport {
|
|
46
47
|
constructor(config) {
|
|
47
48
|
super();
|
|
48
49
|
this.config = config;
|
|
@@ -50,23 +51,13 @@ class BatchWarmingTransport extends subscription_1.SubscriptionTransport {
|
|
|
50
51
|
// Used for recording the cache warmer active metrics accurately
|
|
51
52
|
this.WARMER_ACTIVE = false;
|
|
52
53
|
}
|
|
54
|
+
async initialize(dependencies, config, endpointName) {
|
|
55
|
+
await super.initialize(dependencies, config, endpointName);
|
|
56
|
+
this.requester = new requester_1.Requester(dependencies.requestRateLimiter, config);
|
|
57
|
+
}
|
|
53
58
|
getSubscriptionTtlFromConfig(config) {
|
|
54
59
|
return config.WARMUP_SUBSCRIPTION_TTL;
|
|
55
60
|
}
|
|
56
|
-
async registerRequest(req, config) {
|
|
57
|
-
if (config.BATCH_TRANSPORT_SETUP_VALIDATION) {
|
|
58
|
-
const response = await this.makeRequest([req.requestContext.data], config);
|
|
59
|
-
if (!response.results.length) {
|
|
60
|
-
throw new error_1.AdapterError({
|
|
61
|
-
statusCode: 200,
|
|
62
|
-
providerStatusCode: response.providerResponse.status,
|
|
63
|
-
message: response.providerResponse.message ||
|
|
64
|
-
'There was an error while validating the incoming request before adding to the batch subscription set',
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
return super.registerRequest(req, config);
|
|
69
|
-
}
|
|
70
61
|
async backgroundHandler(context, entries) {
|
|
71
62
|
if (!entries.length) {
|
|
72
63
|
logger.debug('No entries in batch warming set, skipping');
|
|
@@ -82,69 +73,65 @@ class BatchWarmingTransport extends subscription_1.SubscriptionTransport {
|
|
|
82
73
|
cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc();
|
|
83
74
|
this.WARMER_ACTIVE = true;
|
|
84
75
|
}
|
|
85
|
-
|
|
86
|
-
|
|
76
|
+
logger.trace(`Have ${entries.length} entries in batch, preparing request...`);
|
|
77
|
+
const rawRequests = this.config.prepareRequests(entries, context.adapterConfig);
|
|
78
|
+
const requests = Array.isArray(rawRequests) ? rawRequests : [rawRequests];
|
|
79
|
+
logger.trace(`Queueing ${requests.length} requests`);
|
|
80
|
+
await Promise.all(requests.map((r) => this.handleRequest(r, context.adapterConfig)));
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
async handleRequest(requestConfig, adapterConfig) {
|
|
84
|
+
const results = await this.makeRequest(requestConfig, adapterConfig);
|
|
85
|
+
if (!results.length) {
|
|
87
86
|
return;
|
|
88
87
|
}
|
|
89
88
|
logger.debug('Setting adapter responses in cache');
|
|
90
|
-
await this.responseCache.write(
|
|
91
|
-
// Record cost of data provider call
|
|
92
|
-
const cost = rateLimitMetrics.retrieveCost(response.providerResponse.data);
|
|
93
|
-
rateLimitMetrics.rateLimitCreditsSpentTotal
|
|
94
|
-
.labels({
|
|
95
|
-
feed_id: 'N/A',
|
|
96
|
-
participant_id: WARMUP_BATCH_REQUEST_ID,
|
|
97
|
-
})
|
|
98
|
-
.inc(cost);
|
|
99
|
-
return;
|
|
89
|
+
await this.responseCache.write(results);
|
|
100
90
|
}
|
|
101
|
-
async makeRequest(
|
|
102
|
-
logger.trace(`Have ${entries.length} entries in batch, preparing request...`);
|
|
103
|
-
const request = this.config.prepareRequest(entries, config);
|
|
104
|
-
logger.trace(`Sending request to data provider: ${JSON.stringify(request)}`);
|
|
105
|
-
const providerDataRequested = Date.now();
|
|
106
|
-
let providerResponse;
|
|
91
|
+
async makeRequest(requestConfig, adapterConfig) {
|
|
107
92
|
try {
|
|
108
|
-
|
|
93
|
+
const requesterResult = await this.requester.request(requestConfig.params.map((p) => JSON.stringify(p)).join('|'), requestConfig.request);
|
|
94
|
+
// Parse responses and apply timestamps
|
|
95
|
+
const results = this.config
|
|
96
|
+
.parseResponse(requestConfig.params, requesterResult.response, adapterConfig)
|
|
97
|
+
.map((r) => {
|
|
98
|
+
const result = r;
|
|
99
|
+
const partialResponse = r.response;
|
|
100
|
+
result.response.timestamps = {
|
|
101
|
+
...requesterResult.timestamps,
|
|
102
|
+
providerIndicatedTime: partialResponse.timestamps?.providerIndicatedTime,
|
|
103
|
+
};
|
|
104
|
+
return result;
|
|
105
|
+
});
|
|
106
|
+
// Record cost of data provider call
|
|
107
|
+
const cost = rateLimitMetrics.retrieveCost(requesterResult.response.data);
|
|
108
|
+
rateLimitMetrics.rateLimitCreditsSpentTotal
|
|
109
|
+
.labels({
|
|
110
|
+
feed_id: 'N/A',
|
|
111
|
+
participant_id: WARMUP_BATCH_REQUEST_ID,
|
|
112
|
+
})
|
|
113
|
+
.inc(cost);
|
|
114
|
+
return results;
|
|
109
115
|
}
|
|
110
116
|
catch (e) {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
results: entries.map((entry) => ({
|
|
117
|
+
if (e instanceof error_1.AdapterDataProviderError && e.cause instanceof axios_1.AxiosError) {
|
|
118
|
+
const err = e;
|
|
119
|
+
const cause = err.cause;
|
|
120
|
+
return requestConfig.params.map((entry) => ({
|
|
116
121
|
params: entry,
|
|
117
122
|
response: {
|
|
118
|
-
errorMessage: `Provider request failed with status ${
|
|
123
|
+
errorMessage: `Provider request failed with status ${cause.status}: "${cause.response?.data}"`,
|
|
119
124
|
statusCode: 502,
|
|
120
|
-
timestamps:
|
|
121
|
-
providerDataRequested,
|
|
122
|
-
providerDataReceived,
|
|
123
|
-
providerIndicatedTime: undefined,
|
|
124
|
-
},
|
|
125
|
+
timestamps: err.timestamps,
|
|
125
126
|
},
|
|
126
|
-
}))
|
|
127
|
-
|
|
128
|
-
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
logger.error(e);
|
|
131
|
+
return [];
|
|
132
|
+
}
|
|
129
133
|
}
|
|
130
|
-
const providerDataReceived = Date.now();
|
|
131
|
-
// Parse responses and apply timestamps
|
|
132
|
-
const results = this.config.parseResponse(entries, providerResponse, config).map((r) => {
|
|
133
|
-
const result = r;
|
|
134
|
-
const partialResponse = r.response;
|
|
135
|
-
result.response.timestamps = {
|
|
136
|
-
providerDataRequested,
|
|
137
|
-
providerDataReceived,
|
|
138
|
-
providerIndicatedTime: partialResponse.timestamps?.providerIndicatedTime,
|
|
139
|
-
};
|
|
140
|
-
return result;
|
|
141
|
-
});
|
|
142
|
-
logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`);
|
|
143
|
-
return {
|
|
144
|
-
results,
|
|
145
|
-
providerResponse,
|
|
146
|
-
};
|
|
147
134
|
}
|
|
148
135
|
}
|
|
149
|
-
exports.
|
|
136
|
+
exports.HttpTransport = HttpTransport;
|
|
150
137
|
//# sourceMappingURL=batch-warming.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-warming.js","sourceRoot":"","sources":["../../../src/transports/batch-warming.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"batch-warming.js","sourceRoot":"","sources":["../../../src/transports/batch-warming.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iCAAqE;AAErE,+DAAgD;AAEhD,2EAA4D;AAC5D,kCAAoC;AAMpC,iDAA6C;AAC7C,+CAA8D;AAE9D,0DAA+D;AAE/D,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAEtC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,uBAAuB,CAAC,CAAA;AAkDlD;;;;;;;;;GASG;AACH,MAAa,aAA+C,SAAQ,oCAAwB;IAM1F,YAAoB,MAA8B;QAChD,KAAK,EAAE,CAAA;QADW,WAAM,GAAN,MAAM,CAAwB;QALlD,uGAAuG;QACvG,gEAAgE;QAChE,kBAAa,GAAG,KAAK,CAAA;IAKrB,CAAC;IAEQ,KAAK,CAAC,UAAU,CACvB,YAAsC,EACtC,MAA0C,EAC1C,YAAoB;QAEpB,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,CAAA;QAC1D,IAAI,CAAC,SAAS,GAAG,IAAI,qBAAS,CAAC,YAAY,CAAC,kBAAkB,EAAE,MAAuB,CAAC,CAAA;IAC1F,CAAC;IAED,4BAA4B,CAAC,MAA0C;QACrE,OAAO,MAAM,CAAC,uBAAuB,CAAA;IACvC,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,OAA2B,EAC3B,OAAiC;QAEjC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACnB,MAAM,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAA;YACzD,IAAI,IAAI,CAAC,aAAa,EAAE;gBACtB,yEAAyE;gBACzE,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAA;gBACjE,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;aAC3B;YACD,OAAM;SACP;aAAM,IAAI,IAAI,CAAC,aAAa,KAAK,KAAK,EAAE;YACvC,4EAA4E;YAC5E,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAA;YACjE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;SAC1B;QAED,MAAM,CAAC,KAAK,CAAC,QAAQ,OAAO,CAAC,MAAM,yCAAyC,CAAC,CAAA;QAC7E,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;QAC/E,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAA;QAEzE,MAAM,CAAC,KAAK,CAAC,YAAY,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAA;QACpD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAA;QAEpF,OAAM;IACR,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,aAAuC,EACvC,aAAiD;QAEjD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,aAAa,CAAC,CAAA;QAEpE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACnB,OAAM;SACP;QAED,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAClD,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IACzC,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,aAAuC,EACvC,aAAiD;QAEjD,IAAI;YACF,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAClD,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAC5D,aAAa,CAAC,OAAO,CACtB,CAAA;YAED,uCAAuC;YACvC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM;iBACxB,aAAa,CAAC,aAAa,CAAC,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC;iBAC5E,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACT,MAAM,MAAM,GAAG,CAAiC,CAAA;gBAChD,MAAM,eAAe,GAAG,CAAC,CAAC,QAAoD,CAAA;gBAC9E,MAAM,CAAC,QAAQ,CAAC,UAAU,GAAG;oBAC3B,GAAG,eAAe,CAAC,UAAU;oBAC7B,qBAAqB,EAAE,eAAe,CAAC,UAAU,EAAE,qBAAqB;iBACzE,CAAA;gBACD,OAAO,MAAM,CAAA;YACf,CAAC,CAAC,CAAA;YAEJ,oCAAoC;YACpC,MAAM,IAAI,GAAG,gBAAgB,CAAC,YAAY,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;YACzE,gBAAgB,CAAC,0BAA0B;iBACxC,MAAM,CAAC;gBACN,OAAO,EAAE,KAAK;gBACd,cAAc,EAAE,uBAAuB;aACxC,CAAC;iBACD,GAAG,CAAC,IAAI,CAAC,CAAA;YAEZ,OAAO,OAAO,CAAA;SACf;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,YAAY,gCAAwB,IAAI,CAAC,CAAC,KAAK,YAAY,kBAAU,EAAE;gBAC1E,MAAM,GAAG,GAAG,CAA6B,CAAA;gBACzC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAmB,CAAA;gBACrC,OAAO,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1C,MAAM,EAAE,KAAK;oBACb,QAAQ,EAAE;wBACR,YAAY,EAAE,uCAAuC,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,QAAQ,EAAE,IAAI,GAAG;wBAC9F,UAAU,EAAE,GAAG;wBACf,UAAU,EAAE,GAAG,CAAC,UAAU;qBAC3B;iBACF,CAAC,CAAC,CAAA;aACJ;iBAAM;gBACL,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;gBACf,OAAO,EAAE,CAAA;aACV;SACF;IACH,CAAC;CACF;AApHD,sCAoHC"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { AxiosRequestConfig, AxiosResponse } from 'axios';
|
|
2
|
+
import { EndpointContext } from '../adapter';
|
|
3
|
+
import { AdapterConfig } from '../config';
|
|
4
|
+
import { ProviderResult } from '../util/types';
|
|
5
|
+
import { Requester } from '../util/requester';
|
|
6
|
+
import { TransportDependencies, TransportGenerics } from '.';
|
|
7
|
+
import { SubscriptionTransport } from './abstract/subscription';
|
|
8
|
+
/**
|
|
9
|
+
* Helper struct type that will be used to pass types to the generic parameters of a Transport.
|
|
10
|
+
* Extends the common TransportGenerics, adding Provider specific types for this Batch endpoint.
|
|
11
|
+
*/
|
|
12
|
+
type HttpTransportGenerics = TransportGenerics & {
|
|
13
|
+
/**
|
|
14
|
+
* Type details for any provider specific interfaces.
|
|
15
|
+
*/
|
|
16
|
+
Provider: {
|
|
17
|
+
/**
|
|
18
|
+
* Structure of the body of the request that will be sent to the data provider.
|
|
19
|
+
*/
|
|
20
|
+
RequestBody: unknown;
|
|
21
|
+
/**
|
|
22
|
+
* Structure for the body of the response coming from the data provider.
|
|
23
|
+
*/
|
|
24
|
+
ResponseBody: unknown;
|
|
25
|
+
};
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Structure containing the association between EA params and a provider request.
|
|
29
|
+
*/
|
|
30
|
+
type ProviderRequestConfig<T extends HttpTransportGenerics> = {
|
|
31
|
+
/** The input paramters for requests that will get responses from the request in this struct */
|
|
32
|
+
params: T['Request']['Params'][];
|
|
33
|
+
/** The request that will be sent to the data provider to fetch values for the params in this struct */
|
|
34
|
+
request: AxiosRequestConfig<T['Provider']['RequestBody']>;
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* Config object that is provided to the HttpTransport constructor.
|
|
38
|
+
*/
|
|
39
|
+
export interface HttpTransportConfig<T extends HttpTransportGenerics> {
|
|
40
|
+
/**
|
|
41
|
+
* This method should take the list of currently valid input parameters in the subscription set,
|
|
42
|
+
* and build however many requests to the data provider are necessary to fullfill all the information.
|
|
43
|
+
* Some constranints:
|
|
44
|
+
* - Each request should be tied to at least one input parameter
|
|
45
|
+
* - Input parameters should be tied to only one request. You can technically avoid this, but there will be no way
|
|
46
|
+
* to consolidate many of them since the parseResponse method is called independently for each of them.
|
|
47
|
+
*
|
|
48
|
+
* @param params - the list of non-expired input parameters sent to this Adapter
|
|
49
|
+
* @param config - the config for this Adapter
|
|
50
|
+
* @returns one or multiple request configs
|
|
51
|
+
*/
|
|
52
|
+
prepareRequests: (params: T['Request']['Params'][], config: AdapterConfig<T['CustomSettings']>) => ProviderRequestConfig<T> | ProviderRequestConfig<T>[];
|
|
53
|
+
/**
|
|
54
|
+
* This method should take the incoming response from the data provider, and using that and the params, build a
|
|
55
|
+
* list of ProviderResults that will be stored in the response cache for this endpoint.
|
|
56
|
+
* Some notes:
|
|
57
|
+
* - The results don't technically need to be related to the requested params; you could use the response
|
|
58
|
+
* and store more items in the cache than what was requested from the EA (be mindful and do this conservatively if at all)
|
|
59
|
+
* - If no useful information was received, you can return an empty list
|
|
60
|
+
* - Alternatively, do make use of the fact that a ProviderResult is not necessarily a successful one, and you can store errors too
|
|
61
|
+
*
|
|
62
|
+
* @param params - the list of input parameters that should be fulfilled by this incoming provider response
|
|
63
|
+
* @param res - the response from the data provider
|
|
64
|
+
* @param config - the config for this Adapter
|
|
65
|
+
* @returns a list of ProviderResults
|
|
66
|
+
*/
|
|
67
|
+
parseResponse: (params: T['Request']['Params'][], res: AxiosResponse<T['Provider']['ResponseBody']>, config: AdapterConfig<T['CustomSettings']>) => ProviderResult<T>[];
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Transport implementation that takes incoming batches of requests and keeps a warm cache of values.
|
|
71
|
+
* Within the setup function, adapter params are added to a set that also keeps track and expires values.
|
|
72
|
+
* In the background execute, the list of non-expired items in the set is fetched.
|
|
73
|
+
* Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
|
|
74
|
+
* The Data Provider response is then passed through the `parseResponse` function to create a [[CacheEntry]] list.
|
|
75
|
+
* Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
|
|
76
|
+
*
|
|
77
|
+
* @typeParam T - all types related to the [[Transport]]
|
|
78
|
+
*/
|
|
79
|
+
export declare class HttpTransport<T extends HttpTransportGenerics> extends SubscriptionTransport<T> {
|
|
80
|
+
private config;
|
|
81
|
+
WARMER_ACTIVE: boolean;
|
|
82
|
+
requester: Requester;
|
|
83
|
+
constructor(config: HttpTransportConfig<T>);
|
|
84
|
+
initialize(dependencies: TransportDependencies<T>, config: AdapterConfig<T['CustomSettings']>, endpointName: string): Promise<void>;
|
|
85
|
+
getSubscriptionTtlFromConfig(config: AdapterConfig<T['CustomSettings']>): number;
|
|
86
|
+
backgroundHandler(context: EndpointContext<T>, entries: T['Request']['Params'][]): Promise<void>;
|
|
87
|
+
private handleRequest;
|
|
88
|
+
private makeRequest;
|
|
89
|
+
}
|
|
90
|
+
export {};
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.HttpTransport = void 0;
|
|
27
|
+
const axios_1 = require("axios");
|
|
28
|
+
const cacheMetrics = __importStar(require("../cache/metrics"));
|
|
29
|
+
const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
|
|
30
|
+
const util_1 = require("../util");
|
|
31
|
+
const error_1 = require("../validation/error");
|
|
32
|
+
const subscription_1 = require("./abstract/subscription");
|
|
33
|
+
const WARMUP_BATCH_REQUEST_ID = '9002';
|
|
34
|
+
const logger = (0, util_1.makeLogger)('HttpTransport');
|
|
35
|
+
/**
|
|
36
|
+
* Transport implementation that takes incoming batches of requests and keeps a warm cache of values.
|
|
37
|
+
* Within the setup function, adapter params are added to a set that also keeps track and expires values.
|
|
38
|
+
* In the background execute, the list of non-expired items in the set is fetched.
|
|
39
|
+
* Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
|
|
40
|
+
* The Data Provider response is then passed through the `parseResponse` function to create a [[CacheEntry]] list.
|
|
41
|
+
* Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
|
|
42
|
+
*
|
|
43
|
+
* @typeParam T - all types related to the [[Transport]]
|
|
44
|
+
*/
|
|
45
|
+
class HttpTransport extends subscription_1.SubscriptionTransport {
|
|
46
|
+
constructor(config) {
|
|
47
|
+
super();
|
|
48
|
+
this.config = config;
|
|
49
|
+
// Flag used to track whether the warmer has moved from having no entries to having some and vice versa
|
|
50
|
+
// Used for recording the cache warmer active metrics accurately
|
|
51
|
+
this.WARMER_ACTIVE = false;
|
|
52
|
+
}
|
|
53
|
+
async initialize(dependencies, config, endpointName) {
|
|
54
|
+
await super.initialize(dependencies, config, endpointName);
|
|
55
|
+
this.requester = dependencies.requester;
|
|
56
|
+
}
|
|
57
|
+
getSubscriptionTtlFromConfig(config) {
|
|
58
|
+
return config.WARMUP_SUBSCRIPTION_TTL;
|
|
59
|
+
}
|
|
60
|
+
async backgroundHandler(context, entries) {
|
|
61
|
+
if (!entries.length) {
|
|
62
|
+
logger.debug(`No entries in subscription set, sleeping for ${context.adapterConfig.BACKGROUND_EXECUTE_MS_HTTP}ms...`);
|
|
63
|
+
if (this.WARMER_ACTIVE) {
|
|
64
|
+
// Decrement count when warmer changed from having entries to having none
|
|
65
|
+
cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec();
|
|
66
|
+
this.WARMER_ACTIVE = false;
|
|
67
|
+
}
|
|
68
|
+
await (0, util_1.sleep)(context.adapterConfig.BACKGROUND_EXECUTE_MS_HTTP);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
else if (this.WARMER_ACTIVE === false) {
|
|
72
|
+
// Increment count when warmer changed from having no entries to having some
|
|
73
|
+
cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc();
|
|
74
|
+
this.WARMER_ACTIVE = true;
|
|
75
|
+
}
|
|
76
|
+
logger.trace(`Have ${entries.length} entries in batch, preparing requests...`);
|
|
77
|
+
const rawRequests = this.config.prepareRequests(entries, context.adapterConfig);
|
|
78
|
+
const requests = Array.isArray(rawRequests) ? rawRequests : [rawRequests];
|
|
79
|
+
// We're awaiting these promises because although we have request coalescing, new entries
|
|
80
|
+
// could be added to the subscription set if not blocking this operation, so the next time the
|
|
81
|
+
// background execute is triggered if the request is for a fully batched endpoint, we could end up
|
|
82
|
+
// with the full combination of possible params within the request queue
|
|
83
|
+
logger.trace(`Sending ${requests.length} requests...`);
|
|
84
|
+
const start = Date.now();
|
|
85
|
+
await Promise.all(requests.map((r) => this.handleRequest(r, context.adapterConfig)));
|
|
86
|
+
const duration = Date.now() - start;
|
|
87
|
+
logger.trace(`All requests in the background execute were completed`);
|
|
88
|
+
// These logs will surface warnings that operators should take action on, in case the execution of all
|
|
89
|
+
// requests is taking too long so that entries could have expired within this timeframe
|
|
90
|
+
if (duration > context.adapterConfig.WARMUP_SUBSCRIPTION_TTL) {
|
|
91
|
+
logger.warn(`Background execution of all HTTP requests in a batch took ${duration},\
|
|
92
|
+
which is longer than the subscription TTL (${context.adapterConfig.WARMUP_SUBSCRIPTION_TTL}).\
|
|
93
|
+
This might be due to insufficient speed on the selected API tier, please check metrics and logs to confirm and consider moving to a faster tier.`);
|
|
94
|
+
}
|
|
95
|
+
if (duration > context.adapterConfig.CACHE_MAX_AGE) {
|
|
96
|
+
logger.warn(`Background execution of all HTTP requests in a batch took ${duration},\
|
|
97
|
+
which is longer than the max cache age (${context.adapterConfig.CACHE_MAX_AGE}).\
|
|
98
|
+
This might be due to insufficient speed on the selected API tier, please check metrics and logs to confirm and consider moving to a faster tier.`);
|
|
99
|
+
}
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
async handleRequest(requestConfig, adapterConfig) {
|
|
103
|
+
const results = await this.makeRequest(requestConfig, adapterConfig);
|
|
104
|
+
if (!results.length) {
|
|
105
|
+
logger.trace('Got no results from the request.');
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
logger.debug('Setting adapter responses in cache');
|
|
109
|
+
await this.responseCache.write(results);
|
|
110
|
+
}
|
|
111
|
+
async makeRequest(requestConfig, adapterConfig) {
|
|
112
|
+
try {
|
|
113
|
+
const requesterResult = await this.requester.request(requestConfig.params.map((p) => JSON.stringify(p)).join('|'), requestConfig.request);
|
|
114
|
+
// Parse responses and apply timestamps
|
|
115
|
+
const results = this.config
|
|
116
|
+
.parseResponse(requestConfig.params, requesterResult.response, adapterConfig)
|
|
117
|
+
.map((r) => {
|
|
118
|
+
const result = r;
|
|
119
|
+
const partialResponse = r.response;
|
|
120
|
+
result.response.timestamps = {
|
|
121
|
+
...requesterResult.timestamps,
|
|
122
|
+
providerIndicatedTime: partialResponse.timestamps?.providerIndicatedTime,
|
|
123
|
+
};
|
|
124
|
+
return result;
|
|
125
|
+
});
|
|
126
|
+
// Record cost of data provider call
|
|
127
|
+
const cost = rateLimitMetrics.retrieveCost(requesterResult.response.data);
|
|
128
|
+
rateLimitMetrics.rateLimitCreditsSpentTotal
|
|
129
|
+
.labels({
|
|
130
|
+
feed_id: 'N/A',
|
|
131
|
+
participant_id: WARMUP_BATCH_REQUEST_ID,
|
|
132
|
+
})
|
|
133
|
+
.inc(cost);
|
|
134
|
+
return results;
|
|
135
|
+
}
|
|
136
|
+
catch (e) {
|
|
137
|
+
if (e instanceof error_1.AdapterDataProviderError && e.cause instanceof axios_1.AxiosError) {
|
|
138
|
+
const err = e;
|
|
139
|
+
const cause = err.cause;
|
|
140
|
+
return requestConfig.params.map((entry) => ({
|
|
141
|
+
params: entry,
|
|
142
|
+
response: {
|
|
143
|
+
errorMessage: `Provider request failed with status ${cause.status}: "${cause.response?.data}"`,
|
|
144
|
+
statusCode: 502,
|
|
145
|
+
timestamps: err.timestamps,
|
|
146
|
+
},
|
|
147
|
+
}));
|
|
148
|
+
}
|
|
149
|
+
else if (e instanceof error_1.AdapterRateLimitError) {
|
|
150
|
+
const err = e;
|
|
151
|
+
return requestConfig.params.map((entry) => ({
|
|
152
|
+
params: entry,
|
|
153
|
+
response: {
|
|
154
|
+
errorMessage: err.message,
|
|
155
|
+
statusCode: 429,
|
|
156
|
+
timestamps: {
|
|
157
|
+
providerDataReceived: 0,
|
|
158
|
+
providerDataRequested: 0,
|
|
159
|
+
providerIndicatedTime: undefined,
|
|
160
|
+
},
|
|
161
|
+
},
|
|
162
|
+
}));
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
logger.error(e);
|
|
166
|
+
return [];
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
exports.HttpTransport = HttpTransport;
|
|
172
|
+
//# sourceMappingURL=http.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.js","sourceRoot":"","sources":["../../../src/transports/http.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iCAAqE;AAErE,+DAAgD;AAEhD,2EAA4D;AAC5D,kCAA2C;AAG3C,+CAAqF;AAErF,0DAA+D;AAE/D,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAEtC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,eAAe,CAAC,CAAA;AA4E1C;;;;;;;;;GASG;AACH,MAAa,aAA+C,SAAQ,oCAAwB;IAM1F,YAAoB,MAA8B;QAChD,KAAK,EAAE,CAAA;QADW,WAAM,GAAN,MAAM,CAAwB;QALlD,uGAAuG;QACvG,gEAAgE;QAChE,kBAAa,GAAG,KAAK,CAAA;IAKrB,CAAC;IAEQ,KAAK,CAAC,UAAU,CACvB,YAAsC,EACtC,MAA0C,EAC1C,YAAoB;QAEpB,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,CAAA;QAC1D,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,SAAS,CAAA;IACzC,CAAC;IAED,4BAA4B,CAAC,MAA0C;QACrE,OAAO,MAAM,CAAC,uBAAuB,CAAA;IACvC,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,OAA2B,EAC3B,OAAiC;QAEjC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACnB,MAAM,CAAC,KAAK,CACV,gDAAgD,OAAO,CAAC,aAAa,CAAC,0BAA0B,OAAO,CACxG,CAAA;YACD,IAAI,IAAI,CAAC,aAAa,EAAE;gBACtB,yEAAyE;gBACzE,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAA;gBACjE,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;aAC3B;YACD,MAAM,IAAA,YAAK,EAAC,OAAO,CAAC,aAAa,CAAC,0BAA0B,CAAC,CAAA;YAC7D,OAAM;SACP;aAAM,IAAI,IAAI,CAAC,aAAa,KAAK,KAAK,EAAE;YACvC,4EAA4E;YAC5E,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAA;YACjE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;SAC1B;QAED,MAAM,CAAC,KAAK,CAAC,QAAQ,OAAO,CAAC,MAAM,0CAA0C,CAAC,CAAA;QAC9E,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;QAC/E,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAA;QAEzE,yFAAyF;QACzF,8FAA8F;QAC9F,kGAAkG;QAClG,wEAAwE;QACxE,MAAM,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,MAAM,cAAc,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAA;QACpF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;QACnC,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAA;QAErE,sGAAsG;QACtG,uFAAuF;QACvF,IAAI,QAAQ,GAAG,OAAO,CAAC,aAAa,CAAC,uBAAuB,EAAE;YAC5D,MAAM,CAAC,IAAI,CACT,6DAA6D,QAAQ;sDACvB,OAAO,CAAC,aAAa,CAAC,uBAAuB;0JACuD,CACnJ,CAAA;SACF;QACD,IAAI,QAAQ,GAAG,OAAO,CAAC,aAAa,CAAC,aAAa,EAAE;YAClD,MAAM,CAAC,IAAI,CACT,6DAA6D,QAAQ;mDAC1B,OAAO,CAAC,aAAa,CAAC,aAAa;0JACoE,CACnJ,CAAA;SACF;QAED,OAAM;IACR,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,aAAuC,EACvC,aAAiD;QAEjD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,aAAa,CAAC,CAAA;QAEpE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACnB,MAAM,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAA;YAChD,OAAM;SACP;QAED,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAClD,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IACzC,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,aAAuC,EACvC,aAAiD;QAEjD,IAAI;YACF,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAClD,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAC5D,aAAa,CAAC,OAAO,CACtB,CAAA;YAED,uCAAuC;YACvC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM;iBACxB,aAAa,CAAC,aAAa,CAAC,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC;iBAC5E,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACT,MAAM,MAAM,GAAG,CAAiC,CAAA;gBAChD,MAAM,eAAe,GAAG,CAAC,CAAC,QAAoD,CAAA;gBAC9E,MAAM,CAAC,QAAQ,CAAC,UAAU,GAAG;oBAC3B,GAAG,eAAe,CAAC,UAAU;oBAC7B,qBAAqB,EAAE,eAAe,CAAC,UAAU,EAAE,qBAAqB;iBACzE,CAAA;gBACD,OAAO,MAAM,CAAA;YACf,CAAC,CAAC,CAAA;YAEJ,oCAAoC;YACpC,MAAM,IAAI,GAAG,gBAAgB,CAAC,YAAY,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;YACzE,gBAAgB,CAAC,0BAA0B;iBACxC,MAAM,CAAC;gBACN,OAAO,EAAE,KAAK;gBACd,cAAc,EAAE,uBAAuB;aACxC,CAAC;iBACD,GAAG,CAAC,IAAI,CAAC,CAAA;YAEZ,OAAO,OAAO,CAAA;SACf;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,YAAY,gCAAwB,IAAI,CAAC,CAAC,KAAK,YAAY,kBAAU,EAAE;gBAC1E,MAAM,GAAG,GAAG,CAA6B,CAAA;gBACzC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAmB,CAAA;gBACrC,OAAO,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1C,MAAM,EAAE,KAAK;oBACb,QAAQ,EAAE;wBACR,YAAY,EAAE,uCAAuC,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,QAAQ,EAAE,IAAI,GAAG;wBAC9F,UAAU,EAAE,GAAG;wBACf,UAAU,EAAE,GAAG,CAAC,UAAU;qBAC3B;iBACF,CAAC,CAAC,CAAA;aACJ;iBAAM,IAAI,CAAC,YAAY,6BAAqB,EAAE;gBAC7C,MAAM,GAAG,GAAG,CAA0B,CAAA;gBACtC,OAAO,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1C,MAAM,EAAE,KAAK;oBACb,QAAQ,EAAE;wBACR,YAAY,EAAE,GAAG,CAAC,OAAO;wBACzB,UAAU,EAAE,GAAG;wBACf,UAAU,EAAE;4BACV,oBAAoB,EAAE,CAAC;4BACvB,qBAAqB,EAAE,CAAC;4BACxB,qBAAqB,EAAE,SAAS;yBACjC;qBACF;iBACF,CAAC,CAAC,CAAA;aACJ;iBAAM;gBACL,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;gBACf,OAAO,EAAE,CAAA;aACV;SACF;IACH,CAAC;CACF;AA9JD,sCA8JC"}
|
package/transports/index.d.ts
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { AdapterDependencies, EndpointContext } from '../adapter';
|
|
2
2
|
import { ResponseCache } from '../cache/response';
|
|
3
3
|
import { AdapterConfig, SettingsMap } from '../config';
|
|
4
|
-
import { AdapterRequest, AdapterResponse, RequestGenerics, ResponseGenerics } from '../util/
|
|
5
|
-
export * from './
|
|
6
|
-
export * from './rest';
|
|
4
|
+
import { AdapterRequest, AdapterResponse, RequestGenerics, ResponseGenerics } from '../util/types';
|
|
5
|
+
export * from './http';
|
|
7
6
|
export * from './sse';
|
|
8
7
|
export * from './websocket';
|
|
9
8
|
/**
|
package/transports/index.js
CHANGED
|
@@ -14,8 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./
|
|
18
|
-
__exportStar(require("./rest"), exports);
|
|
17
|
+
__exportStar(require("./http"), exports);
|
|
19
18
|
__exportStar(require("./sse"), exports);
|
|
20
19
|
__exportStar(require("./websocket"), exports);
|
|
21
20
|
//# sourceMappingURL=index.js.map
|
package/transports/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transports/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAKA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transports/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAKA,yCAAsB;AACtB,wCAAqB;AACrB,8CAA2B"}
|
package/transports/metrics.d.ts
CHANGED
|
@@ -3,12 +3,6 @@ import { TransportGenerics } from '.';
|
|
|
3
3
|
import { EndpointContext } from '../adapter';
|
|
4
4
|
import { AdapterConfig } from '../config';
|
|
5
5
|
import { InputParameters } from '../validation';
|
|
6
|
-
export declare const dataProviderMetricsLabel: (providerStatusCode?: number, method?: string) => {
|
|
7
|
-
provider_status_code: number | undefined;
|
|
8
|
-
method: string;
|
|
9
|
-
};
|
|
10
|
-
export declare const dataProviderRequests: client.Counter<"method" | "provider_status_code">;
|
|
11
|
-
export declare const dataProviderRequestDurationSeconds: client.Histogram<string>;
|
|
12
6
|
export declare const connectionErrorLabels: (message: string) => {
|
|
13
7
|
message: string;
|
|
14
8
|
};
|
package/transports/metrics.js
CHANGED
|
@@ -23,26 +23,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.transportPollingDurationSeconds = exports.transportPollingFailureCount = exports.bgExecuteSubscriptionSetCount = exports.wsMessageTotal = exports.wsSubscriptionTotal = exports.wsSubscriptionActive = exports.wsConnectionErrors = exports.wsConnectionActive = exports.recordWsMessageMetrics = exports.messageSubsLabels = exports.connectionErrorLabels =
|
|
26
|
+
exports.transportPollingDurationSeconds = exports.transportPollingFailureCount = exports.bgExecuteSubscriptionSetCount = exports.wsMessageTotal = exports.wsSubscriptionTotal = exports.wsSubscriptionActive = exports.wsConnectionErrors = exports.wsConnectionActive = exports.recordWsMessageMetrics = exports.messageSubsLabels = exports.connectionErrorLabels = void 0;
|
|
27
27
|
const client = __importStar(require("prom-client"));
|
|
28
28
|
const cache_1 = require("../cache");
|
|
29
|
-
const constants_1 = require("../metrics/constants");
|
|
30
|
-
// Data Provider Metrics
|
|
31
|
-
const dataProviderMetricsLabel = (providerStatusCode, method = 'get') => ({
|
|
32
|
-
provider_status_code: providerStatusCode,
|
|
33
|
-
method: method.toUpperCase(),
|
|
34
|
-
});
|
|
35
|
-
exports.dataProviderMetricsLabel = dataProviderMetricsLabel;
|
|
36
|
-
exports.dataProviderRequests = new client.Counter({
|
|
37
|
-
name: 'data_provider_requests',
|
|
38
|
-
help: 'The number of http requests that are made to a data provider',
|
|
39
|
-
labelNames: ['method', 'provider_status_code'],
|
|
40
|
-
});
|
|
41
|
-
exports.dataProviderRequestDurationSeconds = new client.Histogram({
|
|
42
|
-
name: 'data_provider_request_duration_seconds',
|
|
43
|
-
help: 'A histogram bucket of the distribution of data provider request durations',
|
|
44
|
-
buckets: constants_1.requestDurationBuckets,
|
|
45
|
-
});
|
|
46
29
|
// Websocket Metrics
|
|
47
30
|
const connectionErrorLabels = (message) => ({
|
|
48
31
|
// Key,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/transports/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AAGrC,oCAA6D;
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/transports/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AAGrC,oCAA6D;AAI7D,oBAAoB;AACb,MAAM,qBAAqB,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,CAAC;IACzD,OAAO;IACP,OAAO;CACR,CAAC,CAAA;AAHW,QAAA,qBAAqB,yBAGhC;AAIK,MAAM,iBAAiB,GAAG,CAC/B,OAIC,EACD,MAA8B,EAC9B,EAAE;IACF,MAAM,MAAM,GAAG,IAAA,uBAAe,EAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC/C,MAAM,QAAQ,GAAG,IAAA,yBAAiB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAEnD,OAAO;QACL,OAAO,EAAE,MAAM;QACf,gBAAgB,EAAE,QAAQ;KAC3B,CAAA;AACH,CAAC,CAAA;AAfY,QAAA,iBAAiB,qBAe7B;AAED,6CAA6C;AAC7C,8CAA8C;AAC9C,2DAA2D;AACpD,MAAM,sBAAsB,GAAG,CACpC,OAA2B,EAC3B,UAAoC,EACpC,YAAsC,EAChC,EAAE;IACR,MAAM,aAAa,GAAG,CAAC,MAA8B,EAAE,IAAqB,EAAE,EAAE;QAC9E,MAAM,UAAU,GAAG,IAAA,yBAAiB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAErD,0CAA0C;QAC1C,sBAAc;aACX,MAAM,CAAC;YACN,GAAG,UAAU;YACb,SAAS,EAAE,MAAM;SAClB,CAAC;aACD,GAAG,EAAE,CAAA;QAER,4CAA4C;QAC5C,IAAI,IAAI,KAAK,KAAK,EAAE;YAClB,2BAAmB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,CAAA;YAC5C,4BAAoB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,CAAA;SAC9C;aAAM;YACL,4BAAoB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,CAAA;SAC9C;IACH,CAAC,CAAA;IAED,UAAU,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;IACF,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QAC9B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAChC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AA/BY,QAAA,sBAAsB,0BA+BlC;AAEY,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACjD,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,kCAAkC;IACxC,UAAU,EAAE,CAAC,KAAK,CAAU;CAC7B,CAAC,CAAA;AAEW,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACnD,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,iCAAiC;IACvC,UAAU,EAAE,CAAC,KAAK,EAAE,SAAS,CAAU;CACxC,CAAC,CAAA;AAEW,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACnD,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,8CAA8C;IACpD,UAAU,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,kBAAkB,CAAU;CACvE,CAAC,CAAA;AAEW,QAAA,mBAAmB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACpD,IAAI,EAAE,uBAAuB;IAC7B,IAAI,EAAE,6CAA6C;IACnD,UAAU,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,kBAAkB,CAAU;CACvE,CAAC,CAAA;AAEW,QAAA,cAAc,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC/C,IAAI,EAAE,kBAAkB;IACxB,IAAI,EAAE,sCAAsC;IAC5C,UAAU,EAAE,CAAC,SAAS,EAAE,kBAAkB,EAAE,WAAW,CAAU;CAClE,CAAC,CAAA;AAEF,sBAAsB;AACT,QAAA,6BAA6B,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IAC5D,IAAI,EAAE,mCAAmC;IACzC,IAAI,EAAE,0DAA0D;IAChE,UAAU,EAAE,CAAC,UAAU,EAAE,gBAAgB,CAAU;CACpD,CAAC,CAAA;AAEW,QAAA,4BAA4B,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC7D,IAAI,EAAE,iCAAiC;IACvC,IAAI,EAAE,+FAA+F;IACrG,UAAU,EAAE,CAAC,UAAU,CAAU;CAClC,CAAC,CAAA;AAEW,QAAA,+BAA+B,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IAC9D,IAAI,EAAE,oCAAoC;IAC1C,IAAI,EAAE,iFAAiF;IACvF,UAAU,EAAE,CAAC,UAAU,EAAE,WAAW,CAAU;CAC/C,CAAC,CAAA"}
|
package/transports/sse.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { AxiosRequestConfig } from 'axios';
|
|
|
2
2
|
import EventSource from 'eventsource';
|
|
3
3
|
import { EndpointContext } from '../adapter';
|
|
4
4
|
import { AdapterConfig } from '../config';
|
|
5
|
-
import { ProviderResult } from '../util/
|
|
5
|
+
import { ProviderResult } from '../util/types';
|
|
6
6
|
import { TransportDependencies, TransportGenerics } from './';
|
|
7
7
|
import { StreamingTransport, SubscriptionDeltas } from './abstract/streaming';
|
|
8
8
|
export interface SSEConfig {
|
|
@@ -11,7 +11,7 @@ export interface SSEConfig {
|
|
|
11
11
|
}
|
|
12
12
|
/**
|
|
13
13
|
* Helper struct type that will be used to pass types to the generic parameters of a Transport.
|
|
14
|
-
* Extends the common TransportGenerics, adding Provider specific types for this
|
|
14
|
+
* Extends the common TransportGenerics, adding Provider specific types for this SSE endpoint.
|
|
15
15
|
*/
|
|
16
16
|
type SSETransportGenerics = TransportGenerics & {
|
|
17
17
|
/**
|
|
@@ -29,7 +29,7 @@ type SSETransportGenerics = TransportGenerics & {
|
|
|
29
29
|
*
|
|
30
30
|
* @typeParam T - Helper struct type that will be used to pass types to the generic parameters (check [[SSETransportGenerics]])
|
|
31
31
|
*/
|
|
32
|
-
export declare class
|
|
32
|
+
export declare class SseTransport<T extends SSETransportGenerics> extends StreamingTransport<T> {
|
|
33
33
|
private config;
|
|
34
34
|
EventSource: typeof EventSource;
|
|
35
35
|
eventListeners: {
|
package/transports/sse.js
CHANGED
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.
|
|
6
|
+
exports.SseTransport = void 0;
|
|
7
7
|
const axios_1 = __importDefault(require("axios"));
|
|
8
8
|
const eventsource_1 = __importDefault(require("eventsource"));
|
|
9
9
|
const util_1 = require("../util");
|
|
@@ -14,7 +14,7 @@ const logger = (0, util_1.makeLogger)('SSETransport');
|
|
|
14
14
|
*
|
|
15
15
|
* @typeParam T - Helper struct type that will be used to pass types to the generic parameters (check [[SSETransportGenerics]])
|
|
16
16
|
*/
|
|
17
|
-
class
|
|
17
|
+
class SseTransport extends streaming_1.StreamingTransport {
|
|
18
18
|
constructor(config) {
|
|
19
19
|
super();
|
|
20
20
|
this.config = config;
|
|
@@ -81,8 +81,11 @@ class SSETransport extends streaming_1.StreamingTransport {
|
|
|
81
81
|
const prepareKeepAliveRequest = this.config.prepareKeepAliveRequest(context);
|
|
82
82
|
makeRequest(prepareKeepAliveRequest);
|
|
83
83
|
}
|
|
84
|
+
// The background execute loop no longer sleeps between executions, so we have to do it here
|
|
85
|
+
logger.trace(`SSE handler complete, sleeping for ${context.adapterConfig.BACKGROUND_EXECUTE_MS_WS}ms...`);
|
|
86
|
+
await (0, util_1.sleep)(context.adapterConfig.BACKGROUND_EXECUTE_MS_SSE);
|
|
84
87
|
return;
|
|
85
88
|
}
|
|
86
89
|
}
|
|
87
|
-
exports.
|
|
90
|
+
exports.SseTransport = SseTransport;
|
|
88
91
|
//# sourceMappingURL=sse.js.map
|
package/transports/sse.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sse.js","sourceRoot":"","sources":["../../../src/transports/sse.ts"],"names":[],"mappings":";;;;;;AAAA,kDAAiD;AACjD,8DAAqC;AAGrC,
|
|
1
|
+
{"version":3,"file":"sse.js","sourceRoot":"","sources":["../../../src/transports/sse.ts"],"names":[],"mappings":";;;;;;AAAA,kDAAiD;AACjD,8DAAqC;AAGrC,kCAA2C;AAG3C,oDAA6E;AAE7E,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,cAAc,CAAC,CAAA;AAuBzC;;;;GAIG;AACH,MAAa,YAA6C,SAAQ,8BAAqB;IASrF,YACU,MAsBP;QAED,KAAK,EAAE,CAAA;QAxBC,WAAM,GAAN,MAAM,CAsBb;QA/BH,gBAAW,GAAuB,qBAAW,CAAA;QAM7C,kBAAa,GAAG,CAAC,CAAA;IA4BjB,CAAC;IAED,4BAA4B,CAAC,MAA0C;QACrE,OAAO,MAAM,CAAC,oBAAoB,CAAA;IACpC,CAAC;IAEQ,KAAK,CAAC,UAAU,CACvB,YAAsC,EACtC,MAA0C,EAC1C,YAAoB;QAEpB,KAAK,CAAC,UAAU,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,CAAA;QACpD,IAAI,YAAY,CAAC,WAAW,EAAE;YAC5B,IAAI,CAAC,WAAW,GAAG,YAAY,CAAC,WAAW,CAAA;SAC5C;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,OAA2B,EAC3B,aAAyD;QAEzD,IACE,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC;YACxD,CAAC,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC,UAAU,KAAK,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAClF;YACA,MAAM,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAA;YAC5F,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YACpF,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC/C,IAAI,CAAC,aAAa,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,mBAAmB,CAAC,CAAA;YAEvF,MAAM,qBAAqB,GAAG,CAAC,QAA8C,EAAE,EAAE;gBAC/E,OAAO,CAAC,CAAe,EAAE,EAAE;oBACzB,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;oBACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;wBAClD,MAAM,eAAe,GAAG,CAAC,CAAC,QAAoD,CAAA;wBAC9E,MAAM,MAAM,GAAG,CAAiC,CAAA;wBAChD,MAAM,CAAC,QAAQ,CAAC,UAAU,GAAG;4BAC3B,6BAA6B,EAAE,IAAI,CAAC,6BAA6B;4BACjE,oBAAoB;4BACpB,qBAAqB,EAAE,eAAe,CAAC,UAAU,EAAE,qBAAqB;yBACzE,CAAA;wBACD,OAAO,MAAM,CAAA;oBACf,CAAC,CAAC,CAAA;oBACF,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;gBACnC,CAAC,CAAA;YACH,CAAC,CAAA;YAED,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAC9C,IAAI,CAAC,aAAa,EAAE,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CAAA;YACtF,CAAC,CAAC,CAAA;SACH;QAED,MAAM,WAAW,GAAG,KAAK,EAAE,GAAqD,EAAE,EAAE;YAClF,IAAI;gBACF,MAAM,GAAG,GAAG,MAAM,eAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;gBACpC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,mBAAmB,GAAG,CAAC,UAAU,yBAAyB,CAAC,CAAA;aACnF;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAA;aAChD;YACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACjC,CAAC,CAAA;QAED,IAAI,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE;YAC5B,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAC3F,WAAW,CAAC,gBAAgB,CAAC,CAAA;SAC9B;QACD,IAAI,aAAa,CAAC,KAAK,CAAC,MAAM,EAAE;YAC9B,MAAM,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,4BAA4B,CACjE,aAAa,CAAC,KAAK,EACnB,OAAO,CACR,CAAA;YACD,WAAW,CAAC,kBAAkB,CAAC,CAAA;SAChC;QACD,IACE,IAAI,CAAC,MAAM,CAAC,uBAAuB;YACnC,aAAa,CAAC,OAAO,CAAC,MAAM;YAC5B,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,mBAAmB,EAC3E;YACA,MAAM,uBAAuB,GAAG,IAAI,CAAC,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAA;YAC5E,WAAW,CAAC,uBAAuB,CAAC,CAAA;SACrC;QAED,4FAA4F;QAC5F,MAAM,CAAC,KAAK,CACV,sCAAsC,OAAO,CAAC,aAAa,CAAC,wBAAwB,OAAO,CAC5F,CAAA;QACD,MAAM,IAAA,YAAK,EAAC,OAAO,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAA;QAE5D,OAAM;IACR,CAAC;CACF;AA7HD,oCA6HC"}
|