@chainlink/external-adapter-framework 0.12.1 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter/basic.js +9 -4
- package/adapter/basic.js.map +1 -1
- package/adapter/endpoint.d.ts +9 -1
- package/adapter/endpoint.js +23 -0
- package/adapter/endpoint.js.map +1 -1
- package/cache/index.d.ts +4 -2
- package/cache/index.js +2 -2
- package/cache/index.js.map +1 -1
- package/cache/local.d.ts +2 -0
- package/cache/local.js +2 -31
- package/cache/local.js.map +1 -1
- package/cache/metrics.d.ts +5 -2
- package/cache/metrics.js +19 -3
- package/cache/metrics.js.map +1 -1
- package/cache/redis.d.ts +2 -0
- package/cache/redis.js +7 -9
- package/cache/redis.js.map +1 -1
- package/cache/response.d.ts +36 -0
- package/cache/response.js +97 -0
- package/cache/response.js.map +1 -0
- package/config/index.d.ts +6 -6
- package/config/index.js +6 -10
- package/config/index.js.map +1 -1
- package/index.js +4 -1
- package/index.js.map +1 -1
- package/metrics/util.d.ts +1 -1
- package/metrics/util.js +2 -2
- package/metrics/util.js.map +1 -1
- package/package.json +2 -2
- package/rate-limiting/metrics.js +1 -1
- package/rate-limiting/metrics.js.map +1 -1
- package/transports/abstract/streaming.d.ts +34 -0
- package/transports/abstract/streaming.js +44 -0
- package/transports/abstract/streaming.js.map +1 -0
- package/transports/abstract/subscription.d.ts +38 -0
- package/transports/abstract/subscription.js +63 -0
- package/transports/abstract/subscription.js.map +1 -0
- package/transports/batch-warming.d.ts +7 -12
- package/transports/batch-warming.js +42 -28
- package/transports/batch-warming.js.map +1 -1
- package/transports/index.d.ts +20 -13
- package/transports/index.js +0 -35
- package/transports/index.js.map +1 -1
- package/transports/meta/index.d.ts +1 -0
- package/transports/meta/index.js +18 -0
- package/transports/meta/index.js.map +1 -0
- package/transports/{routing.d.ts → meta/routing.d.ts} +11 -5
- package/transports/{routing.js → meta/routing.js} +6 -6
- package/transports/meta/routing.js.map +1 -0
- package/transports/metrics.d.ts +11 -4
- package/transports/metrics.js +30 -20
- package/transports/metrics.js.map +1 -1
- package/transports/rest.d.ts +10 -7
- package/transports/rest.js +22 -12
- package/transports/rest.js.map +1 -1
- package/transports/sse.d.ts +8 -12
- package/transports/sse.js +29 -39
- package/transports/sse.js.map +1 -1
- package/transports/websocket.d.ts +15 -14
- package/transports/websocket.js +87 -60
- package/transports/websocket.js.map +1 -1
- package/util/request.d.ts +90 -11
- package/transports/routing.js.map +0 -1
package/transports/index.js
CHANGED
|
@@ -14,43 +14,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.buildCacheEntriesFromResults = void 0;
|
|
18
|
-
const cache_1 = require("../cache");
|
|
19
17
|
__exportStar(require("./batch-warming"), exports);
|
|
20
18
|
__exportStar(require("./rest"), exports);
|
|
21
19
|
__exportStar(require("./sse"), exports);
|
|
22
20
|
__exportStar(require("./websocket"), exports);
|
|
23
|
-
/**
|
|
24
|
-
* Helper method to build cache entries to set after getting a bunch of responses from a DP.
|
|
25
|
-
*
|
|
26
|
-
* @param results - a list of results coming from a DataProvider
|
|
27
|
-
* @param context - context for the Adapter
|
|
28
|
-
* @returns a list of CacheEntries of AdapterResponses
|
|
29
|
-
*/
|
|
30
|
-
const buildCacheEntriesFromResults = (results, context) => results.map((r) => {
|
|
31
|
-
const cacheEntry = {
|
|
32
|
-
key: (0, cache_1.calculateCacheKey)(context, r.params),
|
|
33
|
-
value: {
|
|
34
|
-
result: r.value,
|
|
35
|
-
statusCode: 200,
|
|
36
|
-
data: {
|
|
37
|
-
result: r.value,
|
|
38
|
-
},
|
|
39
|
-
},
|
|
40
|
-
};
|
|
41
|
-
if (context.adapterConfig.METRICS_ENABLED &&
|
|
42
|
-
context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED) {
|
|
43
|
-
const metrics = {
|
|
44
|
-
maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE,
|
|
45
|
-
meta: {
|
|
46
|
-
metrics: {
|
|
47
|
-
feedId: (0, cache_1.calculateFeedId)(context, r.params),
|
|
48
|
-
},
|
|
49
|
-
},
|
|
50
|
-
};
|
|
51
|
-
cacheEntry.value = { ...cacheEntry.value, ...metrics };
|
|
52
|
-
}
|
|
53
|
-
return cacheEntry;
|
|
54
|
-
});
|
|
55
|
-
exports.buildCacheEntriesFromResults = buildCacheEntriesFromResults;
|
|
56
21
|
//# sourceMappingURL=index.js.map
|
package/transports/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transports/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transports/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAKA,kDAA+B;AAC/B,yCAAsB;AACtB,wCAAqB;AACrB,8CAA2B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './routing';
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./routing"), exports);
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/transports/meta/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,4CAAyB"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { EndpointContext } from '../../adapter';
|
|
2
|
+
import { ResponseCache } from '../../cache/response';
|
|
3
|
+
import { AdapterConfig } from '../../config';
|
|
4
|
+
import { AdapterRequest, AdapterResponse } from '../../util';
|
|
5
|
+
import { MetaTransport, Transport, TransportDependencies, TransportGenerics } from '..';
|
|
5
6
|
/**
|
|
6
7
|
* Transport implementation that takes 2 or more transports and a function that determines with transport to use.
|
|
7
8
|
*
|
|
@@ -15,7 +16,12 @@ export declare class RoutingTransport<T extends TransportGenerics> implements Me
|
|
|
15
16
|
constructor(transports: {
|
|
16
17
|
[key: string]: Transport<T>;
|
|
17
18
|
}, route: (req: AdapterRequest<T['Request']>, adapterConfig?: AdapterConfig<T['CustomSettings']>) => string);
|
|
18
|
-
|
|
19
|
+
responseCache: ResponseCache<{
|
|
20
|
+
Request: T['Request'];
|
|
21
|
+
Response: T['Response'];
|
|
22
|
+
}>;
|
|
23
|
+
backgroundExecute?: ((context: EndpointContext<T>) => Promise<number>) | undefined;
|
|
24
|
+
initialize(dependencies: TransportDependencies<T>, adapterConfig: AdapterConfig<T['CustomSettings']>, endpointName: string): Promise<void>;
|
|
19
25
|
registerRequest(req: AdapterRequest<T['Request']>, adapterConfig: AdapterConfig<T['CustomSettings']>): Promise<void>;
|
|
20
26
|
private resolveTransport;
|
|
21
27
|
foregroundExecute(req: AdapterRequest<T['Request']>, adapterConfig: AdapterConfig<T['CustomSettings']>): Promise<AdapterResponse<any> | void>;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.RoutingTransport = void 0;
|
|
4
|
-
const util_1 = require("
|
|
5
|
-
const error_1 = require("
|
|
4
|
+
const util_1 = require("../../util");
|
|
5
|
+
const error_1 = require("../../validation/error");
|
|
6
6
|
const logger = (0, util_1.makeLogger)('RoutingTransport');
|
|
7
7
|
/**
|
|
8
8
|
* Transport implementation that takes 2 or more transports and a function that determines with transport to use.
|
|
@@ -20,10 +20,10 @@ class RoutingTransport {
|
|
|
20
20
|
}
|
|
21
21
|
async initialize(dependencies, adapterConfig, endpointName) {
|
|
22
22
|
logger.debug(`Initializing ${Object.keys(this.transports).length} transports`);
|
|
23
|
-
Object.entries(this.transports).
|
|
24
|
-
logger.debug(`Initializing transport ${
|
|
25
|
-
|
|
26
|
-
});
|
|
23
|
+
await Promise.all(Object.entries(this.transports).map(([name, transport]) => {
|
|
24
|
+
logger.debug(`Initializing transport ${name} (${transport.constructor.name})`);
|
|
25
|
+
return transport.initialize(dependencies, adapterConfig, endpointName);
|
|
26
|
+
}));
|
|
27
27
|
}
|
|
28
28
|
async registerRequest(req, adapterConfig) {
|
|
29
29
|
logger.debug(`registering request using `, req.requestContext.data);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"routing.js","sourceRoot":"","sources":["../../../../src/transports/meta/routing.ts"],"names":[],"mappings":";;;AAGA,qCAAwE;AACxE,kDAAqD;AAGrD,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,kBAAkB,CAAC,CAAA;AAE7C;;;;GAIG;AACH,MAAa,gBAAgB;IAI3B;IACE,6GAA6G;IACtG,UAA2C;IAClD,+FAA+F;IACvF,KAGG;QALJ,eAAU,GAAV,UAAU,CAAiC;QAE1C,UAAK,GAAL,KAAK,CAGF;IACV,CAAC;IAOJ,KAAK,CAAC,UAAU,CACd,YAAsC,EACtC,aAAiD,EACjD,YAAoB;QAEpB,MAAM,CAAC,KAAK,CAAC,gBAAgB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,aAAa,CAAC,CAAA;QAC9E,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,EAAE;YACxD,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,KAAK,SAAS,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,CAAA;YAC9E,OAAO,SAAS,CAAC,UAAU,CAAC,YAAY,EAAE,aAAa,EAAE,YAAY,CAAC,CAAA;QACxE,CAAC,CAAC,CACH,CAAA;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,GAAiC,EACjC,aAAiD;QAEjD,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;QAEnE,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;QAC3D,IAAI,SAAS,CAAC,eAAe,EAAE;YAC7B,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAA;YAC9D,OAAO,SAAS,CAAC,eAAe,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;SACrD;QACD,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAA;IACvE,CAAC;IAEO,gBAAgB,CACtB,GAAiC,EACjC,aAAkD;QAElD,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;QAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,aAAa,IAAI,SAAS,CAAC,CAAA;QACvD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;YACzB,MAAM,CAAC,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAA;YACjD,MAAM,IAAI,oBAAY,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,0BAA0B,GAAG,EAAE,EAAE,CAAC,CAAA;SACtF;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAA;QACtC,MAAM,CAAC,KAAK,CAAC,SAAS,SAAS,CAAC,WAAW,CAAC,IAAI,qBAAqB,GAAG,EAAE,CAAC,CAAA;QAC3E,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,oFAAoF;IACpF,wDAAwD;IACxD,KAAK,CAAC,iBAAiB,CACrB,GAAiC,EACjC,aAAiD;QAEjD,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;QAC3D,IAAI,SAAS,CAAC,iBAAiB,EAAE;YAC/B,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAA;YAChE,OAAO,SAAS,CAAC,iBAAiB,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;SACvD;QACD,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAA;IACzE,CAAC;CACF;AA3ED,4CA2EC"}
|
package/transports/metrics.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import * as client from 'prom-client';
|
|
2
2
|
import { TransportGenerics } from '.';
|
|
3
3
|
import { EndpointContext } from '../adapter';
|
|
4
|
+
import { AdapterConfig } from '../config';
|
|
5
|
+
import { InputParameters } from '../validation';
|
|
4
6
|
export declare const dataProviderMetricsLabel: (providerStatusCode?: number, method?: string) => {
|
|
5
7
|
provider_status_code: number | undefined;
|
|
6
8
|
method: string;
|
|
@@ -10,16 +12,21 @@ export declare const dataProviderRequestDurationSeconds: client.Histogram<string
|
|
|
10
12
|
export declare const connectionErrorLabels: (message: string) => {
|
|
11
13
|
message: string;
|
|
12
14
|
};
|
|
13
|
-
export declare
|
|
15
|
+
export declare type MessageDirection = 'sent' | 'received';
|
|
16
|
+
export declare const messageSubsLabels: <T extends TransportGenerics>(context: {
|
|
17
|
+
inputParameters: InputParameters;
|
|
18
|
+
endpointName: string;
|
|
19
|
+
adapterConfig: AdapterConfig<T["CustomSettings"]>;
|
|
20
|
+
}, params: T["Request"]["Params"]) => {
|
|
14
21
|
feed_id: string;
|
|
15
22
|
subscription_key: string;
|
|
16
23
|
};
|
|
17
24
|
export declare const recordWsMessageMetrics: <T extends TransportGenerics>(context: EndpointContext<T>, subscribes: T["Request"]["Params"][], unsubscribes: T["Request"]["Params"][]) => void;
|
|
18
25
|
export declare const wsConnectionActive: client.Gauge<"url">;
|
|
19
26
|
export declare const wsConnectionErrors: client.Counter<"message" | "url">;
|
|
20
|
-
export declare const wsSubscriptionActive: client.Gauge<"feed_id" | "
|
|
21
|
-
export declare const wsSubscriptionTotal: client.Counter<"feed_id" | "
|
|
22
|
-
export declare const wsMessageTotal: client.Counter<"feed_id" | "subscription_key">;
|
|
27
|
+
export declare const wsSubscriptionActive: client.Gauge<"feed_id" | "subscription_key" | "connection_url">;
|
|
28
|
+
export declare const wsSubscriptionTotal: client.Counter<"feed_id" | "subscription_key" | "connection_url">;
|
|
29
|
+
export declare const wsMessageTotal: client.Counter<"direction" | "feed_id" | "subscription_key">;
|
|
23
30
|
export declare const bgExecuteSubscriptionSetCount: client.Gauge<"endpoint" | "transport_type">;
|
|
24
31
|
export declare const transportPollingFailureCount: client.Counter<"endpoint">;
|
|
25
32
|
export declare const transportPollingDurationSeconds: client.Gauge<"succeeded" | "endpoint">;
|
package/transports/metrics.js
CHANGED
|
@@ -49,32 +49,42 @@ const connectionErrorLabels = (message) => ({
|
|
|
49
49
|
message,
|
|
50
50
|
});
|
|
51
51
|
exports.connectionErrorLabels = connectionErrorLabels;
|
|
52
|
-
const messageSubsLabels = (
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
52
|
+
const messageSubsLabels = (context, params) => {
|
|
53
|
+
const feedId = (0, cache_1.calculateFeedId)(context, params);
|
|
54
|
+
const cacheKey = (0, cache_1.calculateCacheKey)(context, params);
|
|
55
|
+
return {
|
|
56
|
+
feed_id: feedId,
|
|
57
|
+
subscription_key: cacheKey,
|
|
58
|
+
};
|
|
59
|
+
};
|
|
56
60
|
exports.messageSubsLabels = messageSubsLabels;
|
|
57
61
|
// Record WS message and subscription metrics
|
|
58
62
|
// Recalculate cacheKey and feedId for metrics
|
|
59
63
|
// since avoiding storing extra info in expiring sorted set
|
|
60
64
|
const recordWsMessageMetrics = (context, subscribes, unsubscribes) => {
|
|
61
|
-
|
|
62
|
-
const
|
|
63
|
-
const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
|
|
65
|
+
const recordMetrics = (params, type) => {
|
|
66
|
+
const baseLabels = (0, exports.messageSubsLabels)(context, params);
|
|
64
67
|
// Record total number of ws messages sent
|
|
65
|
-
exports.wsMessageTotal
|
|
68
|
+
exports.wsMessageTotal
|
|
69
|
+
.labels({
|
|
70
|
+
...baseLabels,
|
|
71
|
+
direction: 'sent',
|
|
72
|
+
})
|
|
73
|
+
.inc();
|
|
66
74
|
// Record total number of subscriptions made
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
75
|
+
if (type === 'sub') {
|
|
76
|
+
exports.wsSubscriptionTotal.labels(baseLabels).inc();
|
|
77
|
+
exports.wsSubscriptionActive.labels(baseLabels).inc();
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
exports.wsSubscriptionActive.labels(baseLabels).dec();
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
subscribes.forEach((params) => {
|
|
84
|
+
recordMetrics(params, 'sub');
|
|
70
85
|
});
|
|
71
|
-
unsubscribes.forEach((
|
|
72
|
-
|
|
73
|
-
const cacheKey = (0, cache_1.calculateCacheKey)(context, param);
|
|
74
|
-
// Record total number of ws messages sent
|
|
75
|
-
exports.wsMessageTotal.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).inc();
|
|
76
|
-
// Record number of active ws subscriptions
|
|
77
|
-
exports.wsSubscriptionActive.labels((0, exports.messageSubsLabels)(feedId, cacheKey)).dec();
|
|
86
|
+
unsubscribes.forEach((params) => {
|
|
87
|
+
recordMetrics(params, 'unsub');
|
|
78
88
|
});
|
|
79
89
|
};
|
|
80
90
|
exports.recordWsMessageMetrics = recordWsMessageMetrics;
|
|
@@ -100,8 +110,8 @@ exports.wsSubscriptionTotal = new client.Counter({
|
|
|
100
110
|
});
|
|
101
111
|
exports.wsMessageTotal = new client.Counter({
|
|
102
112
|
name: 'ws_message_total',
|
|
103
|
-
help: 'The number of messages
|
|
104
|
-
labelNames: ['feed_id', 'subscription_key'],
|
|
113
|
+
help: 'The number of messages sent in total',
|
|
114
|
+
labelNames: ['feed_id', 'subscription_key', 'direction'],
|
|
105
115
|
});
|
|
106
116
|
// V3 specific metrics
|
|
107
117
|
exports.bgExecuteSubscriptionSetCount = new client.Gauge({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/transports/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AAGrC,oCAA6D;
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/transports/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AAGrC,oCAA6D;AAE7D,oDAA6D;AAG7D,wBAAwB;AACjB,MAAM,wBAAwB,GAAG,CAAC,kBAA2B,EAAE,MAAM,GAAG,KAAK,EAAE,EAAE,CAAC,CAAC;IACxF,oBAAoB,EAAE,kBAAkB;IACxC,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;CAC7B,CAAC,CAAA;AAHW,QAAA,wBAAwB,4BAGnC;AAEW,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACrD,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,8DAA8D;IACpE,UAAU,EAAE,CAAC,QAAQ,EAAE,sBAAsB,CAAU;CACxD,CAAC,CAAA;AAEW,QAAA,kCAAkC,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC;IACrE,IAAI,EAAE,wCAAwC;IAC9C,IAAI,EAAE,2EAA2E;IACjF,OAAO,EAAE,kCAAsB;CAChC,CAAC,CAAA;AAEF,oBAAoB;AACb,MAAM,qBAAqB,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,CAAC;IACzD,OAAO;IACP,OAAO;CACR,CAAC,CAAA;AAHW,QAAA,qBAAqB,yBAGhC;AAIK,MAAM,iBAAiB,GAAG,CAC/B,OAIC,EACD,MAA8B,EAC9B,EAAE;IACF,MAAM,MAAM,GAAG,IAAA,uBAAe,EAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC/C,MAAM,QAAQ,GAAG,IAAA,yBAAiB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAEnD,OAAO;QACL,OAAO,EAAE,MAAM;QACf,gBAAgB,EAAE,QAAQ;KAC3B,CAAA;AACH,CAAC,CAAA;AAfY,QAAA,iBAAiB,qBAe7B;AAED,6CAA6C;AAC7C,8CAA8C;AAC9C,2DAA2D;AACpD,MAAM,sBAAsB,GAAG,CACpC,OAA2B,EAC3B,UAAoC,EACpC,YAAsC,EAChC,EAAE;IACR,MAAM,aAAa,GAAG,CAAC,MAA8B,EAAE,IAAqB,EAAE,EAAE;QAC9E,MAAM,UAAU,GAAG,IAAA,yBAAiB,EAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAErD,0CAA0C;QAC1C,sBAAc;aACX,MAAM,CAAC;YACN,GAAG,UAAU;YACb,SAAS,EAAE,MAAM;SAClB,CAAC;aACD,GAAG,EAAE,CAAA;QAER,4CAA4C;QAC5C,IAAI,IAAI,KAAK,KAAK,EAAE;YAClB,2BAAmB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,CAAA;YAC5C,4BAAoB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,CAAA;SAC9C;aAAM;YACL,4BAAoB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,CAAA;SAC9C;IACH,CAAC,CAAA;IAED,UAAU,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;IACF,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QAC9B,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAChC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AA/BY,QAAA,sBAAsB,0BA+BlC;AAEY,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACjD,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,kCAAkC;IACxC,UAAU,EAAE,CAAC,KAAK,CAAU;CAC7B,CAAC,CAAA;AAEW,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACnD,IAAI,EAAE,sBAAsB;IAC5B,IAAI,EAAE,iCAAiC;IACvC,UAAU,EAAE,CAAC,KAAK,EAAE,SAAS,CAAU;CACxC,CAAC,CAAA;AAEW,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IACnD,IAAI,EAAE,wBAAwB;IAC9B,IAAI,EAAE,8CAA8C;IACpD,UAAU,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,kBAAkB,CAAU;CACvE,CAAC,CAAA;AAEW,QAAA,mBAAmB,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IACpD,IAAI,EAAE,uBAAuB;IAC7B,IAAI,EAAE,6CAA6C;IACnD,UAAU,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,kBAAkB,CAAU;CACvE,CAAC,CAAA;AAEW,QAAA,cAAc,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC/C,IAAI,EAAE,kBAAkB;IACxB,IAAI,EAAE,sCAAsC;IAC5C,UAAU,EAAE,CAAC,SAAS,EAAE,kBAAkB,EAAE,WAAW,CAAU;CAClE,CAAC,CAAA;AAEF,sBAAsB;AACT,QAAA,6BAA6B,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IAC5D,IAAI,EAAE,mCAAmC;IACzC,IAAI,EAAE,0DAA0D;IAChE,UAAU,EAAE,CAAC,UAAU,EAAE,gBAAgB,CAAU;CACpD,CAAC,CAAA;AAEW,QAAA,4BAA4B,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC;IAC7D,IAAI,EAAE,iCAAiC;IACvC,IAAI,EAAE,+FAA+F;IACrG,UAAU,EAAE,CAAC,UAAU,CAAU;CAClC,CAAC,CAAA;AAEW,QAAA,+BAA+B,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC;IAC9D,IAAI,EAAE,oCAAoC;IAC1C,IAAI,EAAE,iFAAiF;IACvF,UAAU,EAAE,CAAC,UAAU,EAAE,WAAW,CAAU;CAC/C,CAAC,CAAA"}
|
package/transports/rest.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { AdapterRequest, AdapterResponse } from '../util/request';
|
|
2
|
-
import { Transport, TransportGenerics } from './';
|
|
3
|
-
import { Cache } from '../cache';
|
|
4
1
|
import { AxiosRequestConfig, AxiosResponse } from 'axios';
|
|
2
|
+
import { Cache } from '../cache';
|
|
3
|
+
import { ResponseCache } from '../cache/response';
|
|
5
4
|
import { AdapterConfig } from '../config';
|
|
6
5
|
import { RequestRateLimiter } from '../rate-limiting';
|
|
7
|
-
import {
|
|
6
|
+
import { AdapterRequest, AdapterResponse, PartialAdapterResponse } from '../util/request';
|
|
7
|
+
import { Transport, TransportDependencies, TransportGenerics } from './';
|
|
8
8
|
export { AxiosRequestConfig as HttpRequestConfig, AxiosResponse as HttpResponse };
|
|
9
9
|
/**
|
|
10
10
|
* Helper struct type that will be used to pass types to the generic parameters of a Transport.
|
|
@@ -30,7 +30,7 @@ declare type RestTransportGenerics = TransportGenerics & {
|
|
|
30
30
|
*/
|
|
31
31
|
export interface RestTransportConfig<T extends RestTransportGenerics> {
|
|
32
32
|
prepareRequest: (req: AdapterRequest<T['Request']>, config: AdapterConfig<T['CustomSettings']>) => AxiosRequestConfig<T['Provider']['RequestBody']> | Promise<AxiosRequestConfig<T['Provider']['RequestBody']>>;
|
|
33
|
-
parseResponse: (req: AdapterRequest<T['Request']>, res: AxiosResponse<T['Provider']['ResponseBody']>, config: AdapterConfig<T['CustomSettings']>) =>
|
|
33
|
+
parseResponse: (req: AdapterRequest<T['Request']>, res: AxiosResponse<T['Provider']['ResponseBody']>, config: AdapterConfig<T['CustomSettings']>) => PartialAdapterResponse<T['Response']> | Promise<PartialAdapterResponse<T['Response']>>;
|
|
34
34
|
options: {
|
|
35
35
|
requestCoalescing: {
|
|
36
36
|
enabled: boolean;
|
|
@@ -50,11 +50,14 @@ export interface RestTransportConfig<T extends RestTransportGenerics> {
|
|
|
50
50
|
export declare class RestTransport<T extends RestTransportGenerics> implements Transport<T> {
|
|
51
51
|
protected config: RestTransportConfig<T>;
|
|
52
52
|
inFlightPrefix: string;
|
|
53
|
-
|
|
53
|
+
responseCache: ResponseCache<{
|
|
54
|
+
Request: T['Request'];
|
|
55
|
+
Response: T['Response'];
|
|
56
|
+
}>;
|
|
54
57
|
inFlightCache: Cache<boolean>;
|
|
55
58
|
rateLimiter: RequestRateLimiter;
|
|
56
59
|
constructor(config: RestTransportConfig<T>);
|
|
57
|
-
initialize(dependencies:
|
|
60
|
+
initialize(dependencies: TransportDependencies<T>, config: AdapterConfig<T['CustomSettings']>): Promise<void>;
|
|
58
61
|
protected waitUntilUnderRateLimit(options: {
|
|
59
62
|
maxRetries: number;
|
|
60
63
|
msBetweenRetries: number;
|
package/transports/rest.js
CHANGED
|
@@ -24,10 +24,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
26
|
exports.RestTransport = void 0;
|
|
27
|
+
const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
|
|
27
28
|
const util_1 = require("../util");
|
|
28
29
|
const error_1 = require("../validation/error");
|
|
29
30
|
const util_2 = require("./util");
|
|
30
|
-
const rateLimitMetrics = __importStar(require("../rate-limiting/metrics"));
|
|
31
31
|
const IN_FLIGHT_PREFIX = 'InFlight';
|
|
32
32
|
const logger = (0, util_1.makeLogger)('RestTransport');
|
|
33
33
|
/**
|
|
@@ -44,8 +44,8 @@ class RestTransport {
|
|
|
44
44
|
this.config = config;
|
|
45
45
|
}
|
|
46
46
|
async initialize(dependencies, config) {
|
|
47
|
+
this.responseCache = dependencies.responseCache;
|
|
47
48
|
this.inFlightPrefix = `${IN_FLIGHT_PREFIX}-`;
|
|
48
|
-
this.cache = dependencies.cache;
|
|
49
49
|
this.inFlightCache = dependencies.cache;
|
|
50
50
|
this.rateLimiter = dependencies.requestRateLimiter;
|
|
51
51
|
// Allow enabling/disabling request coalescing through env var
|
|
@@ -73,8 +73,8 @@ class RestTransport {
|
|
|
73
73
|
const randomMs = Math.random() * (this.config.options.requestCoalescing.entropyMax || 0);
|
|
74
74
|
await (0, util_1.sleep)(randomMs);
|
|
75
75
|
// Check if request is in flight if coalescing is enabled
|
|
76
|
-
const
|
|
77
|
-
|
|
76
|
+
const inFlightKey = this.inFlightPrefix + req.requestContext.cacheKey;
|
|
77
|
+
const inFlight = this.config.options.requestCoalescing.enabled && (await this.inFlightCache.get(inFlightKey));
|
|
78
78
|
if (inFlight) {
|
|
79
79
|
logger.debug('Request is in flight, transport has been set up');
|
|
80
80
|
return;
|
|
@@ -84,7 +84,7 @@ class RestTransport {
|
|
|
84
84
|
const ttl = config.REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES *
|
|
85
85
|
config.REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES;
|
|
86
86
|
logger.debug('Setting up rest transport, setting request in flight in cache');
|
|
87
|
-
await this.inFlightCache.set(
|
|
87
|
+
await this.inFlightCache.set(inFlightKey, true, ttl + 100); // Can't use Infinity for things like Redis
|
|
88
88
|
}
|
|
89
89
|
const request = await this.config.prepareRequest(req, config);
|
|
90
90
|
logger.trace('Check if we are under rate limits to perform request');
|
|
@@ -96,20 +96,30 @@ class RestTransport {
|
|
|
96
96
|
};
|
|
97
97
|
await checkForRateLimit();
|
|
98
98
|
logger.trace('Sending request to data provider...');
|
|
99
|
+
const providerDataRequested = Date.now();
|
|
99
100
|
const providerResponse = await (0, util_2.axiosRequest)(request, config, checkForRateLimit);
|
|
101
|
+
const providerDataReceived = Date.now();
|
|
100
102
|
logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`);
|
|
101
|
-
const
|
|
103
|
+
const response = (await this.config.parseResponse(req, providerResponse, config));
|
|
104
|
+
response.timestamps = {
|
|
105
|
+
providerDataRequested,
|
|
106
|
+
providerDataReceived,
|
|
107
|
+
providerIndicatedTime: response.timestamps?.providerIndicatedTime,
|
|
108
|
+
};
|
|
102
109
|
if (config.API_VERBOSE) {
|
|
103
|
-
|
|
110
|
+
response.data = providerResponse.data;
|
|
104
111
|
}
|
|
105
112
|
if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
|
|
106
|
-
|
|
107
|
-
parsedResponse.meta = {
|
|
113
|
+
response.meta = {
|
|
108
114
|
metrics: { feedId: req.requestContext.meta?.metrics?.feedId || 'N/A' },
|
|
109
115
|
};
|
|
110
116
|
}
|
|
117
|
+
const result = {
|
|
118
|
+
params: req.requestContext.data,
|
|
119
|
+
response,
|
|
120
|
+
};
|
|
111
121
|
logger.debug('Setting provider response in cache');
|
|
112
|
-
await this.
|
|
122
|
+
await this.responseCache.write([result]);
|
|
113
123
|
// Record cost of data provider call
|
|
114
124
|
const cost = rateLimitMetrics.retrieveCost(providerResponse.data);
|
|
115
125
|
rateLimitMetrics.rateLimitCreditsSpentTotal
|
|
@@ -125,9 +135,9 @@ class RestTransport {
|
|
|
125
135
|
};
|
|
126
136
|
if (this.config.options.requestCoalescing.enabled) {
|
|
127
137
|
logger.debug('Set provider response in cache, removing in flight from cache');
|
|
128
|
-
await this.
|
|
138
|
+
await this.inFlightCache.delete(inFlightKey);
|
|
129
139
|
}
|
|
130
|
-
return
|
|
140
|
+
return response;
|
|
131
141
|
}
|
|
132
142
|
}
|
|
133
143
|
exports.RestTransport = RestTransport;
|
package/transports/rest.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rest.js","sourceRoot":"","sources":["../../../src/transports/rest.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"rest.js","sourceRoot":"","sources":["../../../src/transports/rest.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,2EAA4D;AAC5D,kCAA2C;AAO3C,+CAAkD;AAElD,iCAAqC;AAIrC,MAAM,gBAAgB,GAAG,UAAU,CAAA;AAEnC,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,eAAe,CAAC,CAAA;AA8C1C;;;;;;;;GAQG;AACH,MAAa,aAAa;IASxB,YAAsB,MAA8B;QAA9B,WAAM,GAAN,MAAM,CAAwB;IAAG,CAAC;IAExD,KAAK,CAAC,UAAU,CACd,YAAsC,EACtC,MAA0C;QAE1C,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC,aAAa,CAAA;QAC/C,IAAI,CAAC,cAAc,GAAG,GAAG,gBAAgB,GAAG,CAAA;QAC5C,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC,KAAuB,CAAA;QACzD,IAAI,CAAC,WAAW,GAAG,YAAY,CAAC,kBAAkB,CAAA;QAElD,8DAA8D;QAC9D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,OAAO,GAAG,MAAM,CAAC,0BAA0B,CAAA;QACjF,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,UAAU,GAAG,MAAM,CAAC,8BAA8B,CAAA;IAC1F,CAAC;IAES,KAAK,CAAC,uBAAuB,CACrC,OAGC,EACD,KAAK,GAAG,CAAC;QAET,IAAI,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,EAAE;YACpC,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAA;YACrE,OAAM;SACP;QAED,IAAI,KAAK,IAAI,OAAO,CAAC,UAAU,EAAE;YAC/B,MAAM,IAAI,oBAAY,CAAC;gBACrB,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,oFAAoF,OAAO,CAAC,UAAU,GAAG;aACnH,CAAC,CAAA;SACH;QAED,MAAM,CAAC,KAAK,CAAC,mDAAmD,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAA;QAC3F,MAAM,IAAA,YAAK,EAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;QACrC,MAAM,IAAI,CAAC,uBAAuB,CAAC,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;IACxD,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,GAAiC,EACjC,MAA0C;QAE1C,gGAAgG;QAChG,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,UAAU,IAAI,CAAC,CAAC,CAAA;QACxF,MAAM,IAAA,YAAK,EAAC,QAAQ,CAAC,CAAA;QAErB,yDAAyD;QACzD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,GAAG,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAA;QACrE,MAAM,QAAQ,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,OAAO,IAAI,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;QAC9F,IAAI,QAAQ,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAA;YAC/D,OAAM;SACP;aAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,OAAO,EAAE;YACxD,8EAA8E;YAC9E,MAAM,GAAG,GACP,MAAM,CAAC,qCAAqC;gBAC5C,MAAM,CAAC,4CAA4C,CAAA;YACrD,MAAM,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAA;YAC7E,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,GAAG,GAAG,GAAG,CAAC,CAAA,CAAC,2CAA2C;SACvG;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;QAE7D,MAAM,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAA;QACpE,MAAM,iBAAiB,GAAG,KAAK,IAAI,EAAE;YACnC,OAAO,IAAI,CAAC,uBAAuB,CAAC;gBAClC,UAAU,EAAE,MAAM,CAAC,qCAAqC;gBACxD,gBAAgB,EAAE,MAAM,CAAC,4CAA4C;aACtE,CAAC,CAAA;QACJ,CAAC,CAAA;QACD,MAAM,iBAAiB,EAAE,CAAA;QAEzB,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACnD,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxC,MAAM,gBAAgB,GAAG,MAAM,IAAA,mBAAY,EAIzC,OAAO,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAA;QACrC,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAEvC,MAAM,CAAC,KAAK,CAAC,kDAAkD,gBAAgB,CAAC,IAAI,GAAG,CAAC,CAAA;QACxF,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAC/C,GAAG,EACH,gBAAgB,EAChB,MAAM,CACP,CAAmC,CAAA;QACpC,QAAQ,CAAC,UAAU,GAAG;YACpB,qBAAqB;YACrB,oBAAoB;YACpB,qBAAqB,EAAE,QAAQ,CAAC,UAAU,EAAE,qBAAqB;SAClE,CAAA;QAED,IAAI,MAAM,CAAC,WAAW,EAAE;YACtB,QAAQ,CAAC,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAA;SACtC;QAED,IAAI,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,4BAA4B,EAAE;YACjE,QAAQ,CAAC,IAAI,GAAG;gBACd,OAAO,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,IAAI,KAAK,EAAE;aACvE,CAAA;SACF;QAED,MAAM,MAAM,GAAiC;YAC3C,MAAM,EAAE,GAAG,CAAC,cAAc,CAAC,IAAI;YAC/B,QAAQ;SACT,CAAA;QAED,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAClD,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAExC,oCAAoC;QACpC,MAAM,IAAI,GAAG,gBAAgB,CAAC,YAAY,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAA;QACjE,gBAAgB,CAAC,0BAA0B;aACxC,MAAM,CAAC;YACN,OAAO,EAAE,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,IAAI,KAAK;YAC1D,cAAc,EAAE,GAAG,CAAC,cAAc,CAAC,QAAQ;SAC5C,CAAC;aACD,GAAG,CAAC,IAAI,CAAC,CAAA;QAEZ,uDAAuD;QACvD,GAAG,CAAC,cAAc,CAAC,IAAI,GAAG;YACxB,GAAG,GAAG,CAAC,cAAc,CAAC,IAAI;YAC1B,OAAO,EAAE,EAAE,GAAG,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE;SAClE,CAAA;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,OAAO,EAAE;YACjD,MAAM,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAA;YAC7E,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;SAC7C;QAED,OAAO,QAAQ,CAAA;IACjB,CAAC;CACF;AAlJD,sCAkJC"}
|
package/transports/sse.d.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { AxiosRequestConfig } from 'axios';
|
|
2
2
|
import EventSource from 'eventsource';
|
|
3
|
-
import { EndpointContext
|
|
4
|
-
import { Cache } from '../cache';
|
|
3
|
+
import { EndpointContext } from '../adapter';
|
|
5
4
|
import { AdapterConfig } from '../config';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
5
|
+
import { ProviderResult } from '../util/request';
|
|
6
|
+
import { TransportDependencies, TransportGenerics } from './';
|
|
7
|
+
import { StreamingTransport, SubscriptionDeltas } from './abstract/streaming';
|
|
9
8
|
export interface SSEConfig {
|
|
10
9
|
url: string;
|
|
11
10
|
eventSourceInitDict?: EventSource.EventSourceInitDict;
|
|
@@ -30,18 +29,15 @@ declare type SSETransportGenerics = TransportGenerics & {
|
|
|
30
29
|
*
|
|
31
30
|
* @typeParam T - Helper struct type that will be used to pass types to the generic parameters (check [[SSETransportGenerics]])
|
|
32
31
|
*/
|
|
33
|
-
export declare class SSETransport<T extends SSETransportGenerics>
|
|
32
|
+
export declare class SSETransport<T extends SSETransportGenerics> extends StreamingTransport<T> {
|
|
34
33
|
private config;
|
|
35
34
|
EventSource: typeof EventSource;
|
|
36
|
-
cache: Cache<AdapterResponse<T['Response']>>;
|
|
37
35
|
eventListeners: {
|
|
38
36
|
type: string;
|
|
39
37
|
parseResponse: (evt: MessageEvent) => ProviderResult<T>;
|
|
40
38
|
}[];
|
|
41
39
|
sseConnection?: EventSource;
|
|
42
|
-
subscriptionSet: SubscriptionSet<T['Request']['Params']>;
|
|
43
40
|
timeOfLastReq: number;
|
|
44
|
-
localSubscriptions: T['Request']['Params'][];
|
|
45
41
|
constructor(config: {
|
|
46
42
|
prepareSSEConnectionConfig: (params: T['Request']['Params'][], context: EndpointContext<T>) => SSEConfig;
|
|
47
43
|
prepareKeepAliveRequest?: (context: EndpointContext<T>) => AxiosRequestConfig<T['Provider']['RequestBody']>;
|
|
@@ -54,8 +50,8 @@ export declare class SSETransport<T extends SSETransportGenerics> implements Tra
|
|
|
54
50
|
keepaliveSleepMs?: number;
|
|
55
51
|
pollingSleepMs?: number;
|
|
56
52
|
});
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
53
|
+
getSubscriptionTtlFromConfig(config: AdapterConfig<T['CustomSettings']>): number;
|
|
54
|
+
initialize(dependencies: TransportDependencies<T>, config: AdapterConfig<T['CustomSettings']>, endpointName: string): Promise<void>;
|
|
55
|
+
streamHandler(context: EndpointContext<T>, subscriptions: SubscriptionDeltas<T['Request']['Params']>): Promise<void>;
|
|
60
56
|
}
|
|
61
57
|
export {};
|
package/transports/sse.js
CHANGED
|
@@ -7,57 +7,50 @@ exports.SSETransport = void 0;
|
|
|
7
7
|
const axios_1 = __importDefault(require("axios"));
|
|
8
8
|
const eventsource_1 = __importDefault(require("eventsource"));
|
|
9
9
|
const util_1 = require("../util");
|
|
10
|
-
const
|
|
10
|
+
const streaming_1 = require("./abstract/streaming");
|
|
11
11
|
const logger = (0, util_1.makeLogger)('SSETransport');
|
|
12
12
|
/**
|
|
13
13
|
* Transport implementation that establishes a long lived connection to a server using the SSE protocol and subcribes to updates.
|
|
14
14
|
*
|
|
15
15
|
* @typeParam T - Helper struct type that will be used to pass types to the generic parameters (check [[SSETransportGenerics]])
|
|
16
16
|
*/
|
|
17
|
-
class SSETransport {
|
|
17
|
+
class SSETransport extends streaming_1.StreamingTransport {
|
|
18
18
|
constructor(config) {
|
|
19
|
+
super();
|
|
19
20
|
this.config = config;
|
|
20
21
|
this.EventSource = eventsource_1.default;
|
|
21
22
|
this.timeOfLastReq = 0;
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
}
|
|
24
|
+
getSubscriptionTtlFromConfig(config) {
|
|
25
|
+
return config.SSE_SUBSCRIPTION_TTL;
|
|
25
26
|
}
|
|
26
27
|
async initialize(dependencies, config, endpointName) {
|
|
27
|
-
|
|
28
|
-
this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet(endpointName);
|
|
28
|
+
super.initialize(dependencies, config, endpointName);
|
|
29
29
|
if (dependencies.eventSource) {
|
|
30
30
|
this.EventSource = dependencies.eventSource;
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
|
-
async
|
|
34
|
-
|
|
35
|
-
await this.subscriptionSet.add(req.requestContext.cacheKey, req.requestContext.data, config.SSE_SUBSCRIPTION_TTL);
|
|
36
|
-
}
|
|
37
|
-
// Unlike cache warming, this execute will manage subscriptions
|
|
38
|
-
async backgroundExecute(context) {
|
|
39
|
-
logger.debug('Starting background execute, getting subscriptions from sorted set');
|
|
40
|
-
const desiredSubs = await this.subscriptionSet.getAll();
|
|
41
|
-
logger.debug('Generating delta (subscribes & unsubscribes)');
|
|
42
|
-
const subscribeParams = desiredSubs.filter((s) => !this.localSubscriptions.includes(s));
|
|
43
|
-
const unsubscribeParams = this.localSubscriptions.filter((s) => !desiredSubs.includes(s));
|
|
44
|
-
logger.debug(`${subscribeParams.length} new subscriptions; ${unsubscribeParams.length} to unsubscribe`);
|
|
45
|
-
if (subscribeParams.length) {
|
|
46
|
-
logger.trace(`Will subscribe to: ${subscribeParams}`);
|
|
47
|
-
}
|
|
48
|
-
if (unsubscribeParams.length) {
|
|
49
|
-
logger.trace(`Will unsubscribe to: ${unsubscribeParams}`);
|
|
50
|
-
}
|
|
51
|
-
if ((subscribeParams.length || unsubscribeParams.length) &&
|
|
33
|
+
async streamHandler(context, subscriptions) {
|
|
34
|
+
if ((subscriptions.new.length || subscriptions.stale.length) &&
|
|
52
35
|
(!this.sseConnection || this.sseConnection.readyState !== this.sseConnection.OPEN)) {
|
|
53
36
|
logger.debug('No established connection and new subscriptions available, connecting to SSE');
|
|
54
|
-
const sseConfig = this.config.prepareSSEConnectionConfig(
|
|
37
|
+
const sseConfig = this.config.prepareSSEConnectionConfig(subscriptions.new, context);
|
|
38
|
+
this.providerDataStreamEstablished = Date.now();
|
|
55
39
|
this.sseConnection = new this.EventSource(sseConfig.url, sseConfig.eventSourceInitDict);
|
|
56
40
|
const eventHandlerGenerator = (listener) => {
|
|
57
41
|
return (e) => {
|
|
58
|
-
const
|
|
59
|
-
const
|
|
60
|
-
|
|
42
|
+
const providerDataReceived = Date.now();
|
|
43
|
+
const results = listener.parseResponse(e).map((r) => {
|
|
44
|
+
const partialResponse = r.response;
|
|
45
|
+
const result = r;
|
|
46
|
+
result.response.timestamps = {
|
|
47
|
+
providerDataStreamEstablished: this.providerDataStreamEstablished,
|
|
48
|
+
providerDataReceived,
|
|
49
|
+
providerIndicatedTime: partialResponse.timestamps?.providerIndicatedTime,
|
|
50
|
+
};
|
|
51
|
+
return result;
|
|
52
|
+
});
|
|
53
|
+
this.responseCache.write(results);
|
|
61
54
|
};
|
|
62
55
|
};
|
|
63
56
|
this.config.eventListeners.forEach((listener) => {
|
|
@@ -74,24 +67,21 @@ class SSETransport {
|
|
|
74
67
|
}
|
|
75
68
|
this.timeOfLastReq = Date.now();
|
|
76
69
|
};
|
|
77
|
-
if (
|
|
78
|
-
const subscribeRequest = this.config.prepareSubscriptionRequest(
|
|
70
|
+
if (subscriptions.new.length) {
|
|
71
|
+
const subscribeRequest = this.config.prepareSubscriptionRequest(subscriptions.new, context);
|
|
79
72
|
makeRequest(subscribeRequest);
|
|
80
73
|
}
|
|
81
|
-
if (
|
|
82
|
-
const unsubscribeRequest = this.config.prepareUnsubscriptionRequest(
|
|
74
|
+
if (subscriptions.stale.length) {
|
|
75
|
+
const unsubscribeRequest = this.config.prepareUnsubscriptionRequest(subscriptions.stale, context);
|
|
83
76
|
makeRequest(unsubscribeRequest);
|
|
84
77
|
}
|
|
85
78
|
if (this.config.prepareKeepAliveRequest &&
|
|
86
|
-
|
|
79
|
+
subscriptions.desired.length &&
|
|
87
80
|
Date.now() - this.timeOfLastReq > context.adapterConfig.SSE_KEEPALIVE_SLEEP) {
|
|
88
81
|
const prepareKeepAliveRequest = this.config.prepareKeepAliveRequest(context);
|
|
89
82
|
makeRequest(prepareKeepAliveRequest);
|
|
90
83
|
}
|
|
91
|
-
|
|
92
|
-
this.localSubscriptions = desiredSubs;
|
|
93
|
-
logger.debug('Background execute complete');
|
|
94
|
-
return context.adapterConfig.SSE_SUBSCRIPTION_UPDATE_SLEEP;
|
|
84
|
+
return;
|
|
95
85
|
}
|
|
96
86
|
}
|
|
97
87
|
exports.SSETransport = SSETransport;
|
package/transports/sse.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sse.js","sourceRoot":"","sources":["../../../src/transports/sse.ts"],"names":[],"mappings":";;;;;;AAAA,kDAAiD;AACjD,8DAAqC;
|
|
1
|
+
{"version":3,"file":"sse.js","sourceRoot":"","sources":["../../../src/transports/sse.ts"],"names":[],"mappings":";;;;;;AAAA,kDAAiD;AACjD,8DAAqC;AAGrC,kCAAoC;AAOpC,oDAA6E;AAE7E,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,cAAc,CAAC,CAAA;AAuBzC;;;;GAIG;AACH,MAAa,YAA6C,SAAQ,8BAAqB;IASrF,YACU,MAsBP;QAED,KAAK,EAAE,CAAA;QAxBC,WAAM,GAAN,MAAM,CAsBb;QA/BH,gBAAW,GAAuB,qBAAW,CAAA;QAM7C,kBAAa,GAAG,CAAC,CAAA;IA4BjB,CAAC;IAED,4BAA4B,CAAC,MAA0C;QACrE,OAAO,MAAM,CAAC,oBAAoB,CAAA;IACpC,CAAC;IAEQ,KAAK,CAAC,UAAU,CACvB,YAAsC,EACtC,MAA0C,EAC1C,YAAoB;QAEpB,KAAK,CAAC,UAAU,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,CAAA;QACpD,IAAI,YAAY,CAAC,WAAW,EAAE;YAC5B,IAAI,CAAC,WAAW,GAAG,YAAY,CAAC,WAAW,CAAA;SAC5C;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,OAA2B,EAC3B,aAAyD;QAEzD,IACE,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC;YACxD,CAAC,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC,UAAU,KAAK,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAClF;YACA,MAAM,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAA;YAC5F,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YACpF,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC/C,IAAI,CAAC,aAAa,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,mBAAmB,CAAC,CAAA;YAEvF,MAAM,qBAAqB,GAAG,CAAC,QAA8C,EAAE,EAAE;gBAC/E,OAAO,CAAC,CAAe,EAAE,EAAE;oBACzB,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;oBACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;wBAClD,MAAM,eAAe,GAAG,CAAC,CAAC,QAAoD,CAAA;wBAC9E,MAAM,MAAM,GAAG,CAAiC,CAAA;wBAChD,MAAM,CAAC,QAAQ,CAAC,UAAU,GAAG;4BAC3B,6BAA6B,EAAE,IAAI,CAAC,6BAA6B;4BACjE,oBAAoB;4BACpB,qBAAqB,EAAE,eAAe,CAAC,UAAU,EAAE,qBAAqB;yBACzE,CAAA;wBACD,OAAO,MAAM,CAAA;oBACf,CAAC,CAAC,CAAA;oBACF,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;gBACnC,CAAC,CAAA;YACH,CAAC,CAAA;YAED,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAC9C,IAAI,CAAC,aAAa,EAAE,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CAAA;YACtF,CAAC,CAAC,CAAA;SACH;QAED,MAAM,WAAW,GAAG,KAAK,EAAE,GAAqD,EAAE,EAAE;YAClF,IAAI;gBACF,MAAM,GAAG,GAAG,MAAM,eAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;gBACpC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,mBAAmB,GAAG,CAAC,UAAU,yBAAyB,CAAC,CAAA;aACnF;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAA;aAChD;YACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACjC,CAAC,CAAA;QAED,IAAI,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE;YAC5B,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAC3F,WAAW,CAAC,gBAAgB,CAAC,CAAA;SAC9B;QACD,IAAI,aAAa,CAAC,KAAK,CAAC,MAAM,EAAE;YAC9B,MAAM,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,4BAA4B,CACjE,aAAa,CAAC,KAAK,EACnB,OAAO,CACR,CAAA;YACD,WAAW,CAAC,kBAAkB,CAAC,CAAA;SAChC;QACD,IACE,IAAI,CAAC,MAAM,CAAC,uBAAuB;YACnC,aAAa,CAAC,OAAO,CAAC,MAAM;YAC5B,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,mBAAmB,EAC3E;YACA,MAAM,uBAAuB,GAAG,IAAI,CAAC,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAA;YAC5E,WAAW,CAAC,uBAAuB,CAAC,CAAA;SACrC;QAED,OAAM;IACR,CAAC;CACF;AAvHD,oCAuHC"}
|