@chainlink/external-adapter-framework 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapter/basic.js +10 -31
- package/adapter/basic.js.map +1 -1
- package/adapter/price.d.ts +2 -2
- package/adapter/price.js.map +1 -1
- package/adapter/types.d.ts +7 -1
- package/background-executor.js +4 -26
- package/background-executor.js.map +1 -1
- package/cache/local.d.ts +1 -17
- package/cache/local.js +2 -54
- package/cache/local.js.map +1 -1
- package/cache/metrics.d.ts +1 -7
- package/cache/metrics.js +10 -102
- package/cache/metrics.js.map +1 -1
- package/cache/redis.d.ts +3 -4
- package/cache/redis.js +13 -37
- package/cache/redis.js.map +1 -1
- package/config/index.d.ts +6 -0
- package/config/index.js +6 -0
- package/config/index.js.map +1 -1
- package/index.d.ts +0 -5
- package/index.js +2 -33
- package/index.js.map +1 -1
- package/metrics/index.d.ts +43 -5
- package/metrics/index.js +194 -31
- package/metrics/index.js.map +1 -1
- package/package.json +3 -3
- package/rate-limiting/metrics.d.ts +0 -2
- package/rate-limiting/metrics.js +1 -30
- package/rate-limiting/metrics.js.map +1 -1
- package/transports/abstract/subscription.js +4 -26
- package/transports/abstract/subscription.js.map +1 -1
- package/transports/http.js +7 -33
- package/transports/http.js.map +1 -1
- package/transports/metrics.d.ts +0 -9
- package/transports/metrics.js +8 -74
- package/transports/metrics.js.map +1 -1
- package/transports/websocket.js +7 -35
- package/transports/websocket.js.map +1 -1
- package/util/index.d.ts +16 -0
- package/util/index.js +55 -1
- package/util/index.js.map +1 -1
- package/util/metrics.d.ts +7 -8
- package/util/metrics.js +22 -25
- package/util/metrics.js.map +1 -1
- package/util/requester.js +11 -32
- package/util/requester.js.map +1 -1
- package/util/subscription-set/expiring-sorted-set.d.ts +20 -3
- package/util/subscription-set/expiring-sorted-set.js +63 -10
- package/util/subscription-set/expiring-sorted-set.js.map +1 -1
- package/util/subscription-set/redis-sorted-set.d.ts +1 -0
- package/util/subscription-set/redis-sorted-set.js +3 -0
- package/util/subscription-set/redis-sorted-set.js.map +1 -1
- package/util/subscription-set/subscription-set.d.ts +3 -1
- package/util/subscription-set/subscription-set.js +2 -1
- package/util/subscription-set/subscription-set.js.map +1 -1
package/util/metrics.js
CHANGED
|
@@ -23,31 +23,28 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.
|
|
26
|
+
exports.requestMetrics = void 0;
|
|
27
27
|
const client = __importStar(require("prom-client"));
|
|
28
|
+
const metrics_1 = require("../metrics");
|
|
28
29
|
const constants_1 = require("../metrics/constants");
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
})
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
});
|
|
49
|
-
exports.requesterQueueOverflow = new client.Counter({
|
|
50
|
-
name: 'requester_queue_overflow',
|
|
51
|
-
help: 'Total times the requester queue replaced the oldest item to avoid an overflow',
|
|
52
|
-
});
|
|
30
|
+
exports.requestMetrics = new metrics_1.Metrics(() => ({
|
|
31
|
+
dataProviderRequests: new client.Counter({
|
|
32
|
+
name: 'data_provider_requests',
|
|
33
|
+
help: 'The number of http requests that are made to a data provider',
|
|
34
|
+
labelNames: ['method', 'provider_status_code'],
|
|
35
|
+
}),
|
|
36
|
+
dataProviderRequestDurationSeconds: new client.Histogram({
|
|
37
|
+
name: 'data_provider_request_duration_seconds',
|
|
38
|
+
help: 'A histogram bucket of the distribution of data provider request durations',
|
|
39
|
+
buckets: constants_1.requestDurationBuckets,
|
|
40
|
+
}),
|
|
41
|
+
requesterQueueSize: new client.Gauge({
|
|
42
|
+
name: 'requester_queue_size',
|
|
43
|
+
help: 'The number of provider http requests currently queued to be executed',
|
|
44
|
+
}),
|
|
45
|
+
requesterQueueOverflow: new client.Counter({
|
|
46
|
+
name: 'requester_queue_overflow',
|
|
47
|
+
help: 'Total times the requester queue replaced the oldest item to avoid an overflow',
|
|
48
|
+
}),
|
|
49
|
+
}));
|
|
53
50
|
//# sourceMappingURL=metrics.js.map
|
package/util/metrics.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/util/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AACrC,
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/util/metrics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAqC;AACrC,wCAAoC;AACpC,oDAA6D;AAEhD,QAAA,cAAc,GAAG,IAAI,iBAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC/C,oBAAoB,EAAE,IAAI,MAAM,CAAC,OAAO,CAAC;QACvC,IAAI,EAAE,wBAAwB;QAC9B,IAAI,EAAE,8DAA8D;QACpE,UAAU,EAAE,CAAC,QAAQ,EAAE,sBAAsB,CAAU;KACxD,CAAC;IACF,kCAAkC,EAAE,IAAI,MAAM,CAAC,SAAS,CAAC;QACvD,IAAI,EAAE,wCAAwC;QAC9C,IAAI,EAAE,2EAA2E;QACjF,OAAO,EAAE,kCAAsB;KAChC,CAAC;IACF,kBAAkB,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC;QACnC,IAAI,EAAE,sBAAsB;QAC5B,IAAI,EAAE,sEAAsE;KAC7E,CAAC;IACF,sBAAsB,EAAE,IAAI,MAAM,CAAC,OAAO,CAAC;QACzC,IAAI,EAAE,0BAA0B;QAChC,IAAI,EAAE,+EAA+E;KACtF,CAAC;CACH,CAAC,CAAC,CAAA"}
|
package/util/requester.js
CHANGED
|
@@ -1,27 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
4
|
};
|
|
@@ -29,8 +6,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
29
6
|
exports.Requester = void 0;
|
|
30
7
|
const axios_1 = __importDefault(require("axios"));
|
|
31
8
|
const _1 = require(".");
|
|
9
|
+
const metrics_1 = require("../metrics");
|
|
32
10
|
const error_1 = require("../validation/error");
|
|
33
|
-
const metrics = __importStar(require("./metrics"));
|
|
34
11
|
const logger = (0, _1.makeLogger)('Requester');
|
|
35
12
|
class UniqueLinkedList {
|
|
36
13
|
constructor(maxLength) {
|
|
@@ -55,7 +32,7 @@ class UniqueLinkedList {
|
|
|
55
32
|
}
|
|
56
33
|
this.last = node;
|
|
57
34
|
this.length++;
|
|
58
|
-
metrics.requesterQueueSize.inc();
|
|
35
|
+
metrics_1.metrics.get('requesterQueueSize').inc();
|
|
59
36
|
return overflow;
|
|
60
37
|
}
|
|
61
38
|
remove() {
|
|
@@ -65,7 +42,7 @@ class UniqueLinkedList {
|
|
|
65
42
|
}
|
|
66
43
|
this.first = node.next;
|
|
67
44
|
this.length--;
|
|
68
|
-
metrics.requesterQueueSize.dec();
|
|
45
|
+
metrics_1.metrics.get('requesterQueueSize').dec();
|
|
69
46
|
return node.value;
|
|
70
47
|
}
|
|
71
48
|
}
|
|
@@ -127,7 +104,7 @@ class Requester {
|
|
|
127
104
|
if (overflowedRequest) {
|
|
128
105
|
// If we have overflow, it means the oldest request needs to be rejected because the queue is at its limits
|
|
129
106
|
logger.debug(`Request (Key: ${key}, Retry #: ${0}) was removed from the queue to make room for a newer one (Size: ${this.queue.length})`);
|
|
130
|
-
metrics.requesterQueueOverflow.inc();
|
|
107
|
+
metrics_1.metrics.get('requesterQueueOverflow').inc();
|
|
131
108
|
overflowedRequest.reject(new error_1.AdapterRateLimitError({
|
|
132
109
|
message: 'The EA was unable to execute the request to fetch the requested data from the DP because the request queue overflowed. This likely indicates that a higher API tier is needed.',
|
|
133
110
|
statusCode: 429,
|
|
@@ -170,7 +147,7 @@ class Requester {
|
|
|
170
147
|
async executeRequest(req) {
|
|
171
148
|
const { key, config, resolve, reject, retries } = req;
|
|
172
149
|
const providerDataRequested = Date.now();
|
|
173
|
-
const responseTimer = metrics.dataProviderRequestDurationSeconds.startTimer();
|
|
150
|
+
const responseTimer = metrics_1.metrics.get('dataProviderRequestDurationSeconds').startTimer();
|
|
174
151
|
// Set configured timeout for all requests unless manually specified
|
|
175
152
|
config.timeout = config.timeout || this.timeout;
|
|
176
153
|
try {
|
|
@@ -187,8 +164,9 @@ class Requester {
|
|
|
187
164
|
// Remove the request from our map
|
|
188
165
|
delete this.map[key];
|
|
189
166
|
// Record count of successful data provider requests
|
|
190
|
-
metrics
|
|
191
|
-
.
|
|
167
|
+
metrics_1.metrics
|
|
168
|
+
.get('dataProviderRequests')
|
|
169
|
+
.labels((0, metrics_1.dataProviderMetricsLabel)(response.status, config.method))
|
|
192
170
|
.inc();
|
|
193
171
|
}
|
|
194
172
|
catch (e) {
|
|
@@ -197,8 +175,9 @@ class Requester {
|
|
|
197
175
|
const err = e;
|
|
198
176
|
const ErrorClass = err.response?.status ? error_1.AdapterDataProviderError : error_1.AdapterConnectionError;
|
|
199
177
|
// Record count of failed data provider request
|
|
200
|
-
metrics
|
|
201
|
-
.
|
|
178
|
+
metrics_1.metrics
|
|
179
|
+
.get('dataProviderRequests')
|
|
180
|
+
.labels((0, metrics_1.dataProviderMetricsLabel)(err.response?.status || 0, config.method))
|
|
202
181
|
.inc();
|
|
203
182
|
reject(new ErrorClass({
|
|
204
183
|
statusCode: 502,
|
package/util/requester.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"requester.js","sourceRoot":"","sources":["../../../src/util/requester.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"requester.js","sourceRoot":"","sources":["../../../src/util/requester.ts"],"names":[],"mappings":";;;;;;AAAA,kDAA4E;AAC5E,wBAAqC;AAErC,wCAA8D;AAE9D,+CAI4B;AAE5B,MAAM,MAAM,GAAG,IAAA,aAAU,EAAC,WAAW,CAAC,CAAA;AAOtC,MAAM,gBAAgB;IAKpB,YAAoB,SAAiB;QAAjB,cAAS,GAAT,SAAS,CAAQ;QAFrC,WAAM,GAAG,CAAC,CAAA;IAE8B,CAAC;IAEzC,GAAG,CAAC,KAAQ;QACV,IAAI,QAAQ,CAAA;QACZ,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,SAAS,EAAE;YAClC,wFAAwF;YACxF,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;SACzB;QAED,MAAM,IAAI,GAAgB;YACxB,KAAK;YACL,IAAI,EAAE,SAAS;SAChB,CAAA;QAED,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;SAClB;QACD,IAAI,IAAI,CAAC,IAAI,EAAE;YACb,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;SACtB;QAED,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAChB,IAAI,CAAC,MAAM,EAAE,CAAA;QACb,iBAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,GAAG,EAAE,CAAA;QACvC,OAAO,QAAQ,CAAA;IACjB,CAAC;IAED,MAAM;QACJ,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAA;QAEvB,IAAI,CAAC,IAAI,EAAE;YACT,OAAM;SACP;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAA;QACtB,IAAI,CAAC,MAAM,EAAE,CAAA;QACb,iBAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,GAAG,EAAE,CAAA;QACvC,OAAO,IAAI,CAAC,KAAK,CAAA;IACnB,CAAC;CACF;AAmBD;;;;;;;;;;;GAWG;AACH,MAAa,SAAS;IAOpB,YAAoB,WAAwB,EAAE,MAAqB;QAA/C,gBAAW,GAAX,WAAW,CAAa;QANpC,eAAU,GAAG,KAAK,CAAA;QAElB,QAAG,GAAG,EAAmC,CAAA;QAK/C,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,KAAK,CAAA;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,WAAW,CAAA;QACjC,IAAI,CAAC,KAAK,GAAG,IAAI,gBAAgB,CAAgB,MAAM,CAAC,6BAA6B,CAAC,CAAA;IACxF,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAI,GAAW,EAAE,GAAuB;QACnD,mEAAmE;QACnE,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3C,IAAI,qBAAqB,EAAE;YACzB,MAAM,CAAC,KAAK,CAAC,0DAA0D,GAAG,GAAG,CAAC,CAAA;YAC9E,OAAO,qBAAqB,CAAC,OAAsC,CAAA;SACpE;QAED,MAAM,aAAa,GAAG;YACpB,GAAG;YACH,MAAM,EAAE,GAAG;YACX,OAAO,EAAE,CAAC;SACS,CAAA;QAErB,uGAAuG;QACvG,8FAA8F;QAC9F,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC5B,aAAa,CAAC,OAAO,GAAG,IAAI,OAAO,CAAqB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;gBAC3E,aAAa,CAAC,OAAO,GAAG,OAAO,CAAA;gBAC/B,aAAa,CAAC,MAAM,GAAG,OAAO,CAAA;gBAC9B,OAAO,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,iIAAiI;QACjI,+HAA+H;QAC/H,qIAAqI;QACrI,oJAAoJ;QACpJ,iHAAiH;QACjH,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAA8B,CAAC,CAAA;QACxE,IAAI,iBAAiB,EAAE;YACrB,2GAA2G;YAC3G,MAAM,CAAC,KAAK,CACV,iBAAiB,GAAG,cAAc,CAAC,oEACjC,IAAI,CAAC,KAAK,CAAC,MACb,GAAG,CACJ,CAAA;YACD,iBAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,GAAG,EAAE,CAAA;YAC3C,iBAAiB,CAAC,MAAM,CACtB,IAAI,6BAAqB,CAAC;gBACxB,OAAO,EACL,gLAAgL;gBAClL,UAAU,EAAE,GAAG;aAChB,CAAC,CACH,CAAA;SACF;QAED,iFAAiF;QACjF,MAAM,CAAC,KAAK,CACV,uBAAuB,GAAG,cAAc,CAAC,yBAAyB,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CACvF,CAAA;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,aAA8B,CAAA;QAE9C,yCAAyC;QACzC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;YACtB,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;YACnD,iGAAiG;YACjG,4FAA4F;YAC5F,IAAI,CAAC,WAAW,EAAE,CAAA;SACnB;QAED,OAAO,aAAa,CAAC,OAAO,CAAA;IAC9B,CAAC;IAED,+EAA+E;IACvE,KAAK,CAAC,WAAW;QACvB,8GAA8G;QAC9G,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAA;QAEhC,IAAI,CAAC,IAAI,EAAE;YACT,MAAM,CAAC,KAAK,CACV,mFAAmF,CACpF,CAAA;YACD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;YACvB,OAAM;SACP;QAED,MAAM,CAAC,KAAK,CACV,6BAA6B,IAAI,CAAC,GAAG,cAAc,IAAI,CAAC,OAAO,2BAA2B,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAC/G,CAAA;QAED,qDAAqD;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,EAAE,CAAA;QAC1D,IAAI,UAAU,EAAE;YACd,MAAM,CAAC,KAAK,CACV,qCAAqC,UAAU,6BAA6B,IAAI,CAAC,GAAG,cAAc,IAAI,CAAC,OAAO,GAAG,CAClH,CAAA;YACD,MAAM,IAAA,QAAK,EAAC,UAAU,CAAC,CAAA;SACxB;QAED,mHAAmH;QACnH,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAA;QAEpC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAA;IAC3B,CAAC;IAED,uHAAuH;IAC/G,KAAK,CAAC,cAAc,CAAC,GAAkB;QAC7C,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,GAAG,CAAA;QACrD,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxC,MAAM,aAAa,GAAG,iBAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC,UAAU,EAAE,CAAA;QAEpF,oEAAoE;QACpE,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAA;QAE/C,IAAI;YACF,MAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,oBAAoB,CAAC,CAAA;YAC9D,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YAC5C,MAAM,CAAC,KAAK,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,CAAA;YACrD,OAAO,CAAC;gBACN,QAAQ;gBACR,UAAU,EAAE;oBACV,qBAAqB;oBACrB,oBAAoB,EAAE,IAAI,CAAC,GAAG,EAAE;iBACjC;aACF,CAAC,CAAA;YAEF,kCAAkC;YAClC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEpB,oDAAoD;YACpD,iBAAO;iBACJ,GAAG,CAAC,sBAAsB,CAAC;iBAC3B,MAAM,CAAC,IAAA,kCAAwB,EAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;iBAChE,GAAG,EAAE,CAAA;SACT;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE;gBAC9B,MAAM,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAA;gBAClF,MAAM,GAAG,GAAG,CAAe,CAAA;gBAC3B,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,gCAAwB,CAAC,CAAC,CAAC,8BAAsB,CAAA;gBAE3F,+CAA+C;gBAC/C,iBAAO;qBACJ,GAAG,CAAC,sBAAsB,CAAC;qBAC3B,MAAM,CAAC,IAAA,kCAAwB,EAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;qBAC1E,GAAG,EAAE,CAAA;gBAER,MAAM,CACJ,IAAI,UAAU,CACZ;oBACE,UAAU,EAAE,GAAG;oBACf,IAAI,EAAE,qBAAqB;oBAC3B,kBAAkB,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG;oBAChD,OAAO,EAAE,GAAG,EAAE,OAAO;oBACrB,KAAK,EAAE,CAAC;oBACR,aAAa,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI;oBAClC,GAAG,EAAE,MAAM,CAAC,GAAG;iBAChB,EACD;oBACE,qBAAqB;oBACrB,oBAAoB,EAAE,IAAI,CAAC,GAAG,EAAE;oBAChC,qBAAqB,EAAE,SAAS;iBACjC,CACF,CACF,CAAA;gBAED,kCAAkC;gBAClC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;aACrB;iBAAM;gBACL,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,CAAA;gBACzD,MAAM,CAAC,KAAK,CAAC,gCAAgC,WAAW,OAAO,CAAC,CAAA;gBAChE,MAAM,IAAA,QAAK,EAAC,WAAW,CAAC,CAAA;gBAExB,GAAG,CAAC,OAAO,EAAE,CAAA;gBACb,MAAM,CAAC,KAAK,CAAC,qCAAqC,GAAG,cAAc,GAAG,CAAC,OAAO,GAAG,CAAC,CAAA;gBAClF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;aACpB;SACF;gBAAS;YACR,qEAAqE;YACrE,aAAa,EAAE,CAAA;SAChB;IACH,CAAC;CACF;AAjMD,8BAiMC"}
|
|
@@ -1,4 +1,13 @@
|
|
|
1
1
|
import { SubscriptionSet } from './subscription-set';
|
|
2
|
+
import { DoubleLinkedList, LinkedListNode, PromiseOrValue } from '..';
|
|
3
|
+
/**
|
|
4
|
+
* An object describing an entry in the expiring sorted set.
|
|
5
|
+
* @typeParam T - the type of the entry's value
|
|
6
|
+
*/
|
|
7
|
+
interface ExpiringSortedSetEntry<T> {
|
|
8
|
+
value: T;
|
|
9
|
+
expirationTimestamp: number;
|
|
10
|
+
}
|
|
2
11
|
/**
|
|
3
12
|
* This class implements a set of unique items, each of which has an expiration timestamp.
|
|
4
13
|
* On reads, items that have expired will be deleted from the set and not returned.
|
|
@@ -6,7 +15,15 @@ import { SubscriptionSet } from './subscription-set';
|
|
|
6
15
|
* @typeParam T - the type of the set entries' values
|
|
7
16
|
*/
|
|
8
17
|
export declare class ExpiringSortedSet<T> implements SubscriptionSet<T> {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
18
|
+
capacity: number;
|
|
19
|
+
map: Map<string, LinkedListNode<ExpiringSortedSetEntry<T>>>;
|
|
20
|
+
list: DoubleLinkedList;
|
|
21
|
+
constructor(capacity: number);
|
|
22
|
+
add(value: T, ttl: number, key: string): void;
|
|
23
|
+
getAll(): PromiseOrValue<T[]>;
|
|
24
|
+
get(key: string): T | undefined;
|
|
25
|
+
delete(key: string): void;
|
|
26
|
+
private moveToTail;
|
|
27
|
+
private evictIfNeeded;
|
|
12
28
|
}
|
|
29
|
+
export {};
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ExpiringSortedSet = void 0;
|
|
4
|
+
const __1 = require("..");
|
|
5
|
+
const logger = (0, __1.makeLogger)('ExpiringSortedSet');
|
|
4
6
|
/**
|
|
5
7
|
* This class implements a set of unique items, each of which has an expiration timestamp.
|
|
6
8
|
* On reads, items that have expired will be deleted from the set and not returned.
|
|
@@ -8,25 +10,76 @@ exports.ExpiringSortedSet = void 0;
|
|
|
8
10
|
* @typeParam T - the type of the set entries' values
|
|
9
11
|
*/
|
|
10
12
|
class ExpiringSortedSet {
|
|
11
|
-
constructor() {
|
|
13
|
+
constructor(capacity) {
|
|
14
|
+
this.capacity = capacity;
|
|
12
15
|
this.map = new Map();
|
|
16
|
+
this.list = new __1.DoubleLinkedList();
|
|
13
17
|
}
|
|
14
|
-
add(value, ttl) {
|
|
15
|
-
this.map.
|
|
18
|
+
add(value, ttl, key) {
|
|
19
|
+
let node = this.map.get(key);
|
|
20
|
+
if (node) {
|
|
21
|
+
node.data = {
|
|
22
|
+
value,
|
|
23
|
+
expirationTimestamp: Date.now() + ttl,
|
|
24
|
+
};
|
|
25
|
+
this.moveToTail(node);
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
this.evictIfNeeded();
|
|
29
|
+
const data = {
|
|
30
|
+
value,
|
|
31
|
+
expirationTimestamp: Date.now() + ttl,
|
|
32
|
+
};
|
|
33
|
+
node = new __1.LinkedListNode(key, data);
|
|
34
|
+
this.list.insertAtTail(node);
|
|
35
|
+
this.map.set(key, node);
|
|
36
|
+
}
|
|
16
37
|
}
|
|
17
38
|
getAll() {
|
|
18
39
|
const results = [];
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
40
|
+
for (const [key] of this.map.entries()) {
|
|
41
|
+
const value = this.get(key);
|
|
42
|
+
if (value) {
|
|
43
|
+
results.push(value);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return results;
|
|
47
|
+
}
|
|
48
|
+
get(key) {
|
|
49
|
+
const node = this.map.get(key);
|
|
50
|
+
if (node) {
|
|
51
|
+
const expired = node.data.expirationTimestamp <= Date.now();
|
|
52
|
+
if (expired) {
|
|
53
|
+
this.delete(key);
|
|
54
|
+
return undefined;
|
|
24
55
|
}
|
|
25
56
|
else {
|
|
26
|
-
|
|
57
|
+
return node.data.value;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
return undefined;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
delete(key) {
|
|
65
|
+
const node = this.map.get(key);
|
|
66
|
+
if (node) {
|
|
67
|
+
this.list.remove(node);
|
|
68
|
+
this.map.delete(key);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
moveToTail(node) {
|
|
72
|
+
this.list.remove(node);
|
|
73
|
+
this.list.insertAtTail(node);
|
|
74
|
+
}
|
|
75
|
+
evictIfNeeded() {
|
|
76
|
+
if (this.list.size >= this.capacity) {
|
|
77
|
+
const node = this.list.removeHead();
|
|
78
|
+
if (node) {
|
|
79
|
+
logger.warn(`List reached maximum capacity, evicting least recently updated entry. The subscription with key ${node.key} was removed.`);
|
|
80
|
+
this.map.delete(node.key);
|
|
27
81
|
}
|
|
28
82
|
}
|
|
29
|
-
return results;
|
|
30
83
|
}
|
|
31
84
|
}
|
|
32
85
|
exports.ExpiringSortedSet = ExpiringSortedSet;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"expiring-sorted-set.js","sourceRoot":"","sources":["../../../../src/util/subscription-set/expiring-sorted-set.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"expiring-sorted-set.js","sourceRoot":"","sources":["../../../../src/util/subscription-set/expiring-sorted-set.ts"],"names":[],"mappings":";;;AACA,0BAAiF;AAEjF,MAAM,MAAM,GAAG,IAAA,cAAU,EAAC,mBAAmB,CAAC,CAAA;AAW9C;;;;;GAKG;AACH,MAAa,iBAAiB;IAK5B,YAAY,QAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAA;QACxB,IAAI,CAAC,GAAG,GAAG,IAAI,GAAG,EAAqD,CAAA;QACvE,IAAI,CAAC,IAAI,GAAG,IAAI,oBAAgB,EAAE,CAAA;IACpC,CAAC;IAED,GAAG,CAAC,KAAQ,EAAE,GAAW,EAAE,GAAW;QACpC,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,IAAI,EAAE;YACR,IAAI,CAAC,IAAI,GAAG;gBACV,KAAK;gBACL,mBAAmB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG;aACtC,CAAA;YACD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;SACtB;aAAM;YACL,IAAI,CAAC,aAAa,EAAE,CAAA;YACpB,MAAM,IAAI,GAAG;gBACX,KAAK;gBACL,mBAAmB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG;aACtC,CAAA;YACD,IAAI,GAAG,IAAI,kBAAc,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACpC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;YAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;SACxB;IACH,CAAC;IAED,MAAM;QACJ,MAAM,OAAO,GAAQ,EAAE,CAAA;QACvB,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE;YACtC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAM,CAAA;YAChC,IAAI,KAAK,EAAE;gBACT,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;aACpB;SACF;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,GAAG,CAAC,GAAW;QACb,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC9B,IAAI,IAAI,EAAE;YACR,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAA;YAC3D,IAAI,OAAO,EAAE;gBACX,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAChB,OAAO,SAAS,CAAA;aACjB;iBAAM;gBACL,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAA;aACvB;SACF;aAAM;YACL,OAAO,SAAS,CAAA;SACjB;IACH,CAAC;IAED,MAAM,CAAC,GAAW;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC9B,IAAI,IAAI,EAAE;YACR,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACtB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;SACrB;IACH,CAAC;IAEO,UAAU,CAAC,IAAoB;QACrC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QACtB,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;IAC9B,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAA;YACnC,IAAI,IAAI,EAAE;gBACR,MAAM,CAAC,IAAI,CACT,mGAAmG,IAAI,CAAC,GAAG,eAAe,CAC3H,CAAA;gBACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;aAC1B;SACF;IACH,CAAC;CACF;AAjFD,8CAiFC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redis-sorted-set.js","sourceRoot":"","sources":["../../../../src/util/subscription-set/redis-sorted-set.ts"],"names":[],"mappings":";;;AAGA,MAAa,oBAAoB;IAK/B,YAAY,WAAkB,EAAE,kBAA0B;QACxD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAA;QAC9B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAA;IAC9C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAQ,EAAE,GAAW;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QACzC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,WAAW,CAAC,CAAA;QACnF,OAAM;IACR,CAAC;IAED,KAAK,CAAC,MAAM;QACV,sCAAsC;QACtC,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QACpF,MAAM,cAAc,GAAQ,EAAE,CAAA;QAC9B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAClF,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YAC7B,mEAAmE;YACnE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;QACF,OAAO,cAAc,CAAA;IACvB,CAAC;CACF;
|
|
1
|
+
{"version":3,"file":"redis-sorted-set.js","sourceRoot":"","sources":["../../../../src/util/subscription-set/redis-sorted-set.ts"],"names":[],"mappings":";;;AAGA,MAAa,oBAAoB;IAK/B,YAAY,WAAkB,EAAE,kBAA0B;QACxD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAA;QAC9B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAA;IAC9C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAQ,EAAE,GAAW;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QACzC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,WAAW,CAAC,CAAA;QACnF,OAAM;IACR,CAAC;IAED,KAAK,CAAC,MAAM;QACV,sCAAsC;QACtC,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QACpF,MAAM,cAAc,GAAQ,EAAE,CAAA;QAC9B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAClF,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YAC7B,mEAAmE;YACnE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;QACF,OAAO,cAAc,CAAA;IACvB,CAAC;IAED,GAAG;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;CACF;AA/BD,oDA+BC"}
|
|
@@ -6,14 +6,16 @@ import { AdapterConfig } from '../../config';
|
|
|
6
6
|
*/
|
|
7
7
|
export interface SubscriptionSet<T> {
|
|
8
8
|
/** Add a new subscription to the set */
|
|
9
|
-
add(value: T, ttl: number): PromiseOrValue<void>;
|
|
9
|
+
add(value: T, ttl: number, key?: string): PromiseOrValue<void>;
|
|
10
10
|
/** Get all subscriptions from the set as a list */
|
|
11
11
|
getAll(): PromiseOrValue<T[]>;
|
|
12
|
+
get: (key: string) => T | undefined;
|
|
12
13
|
}
|
|
13
14
|
export declare class SubscriptionSetFactory {
|
|
14
15
|
private cacheType;
|
|
15
16
|
private redisClient?;
|
|
16
17
|
private adapterName?;
|
|
18
|
+
private capacity;
|
|
17
19
|
constructor(config: AdapterConfig, adapterName: string, redisClient?: Redis);
|
|
18
20
|
buildSet<T>(endpointName: string): SubscriptionSet<T>;
|
|
19
21
|
}
|
|
@@ -8,11 +8,12 @@ class SubscriptionSetFactory {
|
|
|
8
8
|
this.cacheType = config.CACHE_TYPE;
|
|
9
9
|
this.redisClient = redisClient;
|
|
10
10
|
this.adapterName = adapterName;
|
|
11
|
+
this.capacity = config.SUBSCRIPTION_SET_MAX_ITEMS;
|
|
11
12
|
}
|
|
12
13
|
buildSet(endpointName) {
|
|
13
14
|
switch (this.cacheType) {
|
|
14
15
|
case 'local':
|
|
15
|
-
return new expiring_sorted_set_1.ExpiringSortedSet();
|
|
16
|
+
return new expiring_sorted_set_1.ExpiringSortedSet(this.capacity);
|
|
16
17
|
case 'redis': {
|
|
17
18
|
if (!this.redisClient) {
|
|
18
19
|
throw new Error('Redis client undefined. Cannot create Redis subscription set');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"subscription-set.js","sourceRoot":"","sources":["../../../../src/util/subscription-set/subscription-set.ts"],"names":[],"mappings":";;;AAGA,+DAAyD;AACzD,yDAAyD;
|
|
1
|
+
{"version":3,"file":"subscription-set.js","sourceRoot":"","sources":["../../../../src/util/subscription-set/subscription-set.ts"],"names":[],"mappings":";;;AAGA,+DAAyD;AACzD,yDAAyD;AAezD,MAAa,sBAAsB;IAMjC,YAAY,MAAqB,EAAE,WAAmB,EAAE,WAAmB;QACzE,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,UAAU,CAAA;QAClC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAA;QAC9B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAA;QAC9B,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,0BAA0B,CAAA;IACnD,CAAC;IAED,QAAQ,CAAI,YAAoB;QAC9B,QAAQ,IAAI,CAAC,SAAS,EAAE;YACtB,KAAK,OAAO;gBACV,OAAO,IAAI,uCAAiB,CAAI,IAAI,CAAC,QAAQ,CAAC,CAAA;YAChD,KAAK,OAAO,CAAC,CAAC;gBACZ,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;oBACrB,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAA;iBAChF;gBACD,wDAAwD;gBACxD,MAAM,kBAAkB,GAAG,GAAG,IAAI,CAAC,WAAW,IAAI,YAAY,kBAAkB,CAAA;gBAChF,OAAO,IAAI,uCAAoB,CAAI,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAA;aACzE;SACF;IACH,CAAC;CACF;AA3BD,wDA2BC"}
|