@friggframework/core 2.0.0--canary.658.93c8e07.0 → 2.0.0--canary.658.164d92b.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +13 -54
- package/core/invocation-deadline.js +10 -19
- package/core/invocation-scope.js +1 -9
- package/errors/fetch-error.js +7 -9
- package/errors/rate-limit-error.js +0 -4
- package/integrations/integration-base.js +3 -23
- package/integrations/repositories/integration-repository-documentdb.js +9 -15
- package/integrations/repositories/integration-repository-interface.js +6 -9
- package/integrations/repositories/integration-repository-mongo.js +10 -17
- package/integrations/repositories/integration-repository-postgres.js +10 -17
- package/integrations/repositories/message-item-shared.js +3 -23
- package/integrations/tests/doubles/prisma-message-row.js +24 -0
- package/integrations/tests/doubles/test-integration-repository.js +3 -3
- package/integrations/use-cases/record-rate-limit-message.js +46 -21
- package/integrations/use-cases/update-integration-messages.js +21 -4
- package/logs/serialize.js +15 -0
- package/modules/module.js +6 -9
- package/modules/requester/rate-limit/parsers.js +1 -24
- package/modules/requester/rate-limit/policy.js +21 -52
- package/modules/requester/requester.js +38 -72
- package/package.json +5 -5
- package/types/core/index.d.ts +0 -9
- package/types/errors/index.d.ts +0 -15
- package/types/integrations/index.d.ts +0 -5
- package/types/module-plugin/index.d.ts +1 -15
|
@@ -1,7 +1,19 @@
|
|
|
1
|
-
const
|
|
1
|
+
const { getLogger } = require('../../logs');
|
|
2
|
+
const { MAX_MESSAGES_PER_TYPE } = require('./update-integration-messages');
|
|
3
|
+
|
|
4
|
+
const log = getLogger('frigg.integrations');
|
|
5
|
+
|
|
6
|
+
const RATE_LIMITED_MESSAGE_CODE = 'RATE_LIMITED';
|
|
2
7
|
const SAME_LIMIT_WINDOW_MS = 60_000;
|
|
3
8
|
const MINUTE_MS = 60_000;
|
|
4
9
|
|
|
10
|
+
const isNonEmptyString = (value) => typeof value === 'string' && value !== '';
|
|
11
|
+
|
|
12
|
+
const isValidTime = (value) =>
|
|
13
|
+
value !== null &&
|
|
14
|
+
value !== undefined &&
|
|
15
|
+
!Number.isNaN(new Date(value).getTime());
|
|
16
|
+
|
|
5
17
|
const isLink = (link) =>
|
|
6
18
|
typeof link?.label === 'string' && typeof link?.url === 'string';
|
|
7
19
|
|
|
@@ -10,16 +22,12 @@ const linkActions = (links) =>
|
|
|
10
22
|
.filter(isLink)
|
|
11
23
|
.map(({ label, url }) => ({ type: 'LINK', label, url }));
|
|
12
24
|
|
|
13
|
-
function
|
|
25
|
+
function formatResetTime(date) {
|
|
14
26
|
const minute = new Date(Math.ceil(date.getTime() / MINUTE_MS) * MINUTE_MS);
|
|
15
27
|
return `${minute.toISOString().slice(0, 16).replace('T', ' ')} UTC`;
|
|
16
28
|
}
|
|
17
29
|
|
|
18
30
|
/**
|
|
19
|
-
* Use case that records one user-facing warning when a module reports a rate
|
|
20
|
-
* limit that is too long to wait for. The text is fixed and the payload's
|
|
21
|
-
* message, url, body and headers are never read, because the message reaches
|
|
22
|
-
* end users. It changes no integration status: a rate limit is not an error.
|
|
23
31
|
* @class RecordRateLimitMessage
|
|
24
32
|
*/
|
|
25
33
|
class RecordRateLimitMessage {
|
|
@@ -32,10 +40,6 @@ class RecordRateLimitMessage {
|
|
|
32
40
|
}
|
|
33
41
|
|
|
34
42
|
/**
|
|
35
|
-
* Writes the warning, unless the stored warnings already hold one for the
|
|
36
|
-
* same module whose reset time is within a minute of this one. Many workers
|
|
37
|
-
* hit one limit together and each reports it, so the check reads the
|
|
38
|
-
* stored warnings and not the messages of this instance.
|
|
39
43
|
* @async
|
|
40
44
|
* @param {string} integrationId - ID of the integration to warn on.
|
|
41
45
|
* @param {Object} payload - The RATE_LIMITED payload of the Module.
|
|
@@ -43,12 +47,33 @@ class RecordRateLimitMessage {
|
|
|
43
47
|
* @param {string} payload.reason
|
|
44
48
|
* @param {Date|string|number} payload.retryAt
|
|
45
49
|
* @param {Array<{label: string, url: string}>} [payload.links]
|
|
46
|
-
* @returns {Promise<boolean>} True when it wrote a warning.
|
|
47
|
-
* @throws When the read or the write fails; the caller decides what a
|
|
48
|
-
* failed warning is worth.
|
|
49
50
|
*/
|
|
50
|
-
async execute(integrationId,
|
|
51
|
-
const
|
|
51
|
+
async execute(integrationId, payload) {
|
|
52
|
+
const { moduleName, reason, retryAt, links } = payload ?? {};
|
|
53
|
+
if (!isNonEmptyString(moduleName) || !isValidTime(retryAt)) return;
|
|
54
|
+
|
|
55
|
+
try {
|
|
56
|
+
await this._recordUnlessStored(integrationId, {
|
|
57
|
+
moduleName,
|
|
58
|
+
reason,
|
|
59
|
+
resetAt: new Date(retryAt),
|
|
60
|
+
links,
|
|
61
|
+
});
|
|
62
|
+
} catch (error) {
|
|
63
|
+
log.warn('Failed to record rate limit message', {
|
|
64
|
+
eventName:
|
|
65
|
+
'frigg.integrations.rate_limit_message_record_failed',
|
|
66
|
+
integrationId,
|
|
67
|
+
moduleName,
|
|
68
|
+
error,
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async _recordUnlessStored(
|
|
74
|
+
integrationId,
|
|
75
|
+
{ moduleName, reason, resetAt, links }
|
|
76
|
+
) {
|
|
52
77
|
const warnings =
|
|
53
78
|
await this.integrationRepository.findIntegrationMessages(
|
|
54
79
|
integrationId,
|
|
@@ -56,14 +81,14 @@ class RecordRateLimitMessage {
|
|
|
56
81
|
);
|
|
57
82
|
const alreadyRecorded = warnings.some(
|
|
58
83
|
(warning) =>
|
|
59
|
-
warning?.code ===
|
|
84
|
+
warning?.code === RATE_LIMITED_MESSAGE_CODE &&
|
|
60
85
|
warning.module === moduleName &&
|
|
61
86
|
Math.abs(Date.parse(warning.retryAt) - resetAt.getTime()) <=
|
|
62
87
|
SAME_LIMIT_WINDOW_MS
|
|
63
88
|
);
|
|
64
|
-
if (alreadyRecorded) return
|
|
89
|
+
if (alreadyRecorded) return;
|
|
65
90
|
|
|
66
|
-
const resetsAt =
|
|
91
|
+
const resetsAt = formatResetTime(resetAt);
|
|
67
92
|
await this.integrationRepository.updateIntegrationMessages(
|
|
68
93
|
integrationId,
|
|
69
94
|
'warnings',
|
|
@@ -71,14 +96,14 @@ class RecordRateLimitMessage {
|
|
|
71
96
|
title: 'Rate limit reached',
|
|
72
97
|
message: `The ${moduleName} API rate limit was reached and resets at ${resetsAt}.`,
|
|
73
98
|
timestamp: Date.now(),
|
|
74
|
-
code:
|
|
99
|
+
code: RATE_LIMITED_MESSAGE_CODE,
|
|
75
100
|
module: moduleName,
|
|
76
101
|
reason,
|
|
77
102
|
retryAt: resetAt.toISOString(),
|
|
78
103
|
actions: [{ type: 'RETRY_WHEN_READY' }, ...linkActions(links)],
|
|
79
|
-
}
|
|
104
|
+
},
|
|
105
|
+
{ keepLast: MAX_MESSAGES_PER_TYPE }
|
|
80
106
|
);
|
|
81
|
-
return true;
|
|
82
107
|
}
|
|
83
108
|
}
|
|
84
109
|
|
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
const MAX_MESSAGES_PER_TYPE = 50;
|
|
2
|
+
|
|
3
|
+
const isPlainObject = (value) =>
|
|
4
|
+
value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
5
|
+
|
|
6
|
+
function toMessageItem(titleOrItem, messageBody, messageTimestamp) {
|
|
7
|
+
if (isPlainObject(titleOrItem)) return { ...titleOrItem };
|
|
8
|
+
return {
|
|
9
|
+
title: titleOrItem,
|
|
10
|
+
message: messageBody,
|
|
11
|
+
timestamp: messageTimestamp,
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
|
|
1
15
|
/**
|
|
2
16
|
* Use case for updating messages associated with an integration.
|
|
3
17
|
* @class UpdateIntegrationMessages
|
|
@@ -35,12 +49,15 @@ class UpdateIntegrationMessages {
|
|
|
35
49
|
await this.integrationRepository.updateIntegrationMessages(
|
|
36
50
|
integrationId,
|
|
37
51
|
messageType,
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
52
|
+
toMessageItem(
|
|
53
|
+
messageTitleOrItem,
|
|
54
|
+
messageBody,
|
|
55
|
+
messageTimestamp
|
|
56
|
+
),
|
|
57
|
+
{ keepLast: MAX_MESSAGES_PER_TYPE }
|
|
41
58
|
);
|
|
42
59
|
return integration;
|
|
43
60
|
}
|
|
44
61
|
}
|
|
45
62
|
|
|
46
|
-
module.exports = { UpdateIntegrationMessages };
|
|
63
|
+
module.exports = { MAX_MESSAGES_PER_TYPE, UpdateIntegrationMessages };
|
package/logs/serialize.js
CHANGED
|
@@ -235,6 +235,20 @@ function errorCode(err) {
|
|
|
235
235
|
return undefined;
|
|
236
236
|
}
|
|
237
237
|
|
|
238
|
+
function rateLimitFields(err) {
|
|
239
|
+
const retryAt = attempt(
|
|
240
|
+
() => (err.isRateLimited ? err.retryAt : undefined),
|
|
241
|
+
undefined
|
|
242
|
+
);
|
|
243
|
+
if (!(retryAt instanceof Date) || Number.isNaN(retryAt.getTime())) {
|
|
244
|
+
return {};
|
|
245
|
+
}
|
|
246
|
+
const reason = attempt(() => err.reason, undefined);
|
|
247
|
+
return typeof reason === 'string'
|
|
248
|
+
? { retryAt: retryAt.toISOString(), reason: cleanString(reason) }
|
|
249
|
+
: { retryAt: retryAt.toISOString() };
|
|
250
|
+
}
|
|
251
|
+
|
|
238
252
|
function errorStack(err, type, message) {
|
|
239
253
|
const stack = attempt(() => err.stack, undefined);
|
|
240
254
|
if (typeof stack !== 'string') return undefined;
|
|
@@ -279,6 +293,7 @@ function serializeErrorAt(err, depth, seen) {
|
|
|
279
293
|
if (code !== undefined) out.code = code;
|
|
280
294
|
const status = errorStatus(err);
|
|
281
295
|
if (status !== undefined) out.status = status;
|
|
296
|
+
Object.assign(out, rateLimitFields(err));
|
|
282
297
|
const stack = errorStack(err, type, message);
|
|
283
298
|
if (stack !== undefined) out.stack = stack;
|
|
284
299
|
|
package/modules/module.js
CHANGED
|
@@ -8,6 +8,7 @@ const {
|
|
|
8
8
|
const {
|
|
9
9
|
createModuleRepository,
|
|
10
10
|
} = require('./repositories/module-repository-factory');
|
|
11
|
+
const { readRateLimitPolicy } = require('./requester/rate-limit');
|
|
11
12
|
|
|
12
13
|
// todo: this class should be a Domain class, and the Delegate function is preventing us from
|
|
13
14
|
// doing that, we probably have to get rid of the Delegate class as well as the event based
|
|
@@ -238,15 +239,11 @@ class Module extends Delegate {
|
|
|
238
239
|
}
|
|
239
240
|
}
|
|
240
241
|
|
|
241
|
-
/**
|
|
242
|
-
* Passes a rate limit the api reported to the parent delegate, best
|
|
243
|
-
* effort. The payload never holds the error's message, url, body or
|
|
244
|
-
* headers, because the parent shows it to end users.
|
|
245
|
-
* @param {import('../errors').RateLimitError} rateLimitError
|
|
246
|
-
*/
|
|
247
242
|
async reportRateLimit(rateLimitError) {
|
|
248
243
|
const { reason, retryAt, policy, statusCode } = rateLimitError;
|
|
249
|
-
const links =
|
|
244
|
+
const links =
|
|
245
|
+
readRateLimitPolicy(this.apiClass)?.userHints?.[reason]?.links ??
|
|
246
|
+
[];
|
|
250
247
|
try {
|
|
251
248
|
await this.notify(this.DLGT_RATE_LIMITED, {
|
|
252
249
|
moduleName: this.name,
|
|
@@ -257,8 +254,8 @@ class Module extends Delegate {
|
|
|
257
254
|
links,
|
|
258
255
|
});
|
|
259
256
|
} catch (err) {
|
|
260
|
-
this.logger.
|
|
261
|
-
eventName: `${this.logger.name}.
|
|
257
|
+
this.logger.warn('Failed to propagate RATE_LIMITED', {
|
|
258
|
+
eventName: `${this.logger.name}.rate_limit_propagation_failed`,
|
|
262
259
|
error: err,
|
|
263
260
|
});
|
|
264
261
|
}
|
|
@@ -34,11 +34,6 @@ function readKeys(headers, wanted) {
|
|
|
34
34
|
return undefined;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
/**
|
|
38
|
-
* Reads one header from a Headers-like object, a Map, an entries array or a
|
|
39
|
-
* plain object. The name match is case-insensitive.
|
|
40
|
-
* @returns {string|undefined}
|
|
41
|
-
*/
|
|
42
37
|
function headerValue(headers, name) {
|
|
43
38
|
if (!headers || typeof headers !== 'object') return undefined;
|
|
44
39
|
const wanted = name.toLowerCase();
|
|
@@ -70,10 +65,6 @@ function definedOnly(fields) {
|
|
|
70
65
|
return result;
|
|
71
66
|
}
|
|
72
67
|
|
|
73
|
-
/**
|
|
74
|
-
* Builds a hint from a wait in ms. Returns null for a wait that is not a
|
|
75
|
-
* finite number in [0, MAX_HINT_WAIT_MS].
|
|
76
|
-
*/
|
|
77
68
|
function hintFromWait(waitMs, now, { source = 'header', ...extra } = {}) {
|
|
78
69
|
if (!Number.isFinite(waitMs) || waitMs < 0 || waitMs > MAX_HINT_WAIT_MS) {
|
|
79
70
|
return null;
|
|
@@ -86,9 +77,6 @@ function hintFromWait(waitMs, now, { source = 'header', ...extra } = {}) {
|
|
|
86
77
|
};
|
|
87
78
|
}
|
|
88
79
|
|
|
89
|
-
/**
|
|
90
|
-
* Builds a hint from an absolute time. A time in the past waits 0 ms.
|
|
91
|
-
*/
|
|
92
80
|
function hintFromRetryAt(retryAt, now, { source = 'header', ...extra } = {}) {
|
|
93
81
|
const at = retryAt instanceof Date ? retryAt.getTime() : Number.NaN;
|
|
94
82
|
if (Number.isNaN(at)) return null;
|
|
@@ -103,9 +91,6 @@ function parseDate(text) {
|
|
|
103
91
|
return Number.isNaN(at) ? null : new Date(at);
|
|
104
92
|
}
|
|
105
93
|
|
|
106
|
-
/**
|
|
107
|
-
* Retry-After: delta seconds, an HTTP-date or an ISO timestamp.
|
|
108
|
-
*/
|
|
109
94
|
function parseRetryAfter(value, { now = Date.now() } = {}) {
|
|
110
95
|
if (isMissing(value)) return null;
|
|
111
96
|
const text = String(value).trim();
|
|
@@ -145,10 +130,6 @@ function parseResetValue(text, now, extra) {
|
|
|
145
130
|
return date ? hintFromRetryAt(date, now, extra) : null;
|
|
146
131
|
}
|
|
147
132
|
|
|
148
|
-
/**
|
|
149
|
-
* X-RateLimit-Reset and the other reset headers. The magnitude tells the
|
|
150
|
-
* unit: epoch milliseconds, epoch seconds, or delta seconds.
|
|
151
|
-
*/
|
|
152
133
|
function parseResetHeaders(headers, { now = Date.now() } = {}) {
|
|
153
134
|
const remaining = numberOrUndefined(
|
|
154
135
|
firstHeader(headers, REMAINING_HEADERS)
|
|
@@ -222,11 +203,6 @@ function readKeyValue(text, key) {
|
|
|
222
203
|
return match ? match[1] : undefined;
|
|
223
204
|
}
|
|
224
205
|
|
|
225
|
-
/**
|
|
226
|
-
* The IETF RateLimit field: the key=value form (limit=, remaining=, reset=)
|
|
227
|
-
* and the structured-field form ("name";r=0;t=12). The list form picks the
|
|
228
|
-
* policy with the fewest remaining calls, then the longest wait.
|
|
229
|
-
*/
|
|
230
206
|
function parseIetfRateLimit(headers, { now = Date.now() } = {}) {
|
|
231
207
|
const raw = headerValue(headers, 'ratelimit');
|
|
232
208
|
if (raw === undefined) return null;
|
|
@@ -259,6 +235,7 @@ const BUILT_IN_PARSERS = {
|
|
|
259
235
|
module.exports = {
|
|
260
236
|
BUILT_IN_PARSERS,
|
|
261
237
|
MAX_HINT_WAIT_MS,
|
|
238
|
+
definedOnly,
|
|
262
239
|
headerValue,
|
|
263
240
|
hintFromRetryAt,
|
|
264
241
|
hintFromWait,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const {
|
|
2
2
|
BUILT_IN_PARSERS,
|
|
3
|
+
definedOnly,
|
|
3
4
|
hintFromRetryAt,
|
|
4
5
|
hintFromWait,
|
|
5
6
|
} = require('./parsers');
|
|
@@ -24,6 +25,7 @@ const normalizedPolicies = new WeakSet();
|
|
|
24
25
|
const policiesByClass = new WeakMap();
|
|
25
26
|
|
|
26
27
|
const isObject = (value) => value !== null && typeof value === 'object';
|
|
28
|
+
const isThenable = (value) => typeof value?.then === 'function';
|
|
27
29
|
|
|
28
30
|
function isPlainObject(value) {
|
|
29
31
|
if (!isObject(value) || Array.isArray(value)) return false;
|
|
@@ -31,14 +33,6 @@ function isPlainObject(value) {
|
|
|
31
33
|
return prototype === Object.prototype || prototype === null;
|
|
32
34
|
}
|
|
33
35
|
|
|
34
|
-
function definedOnly(fields) {
|
|
35
|
-
const result = {};
|
|
36
|
-
for (const [key, value] of Object.entries(fields)) {
|
|
37
|
-
if (value !== undefined) result[key] = value;
|
|
38
|
-
}
|
|
39
|
-
return result;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
36
|
function readParsers(raw, owner) {
|
|
43
37
|
if (!Array.isArray(raw.parsers)) return [...DEFAULT_PARSERS];
|
|
44
38
|
for (const name of raw.parsers) {
|
|
@@ -79,11 +73,6 @@ function normalizePolicy(raw, owner) {
|
|
|
79
73
|
return policy;
|
|
80
74
|
}
|
|
81
75
|
|
|
82
|
-
/**
|
|
83
|
-
* Reads the static `rateLimit` policy of an API module class. Returns
|
|
84
|
-
* undefined when the class declares none. The result is memoized per class.
|
|
85
|
-
* Throws a TypeError for a parser name that is not built in.
|
|
86
|
-
*/
|
|
87
76
|
function readRateLimitPolicy(ctor) {
|
|
88
77
|
if (typeof ctor !== 'function') return undefined;
|
|
89
78
|
if (policiesByClass.has(ctor)) return policiesByClass.get(ctor);
|
|
@@ -98,10 +87,6 @@ function asPolicy(value) {
|
|
|
98
87
|
return normalizePolicy(value, 'policy');
|
|
99
88
|
}
|
|
100
89
|
|
|
101
|
-
/**
|
|
102
|
-
* The key a limit counts against, for one requester. Undefined when the
|
|
103
|
-
* scope needs an id that the requester does not have.
|
|
104
|
-
*/
|
|
105
90
|
function computeScopeKey(policy, requester) {
|
|
106
91
|
const label = requester?._telemetryModuleLabel?.() ?? 'unknown';
|
|
107
92
|
const scope = policy?.scope ?? 'entity';
|
|
@@ -136,8 +121,18 @@ function computeScopeKey(policy, requester) {
|
|
|
136
121
|
return undefined;
|
|
137
122
|
}
|
|
138
123
|
|
|
124
|
+
function toDate(value) {
|
|
125
|
+
if (value === undefined || value === null) return null;
|
|
126
|
+
const date = value instanceof Date ? value : new Date(value);
|
|
127
|
+
return Number.isNaN(date.getTime()) ? null : date;
|
|
128
|
+
}
|
|
129
|
+
|
|
139
130
|
function normalizeClassified(raw, now) {
|
|
140
131
|
if (!isObject(raw)) return null;
|
|
132
|
+
const retryAt = toDate(raw.retryAt);
|
|
133
|
+
if (!REASONS.has(raw.reason) && !Number.isFinite(raw.waitMs) && !retryAt) {
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
141
136
|
|
|
142
137
|
const extra = {
|
|
143
138
|
reason: REASONS.has(raw.reason) ? raw.reason : 'unknown',
|
|
@@ -146,12 +141,7 @@ function normalizeClassified(raw, now) {
|
|
|
146
141
|
source: CLASSIFY_SOURCES.has(raw.source) ? raw.source : 'body',
|
|
147
142
|
};
|
|
148
143
|
|
|
149
|
-
let hint = null;
|
|
150
|
-
if (raw.retryAt !== undefined && raw.retryAt !== null) {
|
|
151
|
-
const date =
|
|
152
|
-
raw.retryAt instanceof Date ? raw.retryAt : new Date(raw.retryAt);
|
|
153
|
-
hint = hintFromRetryAt(date, now, extra);
|
|
154
|
-
}
|
|
144
|
+
let hint = retryAt ? hintFromRetryAt(retryAt, now, extra) : null;
|
|
155
145
|
if (!hint && Number.isFinite(raw.waitMs)) {
|
|
156
146
|
hint = hintFromWait(raw.waitMs, now, extra);
|
|
157
147
|
}
|
|
@@ -172,6 +162,12 @@ function runClassify(policy, signal, now, onClassifyError) {
|
|
|
172
162
|
headers: signal.headers,
|
|
173
163
|
body: signal.body,
|
|
174
164
|
});
|
|
165
|
+
if (isThenable(raw)) {
|
|
166
|
+
Promise.resolve(raw).catch(() => {});
|
|
167
|
+
throw new TypeError(
|
|
168
|
+
'classify() must return a hint or null, not a Promise'
|
|
169
|
+
);
|
|
170
|
+
}
|
|
175
171
|
} catch (error) {
|
|
176
172
|
if (!onClassifyError) throw error;
|
|
177
173
|
onClassifyError(error);
|
|
@@ -226,28 +222,12 @@ function resolveCandidates(policy, signal, { now, onClassifyError }) {
|
|
|
226
222
|
return { hint: null, classified };
|
|
227
223
|
}
|
|
228
224
|
|
|
229
|
-
/**
|
|
230
|
-
* Finds the hint of a throttled response: the module's classify(), then the
|
|
231
|
-
* header parsers, then the static policy. Returns null when there is none.
|
|
232
|
-
* A status other than 429 counts only when classify recognised it.
|
|
233
|
-
*
|
|
234
|
-
* @param {object} [policy] The static `rateLimit` object of an API module.
|
|
235
|
-
* @param {{status?: number, headers?: object, body?: unknown}} signal
|
|
236
|
-
* @param {{now?: number, onClassifyError?: (error: Error) => void}} [options]
|
|
237
|
-
* Without `onClassifyError` an error thrown by classify propagates.
|
|
238
|
-
*/
|
|
239
225
|
function classifyRateLimit(policy, signal, options = {}) {
|
|
240
226
|
const { now = Date.now(), onClassifyError } = options;
|
|
241
227
|
return resolveCandidates(asPolicy(policy), signal, { now, onClassifyError })
|
|
242
228
|
.hint;
|
|
243
229
|
}
|
|
244
230
|
|
|
245
|
-
/**
|
|
246
|
-
* Like classifyRateLimit, but a throttled response always gets a hint: with
|
|
247
|
-
* none found it gets the step of the fixed backoff ladder, with source
|
|
248
|
-
* "backoff". Null when the response is not throttled: a status other than 429
|
|
249
|
-
* that classify did not recognise.
|
|
250
|
-
*/
|
|
251
231
|
function resolveRateLimitHint(policy, signal, options = {}) {
|
|
252
232
|
const {
|
|
253
233
|
attempt = 0,
|
|
@@ -276,11 +256,6 @@ function resolveRateLimitHint(policy, signal, options = {}) {
|
|
|
276
256
|
};
|
|
277
257
|
}
|
|
278
258
|
|
|
279
|
-
/**
|
|
280
|
-
* The wait before the next attempt, for a hint that came from the provider
|
|
281
|
-
* or the policy. Jitter only adds to the wait and never passes the budget.
|
|
282
|
-
* `fits` is false when the wait alone is over the budget.
|
|
283
|
-
*/
|
|
284
259
|
function computeWaitMs({ hint, policy, budgetMs, random = Math.random }) {
|
|
285
260
|
const base = Math.max(
|
|
286
261
|
hint.waitMs,
|
|
@@ -295,10 +270,6 @@ function computeWaitMs({ hint, policy, budgetMs, random = Math.random }) {
|
|
|
295
270
|
return { waitMs: Math.min(base + jitter, budgetMs), fits: true };
|
|
296
271
|
}
|
|
297
272
|
|
|
298
|
-
/**
|
|
299
|
-
* How long this request may still sleep in process: the cap, less what it
|
|
300
|
-
* slept already, and the time left in the invocation, less one request.
|
|
301
|
-
*/
|
|
302
273
|
function inProcessBudgetMs({
|
|
303
274
|
policy,
|
|
304
275
|
requestTimeoutMs = 0,
|
|
@@ -306,10 +277,8 @@ function inProcessBudgetMs({
|
|
|
306
277
|
waitedMs = 0,
|
|
307
278
|
} = {}) {
|
|
308
279
|
const cap = policy?.maxInProcessWaitMs ?? DEFAULT_MAX_IN_PROCESS_WAIT_MS;
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
Math.min(cap - waitedMs, remainingMs - (requestTimeoutMs || 0))
|
|
312
|
-
);
|
|
280
|
+
const reserveMs = Math.min(requestTimeoutMs || 0, remainingMs / 2);
|
|
281
|
+
return Math.max(0, Math.min(cap - waitedMs, remainingMs - reserveMs));
|
|
313
282
|
}
|
|
314
283
|
|
|
315
284
|
module.exports = {
|
|
@@ -10,9 +10,9 @@ const { redactUrl } = require('../../logs/redact');
|
|
|
10
10
|
const { toSanitizedSurrogate } = require('../../logs/serialize');
|
|
11
11
|
const { getLoggerScope } = require('../../logs/context');
|
|
12
12
|
const {
|
|
13
|
+
DEFAULT_MAX_IN_PROCESS_WAIT_MS,
|
|
13
14
|
computeScopeKey,
|
|
14
15
|
computeWaitMs,
|
|
15
|
-
headerValue,
|
|
16
16
|
inProcessBudgetMs,
|
|
17
17
|
readRateLimitPolicy,
|
|
18
18
|
resolveRateLimitHint,
|
|
@@ -20,18 +20,6 @@ const {
|
|
|
20
20
|
|
|
21
21
|
const DEFAULT_REQUEST_TIMEOUT_MS = 60_000;
|
|
22
22
|
|
|
23
|
-
function isJsonMediaType(contentType) {
|
|
24
|
-
const mediaType = String(contentType ?? '')
|
|
25
|
-
.split(';')[0]
|
|
26
|
-
.trim()
|
|
27
|
-
.toLowerCase();
|
|
28
|
-
return (
|
|
29
|
-
mediaType === 'application/json' ||
|
|
30
|
-
mediaType === 'text/json' ||
|
|
31
|
-
mediaType.endsWith('+json')
|
|
32
|
-
);
|
|
33
|
-
}
|
|
34
|
-
|
|
35
23
|
// A node-fetch error message holds the raw URL, and util.inspect prints the
|
|
36
24
|
// cause chain, so the FetchError keeps only a sanitized copy.
|
|
37
25
|
function sanitizedCause(err) {
|
|
@@ -237,9 +225,6 @@ class Requester extends Delegate {
|
|
|
237
225
|
* @param {number} attempt - 0-based count of retries already made for
|
|
238
226
|
* this call. Indexes `this.backOff` for the next delay and is passed
|
|
239
227
|
* back in on each recursive retry.
|
|
240
|
-
* @param {number} waitedMs - Milliseconds this call has slept already
|
|
241
|
-
* for a wait that a provider or a policy set. It counts against the
|
|
242
|
-
* in-process cap.
|
|
243
228
|
*/
|
|
244
229
|
async _rawRequest(url, options, attempt = 0, waitedMs = 0) {
|
|
245
230
|
let encodedUrl = encodeURI(url);
|
|
@@ -336,7 +321,7 @@ class Requester extends Delegate {
|
|
|
336
321
|
status,
|
|
337
322
|
attempt
|
|
338
323
|
);
|
|
339
|
-
const
|
|
324
|
+
const hintedDelayMs = await this._hintedRetryDelayMs({
|
|
340
325
|
throttle,
|
|
341
326
|
status,
|
|
342
327
|
attempt,
|
|
@@ -346,21 +331,24 @@ class Requester extends Delegate {
|
|
|
346
331
|
response,
|
|
347
332
|
timeoutMs,
|
|
348
333
|
});
|
|
349
|
-
if (
|
|
334
|
+
if (hintedDelayMs !== null) {
|
|
350
335
|
clearRequestTimer();
|
|
351
336
|
await new Promise((resolve) =>
|
|
352
|
-
setTimeout(resolve,
|
|
337
|
+
setTimeout(resolve, hintedDelayMs)
|
|
353
338
|
);
|
|
354
339
|
return this._rawRequest(
|
|
355
340
|
url,
|
|
356
341
|
options,
|
|
357
342
|
attempt + 1,
|
|
358
|
-
waitedMs +
|
|
343
|
+
waitedMs + hintedDelayMs
|
|
359
344
|
);
|
|
360
345
|
}
|
|
361
346
|
|
|
362
347
|
// If the status is retriable and there are back off requests left, retry the request
|
|
363
|
-
if (
|
|
348
|
+
if (
|
|
349
|
+
(throttle?.throttled || status >= 500) &&
|
|
350
|
+
attempt < this.backOff.length
|
|
351
|
+
) {
|
|
364
352
|
clearRequestTimer();
|
|
365
353
|
const delay = this.backOff[attempt] * 1000;
|
|
366
354
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
@@ -499,16 +487,7 @@ class Requester extends Delegate {
|
|
|
499
487
|
}
|
|
500
488
|
}
|
|
501
489
|
|
|
502
|
-
|
|
503
|
-
* What to do about a throttled response: the delay before the next
|
|
504
|
-
* attempt, or null when there is no attempt left. A wait that a hint set
|
|
505
|
-
* and that does not fit the budget throws RateLimitError.
|
|
506
|
-
*
|
|
507
|
-
* @returns {Promise<{delayMs: number, hintedMs: number}|null>}
|
|
508
|
-
* `hintedMs` is the part of the delay that a provider or a policy set,
|
|
509
|
-
* which counts against the in-process cap.
|
|
510
|
-
*/
|
|
511
|
-
async _throttleRetry({
|
|
490
|
+
async _hintedRetryDelayMs({
|
|
512
491
|
throttle,
|
|
513
492
|
status,
|
|
514
493
|
attempt,
|
|
@@ -518,35 +497,28 @@ class Requester extends Delegate {
|
|
|
518
497
|
response,
|
|
519
498
|
timeoutMs,
|
|
520
499
|
}) {
|
|
521
|
-
if (!throttle?.throttled)
|
|
522
|
-
|
|
523
|
-
if (hint.source === 'backoff') {
|
|
524
|
-
return attempt < this.backOff.length
|
|
525
|
-
? { delayMs: this.backOff[attempt] * 1000, hintedMs: 0 }
|
|
526
|
-
: null;
|
|
500
|
+
if (!throttle?.throttled || throttle.hint.source === 'backoff') {
|
|
501
|
+
return null;
|
|
527
502
|
}
|
|
528
|
-
|
|
529
|
-
const budgetMs =
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
503
|
+
const { hint } = throttle;
|
|
504
|
+
const budgetMs = this.backOff.length
|
|
505
|
+
? inProcessBudgetMs({
|
|
506
|
+
policy: this._rateLimitPolicy,
|
|
507
|
+
requestTimeoutMs: this.requestTimeoutMs,
|
|
508
|
+
remainingMs: remainingInvocationMs(),
|
|
509
|
+
waitedMs,
|
|
510
|
+
})
|
|
511
|
+
: 0;
|
|
535
512
|
const { waitMs, fits } = computeWaitMs({
|
|
536
513
|
hint,
|
|
537
514
|
policy: this._rateLimitPolicy,
|
|
538
515
|
budgetMs,
|
|
539
516
|
random: this._random,
|
|
540
517
|
});
|
|
541
|
-
|
|
542
|
-
status,
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
attempt,
|
|
546
|
-
waitedMs,
|
|
547
|
-
action: fits ? 'wait' : 'throw',
|
|
548
|
-
});
|
|
549
|
-
if (fits) return { delayMs: waitMs, hintedMs: waitMs };
|
|
518
|
+
if (fits) {
|
|
519
|
+
this._logRateLimited({ status, hint, waitMs, attempt, waitedMs });
|
|
520
|
+
return waitMs;
|
|
521
|
+
}
|
|
550
522
|
|
|
551
523
|
this._logRequestFailed(encodedUrl, options, status);
|
|
552
524
|
const rateLimitError = await RateLimitError.create({
|
|
@@ -563,14 +535,6 @@ class Requester extends Delegate {
|
|
|
563
535
|
throw this._maybeFlagTimeoutDuringBodyRead(rateLimitError, timeoutMs);
|
|
564
536
|
}
|
|
565
537
|
|
|
566
|
-
/**
|
|
567
|
-
* Decides whether a response says a limit was hit. A 429 always does. A
|
|
568
|
-
* 4xx or 5xx other than 401 does only when the module's classify() names
|
|
569
|
-
* a limit. The body is read here, once, only for classify(); the text is
|
|
570
|
-
* handed on so the error need not read the stream again.
|
|
571
|
-
*
|
|
572
|
-
* @returns {Promise<{throttled: boolean, hint?: object, responseBody?: string}|null>}
|
|
573
|
-
*/
|
|
574
538
|
async _detectThrottle(response, status, attempt) {
|
|
575
539
|
const policy = this._rateLimitPolicy;
|
|
576
540
|
const canClassify =
|
|
@@ -605,9 +569,6 @@ class Requester extends Delegate {
|
|
|
605
569
|
return {};
|
|
606
570
|
}
|
|
607
571
|
const text = await response.text();
|
|
608
|
-
if (!isJsonMediaType(headerValue(response.headers, 'content-type'))) {
|
|
609
|
-
return { text };
|
|
610
|
-
}
|
|
611
572
|
try {
|
|
612
573
|
return { text, json: JSON.parse(text) };
|
|
613
574
|
} catch {
|
|
@@ -615,9 +576,9 @@ class Requester extends Delegate {
|
|
|
615
576
|
}
|
|
616
577
|
}
|
|
617
578
|
|
|
618
|
-
_logRateLimited({ status, hint, waitMs, attempt, waitedMs
|
|
579
|
+
_logRateLimited({ status, hint, waitMs, attempt, waitedMs }) {
|
|
619
580
|
const logger = this.logger;
|
|
620
|
-
logger.
|
|
581
|
+
logger.trace('Rate limited', {
|
|
621
582
|
eventName: `${logger.name}.rate_limited`,
|
|
622
583
|
statusCode: status,
|
|
623
584
|
waitMs,
|
|
@@ -626,7 +587,6 @@ class Requester extends Delegate {
|
|
|
626
587
|
hintSource: hint.source,
|
|
627
588
|
attempt,
|
|
628
589
|
waitedMs,
|
|
629
|
-
action,
|
|
630
590
|
});
|
|
631
591
|
}
|
|
632
592
|
|
|
@@ -639,12 +599,8 @@ class Requester extends Delegate {
|
|
|
639
599
|
});
|
|
640
600
|
}
|
|
641
601
|
|
|
642
|
-
/**
|
|
643
|
-
* Tells the delegate a wait was too long to sleep. Best effort: a failed
|
|
644
|
-
* notification is logged and never replaces the RateLimitError the caller
|
|
645
|
-
* is about to receive.
|
|
646
|
-
*/
|
|
647
602
|
async _notifyRateLimited(rateLimitError) {
|
|
603
|
+
if (!this._waitsLongerThanInProcessCap(rateLimitError)) return;
|
|
648
604
|
try {
|
|
649
605
|
await this.notify(this.DLGT_RATE_LIMITED, rateLimitError);
|
|
650
606
|
} catch (error) {
|
|
@@ -657,6 +613,16 @@ class Requester extends Delegate {
|
|
|
657
613
|
}
|
|
658
614
|
}
|
|
659
615
|
|
|
616
|
+
_waitsLongerThanInProcessCap(rateLimitError) {
|
|
617
|
+
const waitMs = Number.isFinite(rateLimitError?.waitMs)
|
|
618
|
+
? rateLimitError.waitMs
|
|
619
|
+
: new Date(rateLimitError?.retryAt).getTime() - Date.now();
|
|
620
|
+
const capMs =
|
|
621
|
+
this._rateLimitPolicy?.maxInProcessWaitMs ??
|
|
622
|
+
DEFAULT_MAX_IN_PROCESS_WAIT_MS;
|
|
623
|
+
return waitMs > capMs;
|
|
624
|
+
}
|
|
625
|
+
|
|
660
626
|
_logRequestFailed(encodedUrl, options, status) {
|
|
661
627
|
const logger = this.logger;
|
|
662
628
|
if (!logger.isLevelEnabled('DEBUG')) return;
|