@http-client-toolkit/store-dynamodb 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +270 -0
- package/lib/index.cjs +1807 -0
- package/lib/index.cjs.map +1 -0
- package/lib/index.d.cts +165 -0
- package/lib/index.d.ts +165 -0
- package/lib/index.js +1800 -0
- package/lib/index.js.map +1 -0
- package/package.json +76 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,1800 @@
|
|
|
1
|
+
import { DynamoDBClient, ResourceNotFoundException } from '@aws-sdk/client-dynamodb';
|
|
2
|
+
import { DynamoDBDocumentClient, GetCommand, PutCommand, DeleteCommand, ScanCommand, UpdateCommand, TransactWriteCommand, QueryCommand, BatchWriteCommand } from '@aws-sdk/lib-dynamodb';
|
|
3
|
+
import { randomUUID } from 'crypto';
|
|
4
|
+
import { DEFAULT_RATE_LIMIT, AdaptiveCapacityCalculator } from '@http-client-toolkit/core';
|
|
5
|
+
|
|
6
|
+
var __defProp = Object.defineProperty;
|
|
7
|
+
var __defProps = Object.defineProperties;
|
|
8
|
+
var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
|
|
9
|
+
var __getOwnPropSymbols = Object.getOwnPropertySymbols;
|
|
10
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
11
|
+
var __propIsEnum = Object.prototype.propertyIsEnumerable;
|
|
12
|
+
var __pow = Math.pow;
|
|
13
|
+
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
14
|
+
var __spreadValues = (a, b) => {
|
|
15
|
+
for (var prop in b || (b = {}))
|
|
16
|
+
if (__hasOwnProp.call(b, prop))
|
|
17
|
+
__defNormalProp(a, prop, b[prop]);
|
|
18
|
+
if (__getOwnPropSymbols)
|
|
19
|
+
for (var prop of __getOwnPropSymbols(b)) {
|
|
20
|
+
if (__propIsEnum.call(b, prop))
|
|
21
|
+
__defNormalProp(a, prop, b[prop]);
|
|
22
|
+
}
|
|
23
|
+
return a;
|
|
24
|
+
};
|
|
25
|
+
var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
|
|
26
|
+
var __async = (__this, __arguments, generator) => {
|
|
27
|
+
return new Promise((resolve, reject) => {
|
|
28
|
+
var fulfilled = (value) => {
|
|
29
|
+
try {
|
|
30
|
+
step(generator.next(value));
|
|
31
|
+
} catch (e) {
|
|
32
|
+
reject(e);
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
var rejected = (value) => {
|
|
36
|
+
try {
|
|
37
|
+
step(generator.throw(value));
|
|
38
|
+
} catch (e) {
|
|
39
|
+
reject(e);
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
|
|
43
|
+
step((generator = generator.apply(__this, __arguments)).next());
|
|
44
|
+
});
|
|
45
|
+
};
|
|
46
|
+
var MAX_BATCH_WRITE_RETRIES = 8;
|
|
47
|
+
var MAX_DYNAMO_KEY_PART_BYTES = 512;
|
|
48
|
+
function sleep(ms) {
|
|
49
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
50
|
+
}
|
|
51
|
+
function getRetryDelayMs(attempt) {
|
|
52
|
+
const backoff = Math.min(1e3, 50 * __pow(2, attempt));
|
|
53
|
+
const jitter = Math.floor(Math.random() * 25);
|
|
54
|
+
return backoff + jitter;
|
|
55
|
+
}
|
|
56
|
+
function batchDeleteWithRetries(docClient, tableName, keys) {
|
|
57
|
+
return __async(this, null, function* () {
|
|
58
|
+
var _a, _b;
|
|
59
|
+
for (let i = 0; i < keys.length; i += 25) {
|
|
60
|
+
const batch = keys.slice(i, i + 25);
|
|
61
|
+
let pendingWrites = batch.map((key) => ({ DeleteRequest: { Key: key } }));
|
|
62
|
+
for (let attempt = 0; pendingWrites.length > 0; attempt++) {
|
|
63
|
+
const response = yield docClient.send(
|
|
64
|
+
new BatchWriteCommand({
|
|
65
|
+
RequestItems: {
|
|
66
|
+
[tableName]: pendingWrites
|
|
67
|
+
}
|
|
68
|
+
})
|
|
69
|
+
);
|
|
70
|
+
const unprocessed = (_b = (_a = response.UnprocessedItems) == null ? void 0 : _a[tableName]) != null ? _b : [];
|
|
71
|
+
if (unprocessed.length === 0) {
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
if (attempt >= MAX_BATCH_WRITE_RETRIES) {
|
|
75
|
+
throw new Error(
|
|
76
|
+
`Failed to delete all items from table "${tableName}" after ${MAX_BATCH_WRITE_RETRIES + 1} attempts`
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
pendingWrites = unprocessed.map((request) => {
|
|
80
|
+
var _a2;
|
|
81
|
+
return (_a2 = request.DeleteRequest) == null ? void 0 : _a2.Key;
|
|
82
|
+
}).filter((key) => Boolean(key)).map((key) => ({ DeleteRequest: { Key: key } }));
|
|
83
|
+
yield sleep(getRetryDelayMs(attempt));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
function queryCountAllPages(docClient, input) {
|
|
89
|
+
return __async(this, null, function* () {
|
|
90
|
+
var _a;
|
|
91
|
+
let total = 0;
|
|
92
|
+
let lastEvaluatedKey;
|
|
93
|
+
do {
|
|
94
|
+
const result = yield docClient.send(
|
|
95
|
+
new QueryCommand(__spreadProps(__spreadValues({}, input), {
|
|
96
|
+
ExclusiveStartKey: lastEvaluatedKey
|
|
97
|
+
}))
|
|
98
|
+
);
|
|
99
|
+
total += (_a = result.Count) != null ? _a : 0;
|
|
100
|
+
lastEvaluatedKey = result.LastEvaluatedKey;
|
|
101
|
+
} while (lastEvaluatedKey);
|
|
102
|
+
return total;
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
function queryItemsAllPages(docClient, input) {
|
|
106
|
+
return __async(this, null, function* () {
|
|
107
|
+
var _a;
|
|
108
|
+
const items = [];
|
|
109
|
+
let lastEvaluatedKey;
|
|
110
|
+
do {
|
|
111
|
+
const result = yield docClient.send(
|
|
112
|
+
new QueryCommand(__spreadProps(__spreadValues({}, input), {
|
|
113
|
+
ExclusiveStartKey: lastEvaluatedKey
|
|
114
|
+
}))
|
|
115
|
+
);
|
|
116
|
+
if ((_a = result.Items) == null ? void 0 : _a.length) {
|
|
117
|
+
items.push(...result.Items);
|
|
118
|
+
}
|
|
119
|
+
lastEvaluatedKey = result.LastEvaluatedKey;
|
|
120
|
+
} while (lastEvaluatedKey);
|
|
121
|
+
return items;
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
function queryCountUpTo(docClient, input, maxCount) {
|
|
125
|
+
return __async(this, null, function* () {
|
|
126
|
+
var _a;
|
|
127
|
+
if (maxCount <= 0) {
|
|
128
|
+
return { count: 0, reachedLimit: true };
|
|
129
|
+
}
|
|
130
|
+
let total = 0;
|
|
131
|
+
let lastEvaluatedKey;
|
|
132
|
+
do {
|
|
133
|
+
const remaining = maxCount - total;
|
|
134
|
+
const result = yield docClient.send(
|
|
135
|
+
new QueryCommand(__spreadProps(__spreadValues({}, input), {
|
|
136
|
+
Select: "COUNT",
|
|
137
|
+
Limit: remaining,
|
|
138
|
+
ExclusiveStartKey: lastEvaluatedKey
|
|
139
|
+
}))
|
|
140
|
+
);
|
|
141
|
+
total += (_a = result.Count) != null ? _a : 0;
|
|
142
|
+
if (total >= maxCount) {
|
|
143
|
+
return { count: maxCount, reachedLimit: true };
|
|
144
|
+
}
|
|
145
|
+
lastEvaluatedKey = result.LastEvaluatedKey;
|
|
146
|
+
} while (lastEvaluatedKey);
|
|
147
|
+
return { count: total, reachedLimit: false };
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
function isConditionalTransactionFailure(error) {
|
|
151
|
+
var _a;
|
|
152
|
+
if (!error || typeof error !== "object") {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
const maybeError = error;
|
|
156
|
+
if (maybeError.name !== "TransactionCanceledException") {
|
|
157
|
+
return false;
|
|
158
|
+
}
|
|
159
|
+
const cancellationReasons = (_a = maybeError.CancellationReasons) != null ? _a : maybeError.cancellationReasons;
|
|
160
|
+
if (Array.isArray(cancellationReasons)) {
|
|
161
|
+
return cancellationReasons.some((reason) => {
|
|
162
|
+
if (!reason || typeof reason !== "object") {
|
|
163
|
+
return false;
|
|
164
|
+
}
|
|
165
|
+
return "Code" in reason && reason.Code === "ConditionalCheckFailed";
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
return typeof maybeError.message === "string" && maybeError.message.includes("ConditionalCheckFailed");
|
|
169
|
+
}
|
|
170
|
+
function assertDynamoKeyPart(value, label, maxBytes = MAX_DYNAMO_KEY_PART_BYTES) {
|
|
171
|
+
if (value.length === 0) {
|
|
172
|
+
throw new Error(`${label} must not be empty`);
|
|
173
|
+
}
|
|
174
|
+
for (let i = 0; i < value.length; i++) {
|
|
175
|
+
const charCode = value.charCodeAt(i);
|
|
176
|
+
if (charCode < 32 || charCode === 127) {
|
|
177
|
+
throw new Error(`${label} contains unsupported control characters`);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
if (Buffer.byteLength(value, "utf8") > maxBytes) {
|
|
181
|
+
throw new Error(`${label} exceeds maximum length of ${maxBytes} bytes`);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
function throwIfDynamoTableMissing(error, tableName) {
|
|
185
|
+
if (error instanceof ResourceNotFoundException || error && typeof error === "object" && "name" in error && error.name === "ResourceNotFoundException") {
|
|
186
|
+
throw new Error(
|
|
187
|
+
`DynamoDB table "${tableName}" was not found. Create the table using your infrastructure before using DynamoDB stores.`
|
|
188
|
+
);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// src/table.ts
|
|
193
|
+
var DEFAULT_TABLE_NAME = "http-client-toolkit";
|
|
194
|
+
var TABLE_SCHEMA = {
|
|
195
|
+
KeySchema: [
|
|
196
|
+
{ AttributeName: "pk", KeyType: "HASH" },
|
|
197
|
+
{ AttributeName: "sk", KeyType: "RANGE" }
|
|
198
|
+
],
|
|
199
|
+
AttributeDefinitions: [
|
|
200
|
+
{ AttributeName: "pk", AttributeType: "S" },
|
|
201
|
+
{ AttributeName: "sk", AttributeType: "S" },
|
|
202
|
+
{ AttributeName: "gsi1pk", AttributeType: "S" },
|
|
203
|
+
{ AttributeName: "gsi1sk", AttributeType: "S" }
|
|
204
|
+
],
|
|
205
|
+
GlobalSecondaryIndexes: [
|
|
206
|
+
{
|
|
207
|
+
IndexName: "gsi1",
|
|
208
|
+
KeySchema: [
|
|
209
|
+
{ AttributeName: "gsi1pk", KeyType: "HASH" },
|
|
210
|
+
{ AttributeName: "gsi1sk", KeyType: "RANGE" }
|
|
211
|
+
],
|
|
212
|
+
Projection: { ProjectionType: "ALL" }
|
|
213
|
+
}
|
|
214
|
+
]
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
// src/dynamodb-cache-store.ts
|
|
218
|
+
var DynamoDBCacheStore = class {
|
|
219
|
+
constructor({
|
|
220
|
+
client,
|
|
221
|
+
region,
|
|
222
|
+
tableName = DEFAULT_TABLE_NAME,
|
|
223
|
+
maxEntrySizeBytes = 390 * 1024
|
|
224
|
+
} = {}) {
|
|
225
|
+
this.isDestroyed = false;
|
|
226
|
+
this.tableName = tableName;
|
|
227
|
+
this.maxEntrySizeBytes = maxEntrySizeBytes;
|
|
228
|
+
if (client instanceof DynamoDBDocumentClient) {
|
|
229
|
+
this.docClient = client;
|
|
230
|
+
this.isClientManaged = false;
|
|
231
|
+
} else if (client instanceof DynamoDBClient) {
|
|
232
|
+
this.docClient = DynamoDBDocumentClient.from(client);
|
|
233
|
+
this.isClientManaged = false;
|
|
234
|
+
} else {
|
|
235
|
+
const config = {};
|
|
236
|
+
if (region) config.region = region;
|
|
237
|
+
this.rawClient = new DynamoDBClient(config);
|
|
238
|
+
this.docClient = DynamoDBDocumentClient.from(this.rawClient);
|
|
239
|
+
this.isClientManaged = true;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
get(hash) {
|
|
243
|
+
return __async(this, null, function* () {
|
|
244
|
+
if (this.isDestroyed) {
|
|
245
|
+
throw new Error("Cache store has been destroyed");
|
|
246
|
+
}
|
|
247
|
+
this.assertValidHash(hash);
|
|
248
|
+
const pk = `CACHE#${hash}`;
|
|
249
|
+
let result;
|
|
250
|
+
try {
|
|
251
|
+
result = yield this.docClient.send(
|
|
252
|
+
new GetCommand({
|
|
253
|
+
TableName: this.tableName,
|
|
254
|
+
Key: { pk, sk: pk }
|
|
255
|
+
})
|
|
256
|
+
);
|
|
257
|
+
} catch (error) {
|
|
258
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
259
|
+
throw error;
|
|
260
|
+
}
|
|
261
|
+
if (!result.Item) {
|
|
262
|
+
return void 0;
|
|
263
|
+
}
|
|
264
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
265
|
+
if (result.Item["ttl"] > 0 && now >= result.Item["ttl"]) {
|
|
266
|
+
yield this.delete(hash);
|
|
267
|
+
return void 0;
|
|
268
|
+
}
|
|
269
|
+
try {
|
|
270
|
+
const value = result.Item["value"];
|
|
271
|
+
if (value === "__UNDEFINED__") {
|
|
272
|
+
return void 0;
|
|
273
|
+
}
|
|
274
|
+
return JSON.parse(value);
|
|
275
|
+
} catch (e) {
|
|
276
|
+
yield this.delete(hash);
|
|
277
|
+
return void 0;
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
set(hash, value, ttlSeconds) {
|
|
282
|
+
return __async(this, null, function* () {
|
|
283
|
+
if (this.isDestroyed) {
|
|
284
|
+
throw new Error("Cache store has been destroyed");
|
|
285
|
+
}
|
|
286
|
+
this.assertValidHash(hash);
|
|
287
|
+
const now = Date.now();
|
|
288
|
+
const nowEpoch = Math.floor(now / 1e3);
|
|
289
|
+
let ttl;
|
|
290
|
+
if (ttlSeconds < 0) {
|
|
291
|
+
ttl = nowEpoch;
|
|
292
|
+
} else if (ttlSeconds === 0) {
|
|
293
|
+
ttl = 0;
|
|
294
|
+
} else {
|
|
295
|
+
ttl = nowEpoch + ttlSeconds;
|
|
296
|
+
}
|
|
297
|
+
let serializedValue;
|
|
298
|
+
try {
|
|
299
|
+
if (value === void 0) {
|
|
300
|
+
serializedValue = "__UNDEFINED__";
|
|
301
|
+
} else {
|
|
302
|
+
serializedValue = JSON.stringify(value);
|
|
303
|
+
}
|
|
304
|
+
} catch (error) {
|
|
305
|
+
throw new Error(
|
|
306
|
+
`Failed to serialize value: ${error instanceof Error ? error.message : String(error)}`
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
if (Buffer.byteLength(serializedValue, "utf8") > this.maxEntrySizeBytes) {
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
const pk = `CACHE#${hash}`;
|
|
313
|
+
try {
|
|
314
|
+
yield this.docClient.send(
|
|
315
|
+
new PutCommand({
|
|
316
|
+
TableName: this.tableName,
|
|
317
|
+
Item: {
|
|
318
|
+
pk,
|
|
319
|
+
sk: pk,
|
|
320
|
+
value: serializedValue,
|
|
321
|
+
ttl,
|
|
322
|
+
createdAt: now
|
|
323
|
+
}
|
|
324
|
+
})
|
|
325
|
+
);
|
|
326
|
+
} catch (error) {
|
|
327
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
328
|
+
throw error;
|
|
329
|
+
}
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
delete(hash) {
|
|
333
|
+
return __async(this, null, function* () {
|
|
334
|
+
if (this.isDestroyed) {
|
|
335
|
+
throw new Error("Cache store has been destroyed");
|
|
336
|
+
}
|
|
337
|
+
this.assertValidHash(hash);
|
|
338
|
+
const pk = `CACHE#${hash}`;
|
|
339
|
+
try {
|
|
340
|
+
yield this.docClient.send(
|
|
341
|
+
new DeleteCommand({
|
|
342
|
+
TableName: this.tableName,
|
|
343
|
+
Key: { pk, sk: pk }
|
|
344
|
+
})
|
|
345
|
+
);
|
|
346
|
+
} catch (error) {
|
|
347
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
348
|
+
throw error;
|
|
349
|
+
}
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
clear() {
|
|
353
|
+
return __async(this, null, function* () {
|
|
354
|
+
var _a;
|
|
355
|
+
if (this.isDestroyed) {
|
|
356
|
+
throw new Error("Cache store has been destroyed");
|
|
357
|
+
}
|
|
358
|
+
let lastEvaluatedKey;
|
|
359
|
+
do {
|
|
360
|
+
let scanResult;
|
|
361
|
+
try {
|
|
362
|
+
scanResult = yield this.docClient.send(
|
|
363
|
+
new ScanCommand({
|
|
364
|
+
TableName: this.tableName,
|
|
365
|
+
FilterExpression: "begins_with(pk, :prefix)",
|
|
366
|
+
ExpressionAttributeValues: { ":prefix": "CACHE#" },
|
|
367
|
+
ProjectionExpression: "pk, sk",
|
|
368
|
+
ExclusiveStartKey: lastEvaluatedKey
|
|
369
|
+
})
|
|
370
|
+
);
|
|
371
|
+
} catch (error) {
|
|
372
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
373
|
+
throw error;
|
|
374
|
+
}
|
|
375
|
+
const items = (_a = scanResult.Items) != null ? _a : [];
|
|
376
|
+
if (items.length > 0) {
|
|
377
|
+
try {
|
|
378
|
+
yield batchDeleteWithRetries(
|
|
379
|
+
this.docClient,
|
|
380
|
+
this.tableName,
|
|
381
|
+
items.map((item) => ({ pk: item["pk"], sk: item["sk"] }))
|
|
382
|
+
);
|
|
383
|
+
} catch (error) {
|
|
384
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
385
|
+
throw error;
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
lastEvaluatedKey = scanResult.LastEvaluatedKey;
|
|
389
|
+
} while (lastEvaluatedKey);
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
close() {
|
|
393
|
+
return __async(this, null, function* () {
|
|
394
|
+
this.isDestroyed = true;
|
|
395
|
+
if (this.isClientManaged && this.rawClient) {
|
|
396
|
+
this.rawClient.destroy();
|
|
397
|
+
}
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
destroy() {
|
|
401
|
+
this.close();
|
|
402
|
+
}
|
|
403
|
+
assertValidHash(hash) {
|
|
404
|
+
assertDynamoKeyPart(hash, "hash");
|
|
405
|
+
}
|
|
406
|
+
};
|
|
407
|
+
var DynamoDBDedupeStore = class {
|
|
408
|
+
constructor({
|
|
409
|
+
client,
|
|
410
|
+
region,
|
|
411
|
+
tableName = DEFAULT_TABLE_NAME,
|
|
412
|
+
jobTimeoutMs = 3e5,
|
|
413
|
+
pollIntervalMs = 500
|
|
414
|
+
} = {}) {
|
|
415
|
+
this.jobPromises = /* @__PURE__ */ new Map();
|
|
416
|
+
this.jobSettlers = /* @__PURE__ */ new Map();
|
|
417
|
+
this.isDestroyed = false;
|
|
418
|
+
this.tableName = tableName;
|
|
419
|
+
this.jobTimeoutMs = jobTimeoutMs;
|
|
420
|
+
this.pollIntervalMs = pollIntervalMs;
|
|
421
|
+
if (client instanceof DynamoDBDocumentClient) {
|
|
422
|
+
this.docClient = client;
|
|
423
|
+
this.isClientManaged = false;
|
|
424
|
+
} else if (client instanceof DynamoDBClient) {
|
|
425
|
+
this.docClient = DynamoDBDocumentClient.from(client);
|
|
426
|
+
this.isClientManaged = false;
|
|
427
|
+
} else {
|
|
428
|
+
const config = {};
|
|
429
|
+
if (region) config.region = region;
|
|
430
|
+
this.rawClient = new DynamoDBClient(config);
|
|
431
|
+
this.docClient = DynamoDBDocumentClient.from(this.rawClient);
|
|
432
|
+
this.isClientManaged = true;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
waitFor(hash) {
|
|
436
|
+
return __async(this, null, function* () {
|
|
437
|
+
if (this.isDestroyed) {
|
|
438
|
+
throw new Error("Dedupe store has been destroyed");
|
|
439
|
+
}
|
|
440
|
+
this.assertValidHash(hash);
|
|
441
|
+
const existingPromise = this.jobPromises.get(hash);
|
|
442
|
+
if (existingPromise) {
|
|
443
|
+
return existingPromise;
|
|
444
|
+
}
|
|
445
|
+
const pk = `DEDUPE#${hash}`;
|
|
446
|
+
let item;
|
|
447
|
+
try {
|
|
448
|
+
const result = yield this.docClient.send(
|
|
449
|
+
new GetCommand({
|
|
450
|
+
TableName: this.tableName,
|
|
451
|
+
Key: { pk, sk: pk }
|
|
452
|
+
})
|
|
453
|
+
);
|
|
454
|
+
item = result.Item;
|
|
455
|
+
} catch (error) {
|
|
456
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
457
|
+
throw error;
|
|
458
|
+
}
|
|
459
|
+
if (!item) {
|
|
460
|
+
return void 0;
|
|
461
|
+
}
|
|
462
|
+
if (item["status"] === "completed") {
|
|
463
|
+
return this.deserializeResult(item["result"]);
|
|
464
|
+
}
|
|
465
|
+
if (item["status"] === "failed") {
|
|
466
|
+
return void 0;
|
|
467
|
+
}
|
|
468
|
+
const promise = new Promise((resolve) => {
|
|
469
|
+
let settled = false;
|
|
470
|
+
let timeoutHandle;
|
|
471
|
+
const settle = (value) => {
|
|
472
|
+
if (settled) return;
|
|
473
|
+
settled = true;
|
|
474
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
475
|
+
clearInterval(pollHandle);
|
|
476
|
+
this.jobSettlers.delete(hash);
|
|
477
|
+
this.jobPromises.delete(hash);
|
|
478
|
+
resolve(value);
|
|
479
|
+
};
|
|
480
|
+
this.jobSettlers.set(hash, settle);
|
|
481
|
+
const poll = () => __async(this, null, function* () {
|
|
482
|
+
if (this.isDestroyed) {
|
|
483
|
+
settle(void 0);
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
try {
|
|
487
|
+
const latest = yield this.docClient.send(
|
|
488
|
+
new GetCommand({
|
|
489
|
+
TableName: this.tableName,
|
|
490
|
+
Key: { pk, sk: pk }
|
|
491
|
+
})
|
|
492
|
+
);
|
|
493
|
+
const latestItem = latest.Item;
|
|
494
|
+
if (!latestItem) {
|
|
495
|
+
settle(void 0);
|
|
496
|
+
return;
|
|
497
|
+
}
|
|
498
|
+
const isExpired = this.jobTimeoutMs > 0 && Date.now() - latestItem["createdAt"] >= this.jobTimeoutMs;
|
|
499
|
+
if (isExpired) {
|
|
500
|
+
try {
|
|
501
|
+
yield this.docClient.send(
|
|
502
|
+
new UpdateCommand({
|
|
503
|
+
TableName: this.tableName,
|
|
504
|
+
Key: { pk, sk: pk },
|
|
505
|
+
UpdateExpression: "SET #status = :failed, #error = :error, updatedAt = :now",
|
|
506
|
+
ExpressionAttributeNames: {
|
|
507
|
+
"#status": "status",
|
|
508
|
+
"#error": "error"
|
|
509
|
+
},
|
|
510
|
+
ExpressionAttributeValues: {
|
|
511
|
+
":failed": "failed",
|
|
512
|
+
":error": "Job timed out",
|
|
513
|
+
":now": Date.now()
|
|
514
|
+
}
|
|
515
|
+
})
|
|
516
|
+
);
|
|
517
|
+
} catch (e) {
|
|
518
|
+
}
|
|
519
|
+
settle(void 0);
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
if (latestItem["status"] === "completed") {
|
|
523
|
+
settle(this.deserializeResult(latestItem["result"]));
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
if (latestItem["status"] === "failed") {
|
|
527
|
+
settle(void 0);
|
|
528
|
+
}
|
|
529
|
+
} catch (e) {
|
|
530
|
+
settle(void 0);
|
|
531
|
+
}
|
|
532
|
+
});
|
|
533
|
+
let isPolling = false;
|
|
534
|
+
const pollHandle = setInterval(() => {
|
|
535
|
+
if (isPolling) {
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
isPolling = true;
|
|
539
|
+
void poll().finally(() => {
|
|
540
|
+
isPolling = false;
|
|
541
|
+
});
|
|
542
|
+
}, this.pollIntervalMs);
|
|
543
|
+
if (typeof pollHandle.unref === "function") {
|
|
544
|
+
pollHandle.unref();
|
|
545
|
+
}
|
|
546
|
+
void poll();
|
|
547
|
+
if (this.jobTimeoutMs > 0) {
|
|
548
|
+
timeoutHandle = setTimeout(() => {
|
|
549
|
+
if (this.isDestroyed) {
|
|
550
|
+
settle(void 0);
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
void (() => __async(this, null, function* () {
|
|
554
|
+
try {
|
|
555
|
+
yield this.docClient.send(
|
|
556
|
+
new UpdateCommand({
|
|
557
|
+
TableName: this.tableName,
|
|
558
|
+
Key: { pk, sk: pk },
|
|
559
|
+
UpdateExpression: "SET #status = :failed, #error = :error, updatedAt = :now",
|
|
560
|
+
ExpressionAttributeNames: {
|
|
561
|
+
"#status": "status",
|
|
562
|
+
"#error": "error"
|
|
563
|
+
},
|
|
564
|
+
ExpressionAttributeValues: {
|
|
565
|
+
":failed": "failed",
|
|
566
|
+
":error": "Job timed out",
|
|
567
|
+
":now": Date.now()
|
|
568
|
+
}
|
|
569
|
+
})
|
|
570
|
+
);
|
|
571
|
+
} catch (e) {
|
|
572
|
+
} finally {
|
|
573
|
+
settle(void 0);
|
|
574
|
+
}
|
|
575
|
+
}))();
|
|
576
|
+
}, this.jobTimeoutMs);
|
|
577
|
+
if (typeof timeoutHandle.unref === "function") {
|
|
578
|
+
timeoutHandle.unref();
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
});
|
|
582
|
+
this.jobPromises.set(hash, promise);
|
|
583
|
+
return promise;
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
register(hash) {
|
|
587
|
+
return __async(this, null, function* () {
|
|
588
|
+
const registration = yield this.registerOrJoin(hash);
|
|
589
|
+
return registration.jobId;
|
|
590
|
+
});
|
|
591
|
+
}
|
|
592
|
+
registerOrJoin(hash) {
|
|
593
|
+
return __async(this, null, function* () {
|
|
594
|
+
if (this.isDestroyed) {
|
|
595
|
+
throw new Error("Dedupe store has been destroyed");
|
|
596
|
+
}
|
|
597
|
+
this.assertValidHash(hash);
|
|
598
|
+
const pk = `DEDUPE#${hash}`;
|
|
599
|
+
const maxAttempts = 3;
|
|
600
|
+
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
601
|
+
const candidateJobId = randomUUID();
|
|
602
|
+
const now = Date.now();
|
|
603
|
+
const ttl = this.jobTimeoutMs > 0 ? Math.floor((now + this.jobTimeoutMs) / 1e3) : 0;
|
|
604
|
+
try {
|
|
605
|
+
yield this.docClient.send(
|
|
606
|
+
new PutCommand({
|
|
607
|
+
TableName: this.tableName,
|
|
608
|
+
Item: {
|
|
609
|
+
pk,
|
|
610
|
+
sk: pk,
|
|
611
|
+
jobId: candidateJobId,
|
|
612
|
+
status: "pending",
|
|
613
|
+
result: null,
|
|
614
|
+
error: null,
|
|
615
|
+
createdAt: now,
|
|
616
|
+
updatedAt: now,
|
|
617
|
+
ttl
|
|
618
|
+
},
|
|
619
|
+
ConditionExpression: "attribute_not_exists(pk) OR #status <> :pending",
|
|
620
|
+
ExpressionAttributeNames: { "#status": "status" },
|
|
621
|
+
ExpressionAttributeValues: { ":pending": "pending" }
|
|
622
|
+
})
|
|
623
|
+
);
|
|
624
|
+
return { jobId: candidateJobId, isOwner: true };
|
|
625
|
+
} catch (error) {
|
|
626
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
627
|
+
if (error && typeof error === "object" && "name" in error && error.name === "ConditionalCheckFailedException") {
|
|
628
|
+
const existing = yield this.docClient.send(
|
|
629
|
+
new GetCommand({
|
|
630
|
+
TableName: this.tableName,
|
|
631
|
+
Key: { pk, sk: pk }
|
|
632
|
+
})
|
|
633
|
+
);
|
|
634
|
+
if (existing.Item) {
|
|
635
|
+
return {
|
|
636
|
+
jobId: existing.Item["jobId"],
|
|
637
|
+
isOwner: false
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
continue;
|
|
641
|
+
}
|
|
642
|
+
throw error;
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
throw new Error(
|
|
646
|
+
`Failed to register or join job for hash "${hash}" after ${maxAttempts} attempts`
|
|
647
|
+
);
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
complete(hash, value) {
|
|
651
|
+
return __async(this, null, function* () {
|
|
652
|
+
if (this.isDestroyed) {
|
|
653
|
+
throw new Error("Dedupe store has been destroyed");
|
|
654
|
+
}
|
|
655
|
+
this.assertValidHash(hash);
|
|
656
|
+
let serializedResult;
|
|
657
|
+
if (value === void 0) {
|
|
658
|
+
serializedResult = "__UNDEFINED__";
|
|
659
|
+
} else if (value === null) {
|
|
660
|
+
serializedResult = "__NULL__";
|
|
661
|
+
} else {
|
|
662
|
+
try {
|
|
663
|
+
serializedResult = JSON.stringify(value);
|
|
664
|
+
} catch (error) {
|
|
665
|
+
throw new Error(
|
|
666
|
+
`Failed to serialize result: ${error instanceof Error ? error.message : String(error)}`
|
|
667
|
+
);
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
const pk = `DEDUPE#${hash}`;
|
|
671
|
+
try {
|
|
672
|
+
yield this.docClient.send(
|
|
673
|
+
new UpdateCommand({
|
|
674
|
+
TableName: this.tableName,
|
|
675
|
+
Key: { pk, sk: pk },
|
|
676
|
+
UpdateExpression: "SET #status = :completed, #result = :result, updatedAt = :now",
|
|
677
|
+
ConditionExpression: "attribute_exists(pk) AND #status = :pending",
|
|
678
|
+
ExpressionAttributeNames: {
|
|
679
|
+
"#status": "status",
|
|
680
|
+
"#result": "result"
|
|
681
|
+
},
|
|
682
|
+
ExpressionAttributeValues: {
|
|
683
|
+
":completed": "completed",
|
|
684
|
+
":pending": "pending",
|
|
685
|
+
":result": serializedResult,
|
|
686
|
+
":now": Date.now()
|
|
687
|
+
}
|
|
688
|
+
})
|
|
689
|
+
);
|
|
690
|
+
} catch (error) {
|
|
691
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
692
|
+
if (error && typeof error === "object" && "name" in error && error.name === "ConditionalCheckFailedException") {
|
|
693
|
+
return;
|
|
694
|
+
}
|
|
695
|
+
throw error;
|
|
696
|
+
}
|
|
697
|
+
const settle = this.jobSettlers.get(hash);
|
|
698
|
+
if (settle) {
|
|
699
|
+
settle(value);
|
|
700
|
+
}
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
fail(hash, _error) {
|
|
704
|
+
return __async(this, null, function* () {
|
|
705
|
+
if (this.isDestroyed) {
|
|
706
|
+
throw new Error("Dedupe store has been destroyed");
|
|
707
|
+
}
|
|
708
|
+
this.assertValidHash(hash);
|
|
709
|
+
const pk = `DEDUPE#${hash}`;
|
|
710
|
+
try {
|
|
711
|
+
yield this.docClient.send(
|
|
712
|
+
new UpdateCommand({
|
|
713
|
+
TableName: this.tableName,
|
|
714
|
+
Key: { pk, sk: pk },
|
|
715
|
+
UpdateExpression: "SET #status = :failed, #error = :error, updatedAt = :now",
|
|
716
|
+
ExpressionAttributeNames: {
|
|
717
|
+
"#status": "status",
|
|
718
|
+
"#error": "error"
|
|
719
|
+
},
|
|
720
|
+
ExpressionAttributeValues: {
|
|
721
|
+
":failed": "failed",
|
|
722
|
+
":error": "Job failed",
|
|
723
|
+
":now": Date.now()
|
|
724
|
+
}
|
|
725
|
+
})
|
|
726
|
+
);
|
|
727
|
+
} catch (dynamoError) {
|
|
728
|
+
throwIfDynamoTableMissing(dynamoError, this.tableName);
|
|
729
|
+
throw dynamoError;
|
|
730
|
+
}
|
|
731
|
+
const settle = this.jobSettlers.get(hash);
|
|
732
|
+
if (settle) {
|
|
733
|
+
settle(void 0);
|
|
734
|
+
}
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
isInProgress(hash) {
|
|
738
|
+
return __async(this, null, function* () {
|
|
739
|
+
if (this.isDestroyed) {
|
|
740
|
+
throw new Error("Dedupe store has been destroyed");
|
|
741
|
+
}
|
|
742
|
+
this.assertValidHash(hash);
|
|
743
|
+
const pk = `DEDUPE#${hash}`;
|
|
744
|
+
let result;
|
|
745
|
+
try {
|
|
746
|
+
result = yield this.docClient.send(
|
|
747
|
+
new GetCommand({
|
|
748
|
+
TableName: this.tableName,
|
|
749
|
+
Key: { pk, sk: pk }
|
|
750
|
+
})
|
|
751
|
+
);
|
|
752
|
+
} catch (error) {
|
|
753
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
754
|
+
throw error;
|
|
755
|
+
}
|
|
756
|
+
if (!result.Item) {
|
|
757
|
+
return false;
|
|
758
|
+
}
|
|
759
|
+
const jobExpired = this.jobTimeoutMs > 0 && Date.now() - result.Item["createdAt"] >= this.jobTimeoutMs;
|
|
760
|
+
if (jobExpired) {
|
|
761
|
+
yield this.docClient.send(
|
|
762
|
+
new DeleteCommand({
|
|
763
|
+
TableName: this.tableName,
|
|
764
|
+
Key: { pk, sk: pk }
|
|
765
|
+
})
|
|
766
|
+
);
|
|
767
|
+
return false;
|
|
768
|
+
}
|
|
769
|
+
return result.Item["status"] === "pending";
|
|
770
|
+
});
|
|
771
|
+
}
|
|
772
|
+
clear() {
|
|
773
|
+
return __async(this, null, function* () {
|
|
774
|
+
var _a;
|
|
775
|
+
if (this.isDestroyed) {
|
|
776
|
+
throw new Error("Dedupe store has been destroyed");
|
|
777
|
+
}
|
|
778
|
+
let lastEvaluatedKey;
|
|
779
|
+
do {
|
|
780
|
+
let scanResult;
|
|
781
|
+
try {
|
|
782
|
+
scanResult = yield this.docClient.send(
|
|
783
|
+
new ScanCommand({
|
|
784
|
+
TableName: this.tableName,
|
|
785
|
+
FilterExpression: "begins_with(pk, :prefix)",
|
|
786
|
+
ExpressionAttributeValues: { ":prefix": "DEDUPE#" },
|
|
787
|
+
ProjectionExpression: "pk, sk",
|
|
788
|
+
ExclusiveStartKey: lastEvaluatedKey
|
|
789
|
+
})
|
|
790
|
+
);
|
|
791
|
+
} catch (error) {
|
|
792
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
793
|
+
throw error;
|
|
794
|
+
}
|
|
795
|
+
const items = (_a = scanResult.Items) != null ? _a : [];
|
|
796
|
+
if (items.length > 0) {
|
|
797
|
+
try {
|
|
798
|
+
yield batchDeleteWithRetries(
|
|
799
|
+
this.docClient,
|
|
800
|
+
this.tableName,
|
|
801
|
+
items.map((item) => ({ pk: item["pk"], sk: item["sk"] }))
|
|
802
|
+
);
|
|
803
|
+
} catch (error) {
|
|
804
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
805
|
+
throw error;
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
lastEvaluatedKey = scanResult.LastEvaluatedKey;
|
|
809
|
+
} while (lastEvaluatedKey);
|
|
810
|
+
for (const settle of this.jobSettlers.values()) {
|
|
811
|
+
settle(void 0);
|
|
812
|
+
}
|
|
813
|
+
this.jobPromises.clear();
|
|
814
|
+
this.jobSettlers.clear();
|
|
815
|
+
});
|
|
816
|
+
}
|
|
817
|
+
close() {
|
|
818
|
+
return __async(this, null, function* () {
|
|
819
|
+
this.isDestroyed = true;
|
|
820
|
+
for (const settle of this.jobSettlers.values()) {
|
|
821
|
+
settle(void 0);
|
|
822
|
+
}
|
|
823
|
+
this.jobPromises.clear();
|
|
824
|
+
this.jobSettlers.clear();
|
|
825
|
+
if (this.isClientManaged && this.rawClient) {
|
|
826
|
+
this.rawClient.destroy();
|
|
827
|
+
}
|
|
828
|
+
});
|
|
829
|
+
}
|
|
830
|
+
destroy() {
|
|
831
|
+
this.close();
|
|
832
|
+
}
|
|
833
|
+
deserializeResult(serializedResult) {
|
|
834
|
+
try {
|
|
835
|
+
if (serializedResult === "__UNDEFINED__") {
|
|
836
|
+
return void 0;
|
|
837
|
+
}
|
|
838
|
+
if (serializedResult === "__NULL__") {
|
|
839
|
+
return null;
|
|
840
|
+
}
|
|
841
|
+
if (serializedResult) {
|
|
842
|
+
return JSON.parse(serializedResult);
|
|
843
|
+
}
|
|
844
|
+
return void 0;
|
|
845
|
+
} catch (e) {
|
|
846
|
+
return void 0;
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
assertValidHash(hash) {
|
|
850
|
+
assertDynamoKeyPart(hash, "hash");
|
|
851
|
+
}
|
|
852
|
+
};
|
|
853
|
+
var DynamoDBRateLimitStore = class {
|
|
854
|
+
constructor({
|
|
855
|
+
client,
|
|
856
|
+
region,
|
|
857
|
+
tableName = DEFAULT_TABLE_NAME,
|
|
858
|
+
defaultConfig = DEFAULT_RATE_LIMIT,
|
|
859
|
+
resourceConfigs = /* @__PURE__ */ new Map()
|
|
860
|
+
} = {}) {
|
|
861
|
+
this.isDestroyed = false;
|
|
862
|
+
this.tableName = tableName;
|
|
863
|
+
this.defaultConfig = defaultConfig;
|
|
864
|
+
this.resourceConfigs = resourceConfigs;
|
|
865
|
+
if (client instanceof DynamoDBDocumentClient) {
|
|
866
|
+
this.docClient = client;
|
|
867
|
+
this.isClientManaged = false;
|
|
868
|
+
} else if (client instanceof DynamoDBClient) {
|
|
869
|
+
this.docClient = DynamoDBDocumentClient.from(client);
|
|
870
|
+
this.isClientManaged = false;
|
|
871
|
+
} else {
|
|
872
|
+
const config = {};
|
|
873
|
+
if (region) config.region = region;
|
|
874
|
+
this.rawClient = new DynamoDBClient(config);
|
|
875
|
+
this.docClient = DynamoDBDocumentClient.from(this.rawClient);
|
|
876
|
+
this.isClientManaged = true;
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
canProceed(resource) {
|
|
880
|
+
return __async(this, null, function* () {
|
|
881
|
+
var _a;
|
|
882
|
+
if (this.isDestroyed) {
|
|
883
|
+
throw new Error("Rate limit store has been destroyed");
|
|
884
|
+
}
|
|
885
|
+
this.assertValidResource(resource);
|
|
886
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
887
|
+
const now = Date.now();
|
|
888
|
+
const windowStart = now - config.windowMs;
|
|
889
|
+
const hasCapacity = yield this.hasCapacityInWindow(
|
|
890
|
+
resource,
|
|
891
|
+
windowStart,
|
|
892
|
+
config.limit
|
|
893
|
+
);
|
|
894
|
+
return hasCapacity;
|
|
895
|
+
});
|
|
896
|
+
}
|
|
897
|
+
acquire(resource) {
|
|
898
|
+
return __async(this, null, function* () {
|
|
899
|
+
var _a;
|
|
900
|
+
if (this.isDestroyed) {
|
|
901
|
+
throw new Error("Rate limit store has been destroyed");
|
|
902
|
+
}
|
|
903
|
+
this.assertValidResource(resource);
|
|
904
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
905
|
+
if (config.limit <= 0) {
|
|
906
|
+
return false;
|
|
907
|
+
}
|
|
908
|
+
const now = Date.now();
|
|
909
|
+
const windowStart = now - config.windowMs;
|
|
910
|
+
const ttl = Math.floor((now + config.windowMs) / 1e3);
|
|
911
|
+
const eventId = randomUUID();
|
|
912
|
+
const slotPrefix = `RATELIMIT_SLOT#${resource}`;
|
|
913
|
+
const startSlot = Math.floor(Math.random() * config.limit);
|
|
914
|
+
for (let offset = 0; offset < config.limit; offset++) {
|
|
915
|
+
const slot = (startSlot + offset) % config.limit;
|
|
916
|
+
try {
|
|
917
|
+
yield this.docClient.send(
|
|
918
|
+
new TransactWriteCommand({
|
|
919
|
+
TransactItems: [
|
|
920
|
+
{
|
|
921
|
+
Put: {
|
|
922
|
+
TableName: this.tableName,
|
|
923
|
+
Item: {
|
|
924
|
+
pk: slotPrefix,
|
|
925
|
+
sk: `SLOT#${slot}`,
|
|
926
|
+
timestamp: now,
|
|
927
|
+
ttl
|
|
928
|
+
},
|
|
929
|
+
ConditionExpression: "attribute_not_exists(pk) OR #timestamp < :windowStart",
|
|
930
|
+
ExpressionAttributeNames: {
|
|
931
|
+
"#timestamp": "timestamp"
|
|
932
|
+
},
|
|
933
|
+
ExpressionAttributeValues: {
|
|
934
|
+
":windowStart": windowStart
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
},
|
|
938
|
+
{
|
|
939
|
+
Put: {
|
|
940
|
+
TableName: this.tableName,
|
|
941
|
+
Item: {
|
|
942
|
+
pk: `RATELIMIT#${resource}`,
|
|
943
|
+
sk: `TS#${now}#${eventId}`,
|
|
944
|
+
ttl,
|
|
945
|
+
timestamp: now
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
]
|
|
950
|
+
})
|
|
951
|
+
);
|
|
952
|
+
return true;
|
|
953
|
+
} catch (error) {
|
|
954
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
955
|
+
if (isConditionalTransactionFailure(error)) {
|
|
956
|
+
continue;
|
|
957
|
+
}
|
|
958
|
+
throw error;
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
return false;
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
record(resource) {
|
|
965
|
+
return __async(this, null, function* () {
|
|
966
|
+
var _a;
|
|
967
|
+
if (this.isDestroyed) {
|
|
968
|
+
throw new Error("Rate limit store has been destroyed");
|
|
969
|
+
}
|
|
970
|
+
this.assertValidResource(resource);
|
|
971
|
+
const now = Date.now();
|
|
972
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
973
|
+
const ttl = Math.floor((now + config.windowMs) / 1e3);
|
|
974
|
+
const uuid = randomUUID();
|
|
975
|
+
try {
|
|
976
|
+
yield this.docClient.send(
|
|
977
|
+
new PutCommand({
|
|
978
|
+
TableName: this.tableName,
|
|
979
|
+
Item: {
|
|
980
|
+
pk: `RATELIMIT#${resource}`,
|
|
981
|
+
sk: `TS#${now}#${uuid}`,
|
|
982
|
+
ttl,
|
|
983
|
+
timestamp: now
|
|
984
|
+
}
|
|
985
|
+
})
|
|
986
|
+
);
|
|
987
|
+
} catch (error) {
|
|
988
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
989
|
+
throw error;
|
|
990
|
+
}
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
getStatus(resource) {
|
|
994
|
+
return __async(this, null, function* () {
|
|
995
|
+
var _a;
|
|
996
|
+
if (this.isDestroyed) {
|
|
997
|
+
throw new Error("Rate limit store has been destroyed");
|
|
998
|
+
}
|
|
999
|
+
this.assertValidResource(resource);
|
|
1000
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1001
|
+
const now = Date.now();
|
|
1002
|
+
const windowStart = now - config.windowMs;
|
|
1003
|
+
const currentRequests = yield this.countRequestsInWindow(
|
|
1004
|
+
resource,
|
|
1005
|
+
windowStart
|
|
1006
|
+
);
|
|
1007
|
+
const remaining = Math.max(0, config.limit - currentRequests);
|
|
1008
|
+
return {
|
|
1009
|
+
remaining,
|
|
1010
|
+
resetTime: new Date(now + config.windowMs),
|
|
1011
|
+
limit: config.limit
|
|
1012
|
+
};
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
reset(resource) {
|
|
1016
|
+
return __async(this, null, function* () {
|
|
1017
|
+
if (this.isDestroyed) {
|
|
1018
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1019
|
+
}
|
|
1020
|
+
this.assertValidResource(resource);
|
|
1021
|
+
yield this.deleteResourceItems(resource);
|
|
1022
|
+
});
|
|
1023
|
+
}
|
|
1024
|
+
getWaitTime(resource) {
|
|
1025
|
+
return __async(this, null, function* () {
|
|
1026
|
+
var _a, _b;
|
|
1027
|
+
if (this.isDestroyed) {
|
|
1028
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1029
|
+
}
|
|
1030
|
+
this.assertValidResource(resource);
|
|
1031
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1032
|
+
if (config.limit === 0) {
|
|
1033
|
+
return config.windowMs;
|
|
1034
|
+
}
|
|
1035
|
+
const now = Date.now();
|
|
1036
|
+
const windowStart = now - config.windowMs;
|
|
1037
|
+
const hasCapacity = yield this.hasCapacityInWindow(
|
|
1038
|
+
resource,
|
|
1039
|
+
windowStart,
|
|
1040
|
+
config.limit
|
|
1041
|
+
);
|
|
1042
|
+
if (hasCapacity) {
|
|
1043
|
+
return 0;
|
|
1044
|
+
}
|
|
1045
|
+
let result;
|
|
1046
|
+
try {
|
|
1047
|
+
result = yield this.docClient.send(
|
|
1048
|
+
new QueryCommand({
|
|
1049
|
+
TableName: this.tableName,
|
|
1050
|
+
KeyConditionExpression: "pk = :pk AND sk >= :skStart",
|
|
1051
|
+
ExpressionAttributeValues: {
|
|
1052
|
+
":pk": `RATELIMIT#${resource}`,
|
|
1053
|
+
":skStart": `TS#${windowStart}`
|
|
1054
|
+
},
|
|
1055
|
+
Limit: 1,
|
|
1056
|
+
ScanIndexForward: true
|
|
1057
|
+
})
|
|
1058
|
+
);
|
|
1059
|
+
} catch (error) {
|
|
1060
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1061
|
+
throw error;
|
|
1062
|
+
}
|
|
1063
|
+
const oldestItem = (_b = result.Items) == null ? void 0 : _b[0];
|
|
1064
|
+
if (!oldestItem) {
|
|
1065
|
+
return 0;
|
|
1066
|
+
}
|
|
1067
|
+
const oldestTimestamp = oldestItem["timestamp"];
|
|
1068
|
+
if (!oldestTimestamp) {
|
|
1069
|
+
return 0;
|
|
1070
|
+
}
|
|
1071
|
+
const timeUntilOldestExpires = oldestTimestamp + config.windowMs - now;
|
|
1072
|
+
return Math.max(0, timeUntilOldestExpires);
|
|
1073
|
+
});
|
|
1074
|
+
}
|
|
1075
|
+
setResourceConfig(resource, config) {
|
|
1076
|
+
this.assertValidResource(resource);
|
|
1077
|
+
this.resourceConfigs.set(resource, config);
|
|
1078
|
+
}
|
|
1079
|
+
getResourceConfig(resource) {
|
|
1080
|
+
var _a;
|
|
1081
|
+
this.assertValidResource(resource);
|
|
1082
|
+
return (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1083
|
+
}
|
|
1084
|
+
clear() {
|
|
1085
|
+
return __async(this, null, function* () {
|
|
1086
|
+
var _a;
|
|
1087
|
+
if (this.isDestroyed) {
|
|
1088
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1089
|
+
}
|
|
1090
|
+
let lastEvaluatedKey;
|
|
1091
|
+
do {
|
|
1092
|
+
let scanResult;
|
|
1093
|
+
try {
|
|
1094
|
+
scanResult = yield this.docClient.send(
|
|
1095
|
+
new ScanCommand({
|
|
1096
|
+
TableName: this.tableName,
|
|
1097
|
+
FilterExpression: "begins_with(pk, :prefix) OR begins_with(pk, :slotPrefix)",
|
|
1098
|
+
ExpressionAttributeValues: {
|
|
1099
|
+
":prefix": "RATELIMIT#",
|
|
1100
|
+
":slotPrefix": "RATELIMIT_SLOT#"
|
|
1101
|
+
},
|
|
1102
|
+
ProjectionExpression: "pk, sk",
|
|
1103
|
+
ExclusiveStartKey: lastEvaluatedKey
|
|
1104
|
+
})
|
|
1105
|
+
);
|
|
1106
|
+
} catch (error) {
|
|
1107
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1108
|
+
throw error;
|
|
1109
|
+
}
|
|
1110
|
+
const items = (_a = scanResult.Items) != null ? _a : [];
|
|
1111
|
+
if (items.length > 0) {
|
|
1112
|
+
try {
|
|
1113
|
+
yield batchDeleteWithRetries(
|
|
1114
|
+
this.docClient,
|
|
1115
|
+
this.tableName,
|
|
1116
|
+
items.map((item) => ({ pk: item["pk"], sk: item["sk"] }))
|
|
1117
|
+
);
|
|
1118
|
+
} catch (error) {
|
|
1119
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1120
|
+
throw error;
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
lastEvaluatedKey = scanResult.LastEvaluatedKey;
|
|
1124
|
+
} while (lastEvaluatedKey);
|
|
1125
|
+
});
|
|
1126
|
+
}
|
|
1127
|
+
close() {
|
|
1128
|
+
return __async(this, null, function* () {
|
|
1129
|
+
this.isDestroyed = true;
|
|
1130
|
+
if (this.isClientManaged && this.rawClient) {
|
|
1131
|
+
this.rawClient.destroy();
|
|
1132
|
+
}
|
|
1133
|
+
});
|
|
1134
|
+
}
|
|
1135
|
+
destroy() {
|
|
1136
|
+
this.close();
|
|
1137
|
+
}
|
|
1138
|
+
countRequestsInWindow(resource, windowStart) {
|
|
1139
|
+
return __async(this, null, function* () {
|
|
1140
|
+
try {
|
|
1141
|
+
return yield queryCountAllPages(this.docClient, {
|
|
1142
|
+
TableName: this.tableName,
|
|
1143
|
+
KeyConditionExpression: "pk = :pk AND sk >= :skStart",
|
|
1144
|
+
ExpressionAttributeValues: {
|
|
1145
|
+
":pk": `RATELIMIT#${resource}`,
|
|
1146
|
+
":skStart": `TS#${windowStart}`
|
|
1147
|
+
},
|
|
1148
|
+
Select: "COUNT"
|
|
1149
|
+
});
|
|
1150
|
+
} catch (error) {
|
|
1151
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1152
|
+
throw error;
|
|
1153
|
+
}
|
|
1154
|
+
});
|
|
1155
|
+
}
|
|
1156
|
+
hasCapacityInWindow(resource, windowStart, limit) {
|
|
1157
|
+
return __async(this, null, function* () {
|
|
1158
|
+
try {
|
|
1159
|
+
const { reachedLimit } = yield queryCountUpTo(
|
|
1160
|
+
this.docClient,
|
|
1161
|
+
{
|
|
1162
|
+
TableName: this.tableName,
|
|
1163
|
+
KeyConditionExpression: "pk = :pk AND sk >= :skStart",
|
|
1164
|
+
ExpressionAttributeValues: {
|
|
1165
|
+
":pk": `RATELIMIT#${resource}`,
|
|
1166
|
+
":skStart": `TS#${windowStart}`
|
|
1167
|
+
}
|
|
1168
|
+
},
|
|
1169
|
+
limit
|
|
1170
|
+
);
|
|
1171
|
+
return !reachedLimit;
|
|
1172
|
+
} catch (error) {
|
|
1173
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1174
|
+
throw error;
|
|
1175
|
+
}
|
|
1176
|
+
});
|
|
1177
|
+
}
|
|
1178
|
+
deleteResourceItems(resource) {
|
|
1179
|
+
return __async(this, null, function* () {
|
|
1180
|
+
var _a;
|
|
1181
|
+
const partitionKeys = [
|
|
1182
|
+
`RATELIMIT#${resource}`,
|
|
1183
|
+
`RATELIMIT_SLOT#${resource}`
|
|
1184
|
+
];
|
|
1185
|
+
for (const pk of partitionKeys) {
|
|
1186
|
+
let lastEvaluatedKey;
|
|
1187
|
+
do {
|
|
1188
|
+
let queryResult;
|
|
1189
|
+
try {
|
|
1190
|
+
queryResult = yield this.docClient.send(
|
|
1191
|
+
new QueryCommand({
|
|
1192
|
+
TableName: this.tableName,
|
|
1193
|
+
KeyConditionExpression: "pk = :pk",
|
|
1194
|
+
ExpressionAttributeValues: { ":pk": pk },
|
|
1195
|
+
ProjectionExpression: "pk, sk",
|
|
1196
|
+
ExclusiveStartKey: lastEvaluatedKey
|
|
1197
|
+
})
|
|
1198
|
+
);
|
|
1199
|
+
} catch (error) {
|
|
1200
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1201
|
+
throw error;
|
|
1202
|
+
}
|
|
1203
|
+
const items = (_a = queryResult.Items) != null ? _a : [];
|
|
1204
|
+
if (items.length > 0) {
|
|
1205
|
+
try {
|
|
1206
|
+
yield batchDeleteWithRetries(
|
|
1207
|
+
this.docClient,
|
|
1208
|
+
this.tableName,
|
|
1209
|
+
items.map((item) => ({ pk: item["pk"], sk: item["sk"] }))
|
|
1210
|
+
);
|
|
1211
|
+
} catch (error) {
|
|
1212
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1213
|
+
throw error;
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
lastEvaluatedKey = queryResult.LastEvaluatedKey;
|
|
1217
|
+
} while (lastEvaluatedKey);
|
|
1218
|
+
}
|
|
1219
|
+
});
|
|
1220
|
+
}
|
|
1221
|
+
assertValidResource(resource) {
|
|
1222
|
+
assertDynamoKeyPart(resource, "resource");
|
|
1223
|
+
}
|
|
1224
|
+
};
|
|
1225
|
+
var DEFAULT_ADAPTIVE_RATE_LIMIT = {
|
|
1226
|
+
limit: 200,
|
|
1227
|
+
windowMs: 36e5
|
|
1228
|
+
// 1 hour
|
|
1229
|
+
};
|
|
1230
|
+
var DynamoDBAdaptiveRateLimitStore = class {
|
|
1231
|
+
constructor({
|
|
1232
|
+
client,
|
|
1233
|
+
region,
|
|
1234
|
+
tableName = DEFAULT_TABLE_NAME,
|
|
1235
|
+
defaultConfig = DEFAULT_ADAPTIVE_RATE_LIMIT,
|
|
1236
|
+
resourceConfigs = /* @__PURE__ */ new Map(),
|
|
1237
|
+
adaptiveConfig = {}
|
|
1238
|
+
} = {}) {
|
|
1239
|
+
this.isDestroyed = false;
|
|
1240
|
+
this.activityMetrics = /* @__PURE__ */ new Map();
|
|
1241
|
+
this.lastCapacityUpdate = /* @__PURE__ */ new Map();
|
|
1242
|
+
this.cachedCapacity = /* @__PURE__ */ new Map();
|
|
1243
|
+
this.tableName = tableName;
|
|
1244
|
+
this.defaultConfig = defaultConfig;
|
|
1245
|
+
this.resourceConfigs = resourceConfigs;
|
|
1246
|
+
this.capacityCalculator = new AdaptiveCapacityCalculator(adaptiveConfig);
|
|
1247
|
+
this.maxMetricSamples = Math.max(
|
|
1248
|
+
100,
|
|
1249
|
+
this.capacityCalculator.config.highActivityThreshold * 20
|
|
1250
|
+
);
|
|
1251
|
+
if (client instanceof DynamoDBDocumentClient) {
|
|
1252
|
+
this.docClient = client;
|
|
1253
|
+
this.isClientManaged = false;
|
|
1254
|
+
} else if (client instanceof DynamoDBClient) {
|
|
1255
|
+
this.docClient = DynamoDBDocumentClient.from(client);
|
|
1256
|
+
this.isClientManaged = false;
|
|
1257
|
+
} else {
|
|
1258
|
+
const config = {};
|
|
1259
|
+
if (region) config.region = region;
|
|
1260
|
+
this.rawClient = new DynamoDBClient(config);
|
|
1261
|
+
this.docClient = DynamoDBDocumentClient.from(this.rawClient);
|
|
1262
|
+
this.isClientManaged = true;
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
canProceed(resource, priority = "background") {
|
|
1266
|
+
return __async(this, null, function* () {
|
|
1267
|
+
if (this.isDestroyed) {
|
|
1268
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1269
|
+
}
|
|
1270
|
+
this.assertValidResource(resource);
|
|
1271
|
+
yield this.ensureActivityMetrics(resource);
|
|
1272
|
+
const metrics = this.getOrCreateActivityMetrics(resource);
|
|
1273
|
+
const capacity = this.calculateCurrentCapacity(resource, metrics);
|
|
1274
|
+
if (priority === "background" && capacity.backgroundPaused) {
|
|
1275
|
+
return false;
|
|
1276
|
+
}
|
|
1277
|
+
if (priority === "user") {
|
|
1278
|
+
if (capacity.userReserved <= 0) {
|
|
1279
|
+
return false;
|
|
1280
|
+
}
|
|
1281
|
+
return this.hasPriorityCapacityInWindow(
|
|
1282
|
+
resource,
|
|
1283
|
+
"user",
|
|
1284
|
+
capacity.userReserved
|
|
1285
|
+
);
|
|
1286
|
+
} else {
|
|
1287
|
+
if (capacity.backgroundMax <= 0) {
|
|
1288
|
+
return false;
|
|
1289
|
+
}
|
|
1290
|
+
return this.hasPriorityCapacityInWindow(
|
|
1291
|
+
resource,
|
|
1292
|
+
"background",
|
|
1293
|
+
capacity.backgroundMax
|
|
1294
|
+
);
|
|
1295
|
+
}
|
|
1296
|
+
});
|
|
1297
|
+
}
|
|
1298
|
+
acquire(resource, priority = "background") {
|
|
1299
|
+
return __async(this, null, function* () {
|
|
1300
|
+
var _a;
|
|
1301
|
+
if (this.isDestroyed) {
|
|
1302
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1303
|
+
}
|
|
1304
|
+
this.assertValidResource(resource);
|
|
1305
|
+
yield this.ensureActivityMetrics(resource);
|
|
1306
|
+
const metrics = this.getOrCreateActivityMetrics(resource);
|
|
1307
|
+
const capacity = this.calculateCurrentCapacity(resource, metrics);
|
|
1308
|
+
if (priority === "background" && capacity.backgroundPaused) {
|
|
1309
|
+
return false;
|
|
1310
|
+
}
|
|
1311
|
+
const limitForPriority = priority === "user" ? capacity.userReserved : capacity.backgroundMax;
|
|
1312
|
+
if (limitForPriority <= 0) {
|
|
1313
|
+
return false;
|
|
1314
|
+
}
|
|
1315
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1316
|
+
const now = Date.now();
|
|
1317
|
+
const windowStart = now - config.windowMs;
|
|
1318
|
+
const ttl = Math.floor((now + config.windowMs) / 1e3);
|
|
1319
|
+
const uuid = randomUUID();
|
|
1320
|
+
const slotPrefix = `RATELIMIT_SLOT#${resource}#${priority}`;
|
|
1321
|
+
const startSlot = Math.floor(Math.random() * limitForPriority);
|
|
1322
|
+
for (let offset = 0; offset < limitForPriority; offset++) {
|
|
1323
|
+
const slot = (startSlot + offset) % limitForPriority;
|
|
1324
|
+
try {
|
|
1325
|
+
yield this.docClient.send(
|
|
1326
|
+
new TransactWriteCommand({
|
|
1327
|
+
TransactItems: [
|
|
1328
|
+
{
|
|
1329
|
+
Put: {
|
|
1330
|
+
TableName: this.tableName,
|
|
1331
|
+
Item: {
|
|
1332
|
+
pk: slotPrefix,
|
|
1333
|
+
sk: `SLOT#${slot}`,
|
|
1334
|
+
timestamp: now,
|
|
1335
|
+
ttl
|
|
1336
|
+
},
|
|
1337
|
+
ConditionExpression: "attribute_not_exists(pk) OR #timestamp < :windowStart",
|
|
1338
|
+
ExpressionAttributeNames: {
|
|
1339
|
+
"#timestamp": "timestamp"
|
|
1340
|
+
},
|
|
1341
|
+
ExpressionAttributeValues: {
|
|
1342
|
+
":windowStart": windowStart
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
},
|
|
1346
|
+
{
|
|
1347
|
+
Put: {
|
|
1348
|
+
TableName: this.tableName,
|
|
1349
|
+
Item: {
|
|
1350
|
+
pk: `RATELIMIT#${resource}`,
|
|
1351
|
+
sk: `TS#${now}#${uuid}`,
|
|
1352
|
+
gsi1pk: `RATELIMIT#${resource}#${priority}`,
|
|
1353
|
+
gsi1sk: `TS#${now}#${uuid}`,
|
|
1354
|
+
ttl,
|
|
1355
|
+
timestamp: now,
|
|
1356
|
+
priority
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
]
|
|
1361
|
+
})
|
|
1362
|
+
);
|
|
1363
|
+
if (priority === "user") {
|
|
1364
|
+
this.pushRecentRequest(metrics.recentUserRequests, now);
|
|
1365
|
+
} else {
|
|
1366
|
+
this.pushRecentRequest(metrics.recentBackgroundRequests, now);
|
|
1367
|
+
}
|
|
1368
|
+
metrics.userActivityTrend = this.capacityCalculator.calculateActivityTrend(
|
|
1369
|
+
metrics.recentUserRequests
|
|
1370
|
+
);
|
|
1371
|
+
return true;
|
|
1372
|
+
} catch (error) {
|
|
1373
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1374
|
+
if (isConditionalTransactionFailure(error)) {
|
|
1375
|
+
continue;
|
|
1376
|
+
}
|
|
1377
|
+
throw error;
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
return false;
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1383
|
+
record(resource, priority = "background") {
|
|
1384
|
+
return __async(this, null, function* () {
|
|
1385
|
+
var _a;
|
|
1386
|
+
if (this.isDestroyed) {
|
|
1387
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1388
|
+
}
|
|
1389
|
+
this.assertValidResource(resource);
|
|
1390
|
+
const now = Date.now();
|
|
1391
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1392
|
+
const ttl = Math.floor((now + config.windowMs) / 1e3);
|
|
1393
|
+
const uuid = randomUUID();
|
|
1394
|
+
try {
|
|
1395
|
+
yield this.docClient.send(
|
|
1396
|
+
new PutCommand({
|
|
1397
|
+
TableName: this.tableName,
|
|
1398
|
+
Item: {
|
|
1399
|
+
pk: `RATELIMIT#${resource}`,
|
|
1400
|
+
sk: `TS#${now}#${uuid}`,
|
|
1401
|
+
gsi1pk: `RATELIMIT#${resource}#${priority}`,
|
|
1402
|
+
gsi1sk: `TS#${now}#${uuid}`,
|
|
1403
|
+
ttl,
|
|
1404
|
+
timestamp: now,
|
|
1405
|
+
priority
|
|
1406
|
+
}
|
|
1407
|
+
})
|
|
1408
|
+
);
|
|
1409
|
+
} catch (error) {
|
|
1410
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1411
|
+
throw error;
|
|
1412
|
+
}
|
|
1413
|
+
const metrics = this.getOrCreateActivityMetrics(resource);
|
|
1414
|
+
if (priority === "user") {
|
|
1415
|
+
this.pushRecentRequest(metrics.recentUserRequests, now);
|
|
1416
|
+
} else {
|
|
1417
|
+
this.pushRecentRequest(metrics.recentBackgroundRequests, now);
|
|
1418
|
+
}
|
|
1419
|
+
metrics.userActivityTrend = this.capacityCalculator.calculateActivityTrend(
|
|
1420
|
+
metrics.recentUserRequests
|
|
1421
|
+
);
|
|
1422
|
+
});
|
|
1423
|
+
}
|
|
1424
|
+
getStatus(resource) {
|
|
1425
|
+
return __async(this, null, function* () {
|
|
1426
|
+
var _a;
|
|
1427
|
+
if (this.isDestroyed) {
|
|
1428
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1429
|
+
}
|
|
1430
|
+
this.assertValidResource(resource);
|
|
1431
|
+
yield this.ensureActivityMetrics(resource);
|
|
1432
|
+
const metrics = this.getOrCreateActivityMetrics(resource);
|
|
1433
|
+
const capacity = this.calculateCurrentCapacity(resource, metrics);
|
|
1434
|
+
const [currentUserUsage, currentBackgroundUsage] = yield Promise.all([
|
|
1435
|
+
this.getCurrentUsage(resource, "user"),
|
|
1436
|
+
this.getCurrentUsage(resource, "background")
|
|
1437
|
+
]);
|
|
1438
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1439
|
+
return {
|
|
1440
|
+
remaining: capacity.userReserved - currentUserUsage + (capacity.backgroundMax - currentBackgroundUsage),
|
|
1441
|
+
resetTime: new Date(Date.now() + config.windowMs),
|
|
1442
|
+
limit: this.getResourceLimit(resource),
|
|
1443
|
+
adaptive: {
|
|
1444
|
+
userReserved: capacity.userReserved,
|
|
1445
|
+
backgroundMax: capacity.backgroundMax,
|
|
1446
|
+
backgroundPaused: capacity.backgroundPaused,
|
|
1447
|
+
recentUserActivity: this.capacityCalculator.getRecentActivity(
|
|
1448
|
+
metrics.recentUserRequests
|
|
1449
|
+
),
|
|
1450
|
+
reason: capacity.reason
|
|
1451
|
+
}
|
|
1452
|
+
};
|
|
1453
|
+
});
|
|
1454
|
+
}
|
|
1455
|
+
reset(resource) {
|
|
1456
|
+
return __async(this, null, function* () {
|
|
1457
|
+
if (this.isDestroyed) {
|
|
1458
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1459
|
+
}
|
|
1460
|
+
this.assertValidResource(resource);
|
|
1461
|
+
yield this.deleteResourceItems(resource);
|
|
1462
|
+
this.activityMetrics.delete(resource);
|
|
1463
|
+
this.cachedCapacity.delete(resource);
|
|
1464
|
+
this.lastCapacityUpdate.delete(resource);
|
|
1465
|
+
});
|
|
1466
|
+
}
|
|
1467
|
+
getWaitTime(resource, priority = "background") {
|
|
1468
|
+
return __async(this, null, function* () {
|
|
1469
|
+
var _a, _b;
|
|
1470
|
+
if (this.isDestroyed) {
|
|
1471
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1472
|
+
}
|
|
1473
|
+
this.assertValidResource(resource);
|
|
1474
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1475
|
+
if (config.limit === 0) {
|
|
1476
|
+
return config.windowMs;
|
|
1477
|
+
}
|
|
1478
|
+
const canProceed = yield this.canProceed(resource, priority);
|
|
1479
|
+
if (canProceed) {
|
|
1480
|
+
return 0;
|
|
1481
|
+
}
|
|
1482
|
+
yield this.ensureActivityMetrics(resource);
|
|
1483
|
+
const metrics = this.getOrCreateActivityMetrics(resource);
|
|
1484
|
+
const capacity = this.calculateCurrentCapacity(resource, metrics);
|
|
1485
|
+
if (priority === "background" && capacity.backgroundPaused) {
|
|
1486
|
+
return this.capacityCalculator.config.recalculationIntervalMs;
|
|
1487
|
+
}
|
|
1488
|
+
const now = Date.now();
|
|
1489
|
+
const windowStart = now - config.windowMs;
|
|
1490
|
+
let result;
|
|
1491
|
+
try {
|
|
1492
|
+
result = yield this.docClient.send(
|
|
1493
|
+
new QueryCommand({
|
|
1494
|
+
TableName: this.tableName,
|
|
1495
|
+
IndexName: "gsi1",
|
|
1496
|
+
KeyConditionExpression: "gsi1pk = :gsi1pk AND gsi1sk >= :skStart",
|
|
1497
|
+
ExpressionAttributeValues: {
|
|
1498
|
+
":gsi1pk": `RATELIMIT#${resource}#${priority}`,
|
|
1499
|
+
":skStart": `TS#${windowStart}`
|
|
1500
|
+
},
|
|
1501
|
+
Limit: 1,
|
|
1502
|
+
ScanIndexForward: true
|
|
1503
|
+
})
|
|
1504
|
+
);
|
|
1505
|
+
} catch (error) {
|
|
1506
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1507
|
+
throw error;
|
|
1508
|
+
}
|
|
1509
|
+
const oldestItem = (_b = result.Items) == null ? void 0 : _b[0];
|
|
1510
|
+
if (!oldestItem) {
|
|
1511
|
+
return 0;
|
|
1512
|
+
}
|
|
1513
|
+
const oldestTimestamp = oldestItem["timestamp"];
|
|
1514
|
+
if (!oldestTimestamp) {
|
|
1515
|
+
return 0;
|
|
1516
|
+
}
|
|
1517
|
+
const timeUntilOldestExpires = oldestTimestamp + config.windowMs - now;
|
|
1518
|
+
return Math.max(0, timeUntilOldestExpires);
|
|
1519
|
+
});
|
|
1520
|
+
}
|
|
1521
|
+
setResourceConfig(resource, config) {
|
|
1522
|
+
this.assertValidResource(resource);
|
|
1523
|
+
this.resourceConfigs.set(resource, config);
|
|
1524
|
+
}
|
|
1525
|
+
getResourceConfig(resource) {
|
|
1526
|
+
var _a;
|
|
1527
|
+
this.assertValidResource(resource);
|
|
1528
|
+
return (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1529
|
+
}
|
|
1530
|
+
clear() {
|
|
1531
|
+
return __async(this, null, function* () {
|
|
1532
|
+
var _a;
|
|
1533
|
+
if (this.isDestroyed) {
|
|
1534
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1535
|
+
}
|
|
1536
|
+
let lastEvaluatedKey;
|
|
1537
|
+
do {
|
|
1538
|
+
let scanResult;
|
|
1539
|
+
try {
|
|
1540
|
+
scanResult = yield this.docClient.send(
|
|
1541
|
+
new ScanCommand({
|
|
1542
|
+
TableName: this.tableName,
|
|
1543
|
+
FilterExpression: "begins_with(pk, :prefix) OR begins_with(pk, :slotPrefix)",
|
|
1544
|
+
ExpressionAttributeValues: {
|
|
1545
|
+
":prefix": "RATELIMIT#",
|
|
1546
|
+
":slotPrefix": "RATELIMIT_SLOT#"
|
|
1547
|
+
},
|
|
1548
|
+
ProjectionExpression: "pk, sk",
|
|
1549
|
+
ExclusiveStartKey: lastEvaluatedKey
|
|
1550
|
+
})
|
|
1551
|
+
);
|
|
1552
|
+
} catch (error) {
|
|
1553
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1554
|
+
throw error;
|
|
1555
|
+
}
|
|
1556
|
+
const items = (_a = scanResult.Items) != null ? _a : [];
|
|
1557
|
+
if (items.length > 0) {
|
|
1558
|
+
try {
|
|
1559
|
+
yield batchDeleteWithRetries(
|
|
1560
|
+
this.docClient,
|
|
1561
|
+
this.tableName,
|
|
1562
|
+
items.map((item) => ({ pk: item["pk"], sk: item["sk"] }))
|
|
1563
|
+
);
|
|
1564
|
+
} catch (error) {
|
|
1565
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1566
|
+
throw error;
|
|
1567
|
+
}
|
|
1568
|
+
}
|
|
1569
|
+
lastEvaluatedKey = scanResult.LastEvaluatedKey;
|
|
1570
|
+
} while (lastEvaluatedKey);
|
|
1571
|
+
this.activityMetrics.clear();
|
|
1572
|
+
this.cachedCapacity.clear();
|
|
1573
|
+
this.lastCapacityUpdate.clear();
|
|
1574
|
+
});
|
|
1575
|
+
}
|
|
1576
|
+
close() {
|
|
1577
|
+
return __async(this, null, function* () {
|
|
1578
|
+
this.isDestroyed = true;
|
|
1579
|
+
if (this.isClientManaged && this.rawClient) {
|
|
1580
|
+
this.rawClient.destroy();
|
|
1581
|
+
}
|
|
1582
|
+
});
|
|
1583
|
+
}
|
|
1584
|
+
destroy() {
|
|
1585
|
+
this.close();
|
|
1586
|
+
}
|
|
1587
|
+
// Private helper methods
|
|
1588
|
+
calculateCurrentCapacity(resource, metrics) {
|
|
1589
|
+
var _a, _b;
|
|
1590
|
+
const lastUpdate = (_a = this.lastCapacityUpdate.get(resource)) != null ? _a : 0;
|
|
1591
|
+
const recalcInterval = this.capacityCalculator.config.recalculationIntervalMs;
|
|
1592
|
+
if (Date.now() - lastUpdate < recalcInterval) {
|
|
1593
|
+
return (_b = this.cachedCapacity.get(resource)) != null ? _b : this.getDefaultCapacity(resource);
|
|
1594
|
+
}
|
|
1595
|
+
const totalLimit = this.getResourceLimit(resource);
|
|
1596
|
+
const capacity = this.capacityCalculator.calculateDynamicCapacity(
|
|
1597
|
+
resource,
|
|
1598
|
+
totalLimit,
|
|
1599
|
+
metrics
|
|
1600
|
+
);
|
|
1601
|
+
this.cachedCapacity.set(resource, capacity);
|
|
1602
|
+
this.lastCapacityUpdate.set(resource, Date.now());
|
|
1603
|
+
return capacity;
|
|
1604
|
+
}
|
|
1605
|
+
getOrCreateActivityMetrics(resource) {
|
|
1606
|
+
if (!this.activityMetrics.has(resource)) {
|
|
1607
|
+
this.activityMetrics.set(resource, {
|
|
1608
|
+
recentUserRequests: [],
|
|
1609
|
+
recentBackgroundRequests: [],
|
|
1610
|
+
userActivityTrend: "none"
|
|
1611
|
+
});
|
|
1612
|
+
}
|
|
1613
|
+
return this.activityMetrics.get(resource);
|
|
1614
|
+
}
|
|
1615
|
+
ensureActivityMetrics(resource) {
|
|
1616
|
+
return __async(this, null, function* () {
|
|
1617
|
+
if (this.activityMetrics.has(resource)) {
|
|
1618
|
+
return;
|
|
1619
|
+
}
|
|
1620
|
+
const now = Date.now();
|
|
1621
|
+
const windowStart = now - this.capacityCalculator.config.monitoringWindowMs;
|
|
1622
|
+
let userItems;
|
|
1623
|
+
let backgroundItems;
|
|
1624
|
+
try {
|
|
1625
|
+
[userItems, backgroundItems] = yield Promise.all([
|
|
1626
|
+
queryItemsAllPages(this.docClient, {
|
|
1627
|
+
TableName: this.tableName,
|
|
1628
|
+
IndexName: "gsi1",
|
|
1629
|
+
KeyConditionExpression: "gsi1pk = :gsi1pk AND gsi1sk >= :skStart",
|
|
1630
|
+
ExpressionAttributeValues: {
|
|
1631
|
+
":gsi1pk": `RATELIMIT#${resource}#user`,
|
|
1632
|
+
":skStart": `TS#${windowStart}`
|
|
1633
|
+
},
|
|
1634
|
+
ProjectionExpression: "#ts",
|
|
1635
|
+
ExpressionAttributeNames: { "#ts": "timestamp" }
|
|
1636
|
+
}),
|
|
1637
|
+
queryItemsAllPages(this.docClient, {
|
|
1638
|
+
TableName: this.tableName,
|
|
1639
|
+
IndexName: "gsi1",
|
|
1640
|
+
KeyConditionExpression: "gsi1pk = :gsi1pk AND gsi1sk >= :skStart",
|
|
1641
|
+
ExpressionAttributeValues: {
|
|
1642
|
+
":gsi1pk": `RATELIMIT#${resource}#background`,
|
|
1643
|
+
":skStart": `TS#${windowStart}`
|
|
1644
|
+
},
|
|
1645
|
+
ProjectionExpression: "#ts",
|
|
1646
|
+
ExpressionAttributeNames: { "#ts": "timestamp" }
|
|
1647
|
+
})
|
|
1648
|
+
]);
|
|
1649
|
+
} catch (error) {
|
|
1650
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1651
|
+
throw error;
|
|
1652
|
+
}
|
|
1653
|
+
const metrics = {
|
|
1654
|
+
recentUserRequests: userItems.map((item) => item["timestamp"]).slice(-this.maxMetricSamples),
|
|
1655
|
+
recentBackgroundRequests: backgroundItems.map((item) => item["timestamp"]).slice(-this.maxMetricSamples),
|
|
1656
|
+
userActivityTrend: "none"
|
|
1657
|
+
};
|
|
1658
|
+
this.cleanupOldRequests(metrics.recentUserRequests);
|
|
1659
|
+
this.cleanupOldRequests(metrics.recentBackgroundRequests);
|
|
1660
|
+
metrics.userActivityTrend = this.capacityCalculator.calculateActivityTrend(
|
|
1661
|
+
metrics.recentUserRequests
|
|
1662
|
+
);
|
|
1663
|
+
this.activityMetrics.set(resource, metrics);
|
|
1664
|
+
});
|
|
1665
|
+
}
|
|
1666
|
+
getCurrentUsage(resource, priority) {
|
|
1667
|
+
return __async(this, null, function* () {
|
|
1668
|
+
var _a;
|
|
1669
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1670
|
+
const now = Date.now();
|
|
1671
|
+
const windowStart = now - config.windowMs;
|
|
1672
|
+
try {
|
|
1673
|
+
return yield queryCountAllPages(this.docClient, {
|
|
1674
|
+
TableName: this.tableName,
|
|
1675
|
+
IndexName: "gsi1",
|
|
1676
|
+
KeyConditionExpression: "gsi1pk = :gsi1pk AND gsi1sk >= :skStart",
|
|
1677
|
+
ExpressionAttributeValues: {
|
|
1678
|
+
":gsi1pk": `RATELIMIT#${resource}#${priority}`,
|
|
1679
|
+
":skStart": `TS#${windowStart}`
|
|
1680
|
+
},
|
|
1681
|
+
Select: "COUNT"
|
|
1682
|
+
});
|
|
1683
|
+
} catch (error) {
|
|
1684
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1685
|
+
throw error;
|
|
1686
|
+
}
|
|
1687
|
+
});
|
|
1688
|
+
}
|
|
1689
|
+
hasPriorityCapacityInWindow(resource, priority, limit) {
|
|
1690
|
+
return __async(this, null, function* () {
|
|
1691
|
+
var _a;
|
|
1692
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1693
|
+
const now = Date.now();
|
|
1694
|
+
const windowStart = now - config.windowMs;
|
|
1695
|
+
try {
|
|
1696
|
+
const { reachedLimit } = yield queryCountUpTo(
|
|
1697
|
+
this.docClient,
|
|
1698
|
+
{
|
|
1699
|
+
TableName: this.tableName,
|
|
1700
|
+
IndexName: "gsi1",
|
|
1701
|
+
KeyConditionExpression: "gsi1pk = :gsi1pk AND gsi1sk >= :skStart",
|
|
1702
|
+
ExpressionAttributeValues: {
|
|
1703
|
+
":gsi1pk": `RATELIMIT#${resource}#${priority}`,
|
|
1704
|
+
":skStart": `TS#${windowStart}`
|
|
1705
|
+
}
|
|
1706
|
+
},
|
|
1707
|
+
limit
|
|
1708
|
+
);
|
|
1709
|
+
return !reachedLimit;
|
|
1710
|
+
} catch (error) {
|
|
1711
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1712
|
+
throw error;
|
|
1713
|
+
}
|
|
1714
|
+
});
|
|
1715
|
+
}
|
|
1716
|
+
cleanupOldRequests(requests) {
|
|
1717
|
+
const cutoff = Date.now() - this.capacityCalculator.config.monitoringWindowMs;
|
|
1718
|
+
const idx = requests.findIndex((t) => t >= cutoff);
|
|
1719
|
+
if (idx > 0) {
|
|
1720
|
+
requests.splice(0, idx);
|
|
1721
|
+
} else if (idx === -1 && requests.length > 0) {
|
|
1722
|
+
requests.length = 0;
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
pushRecentRequest(requests, timestamp) {
|
|
1726
|
+
requests.push(timestamp);
|
|
1727
|
+
this.cleanupOldRequests(requests);
|
|
1728
|
+
const overflow = requests.length - this.maxMetricSamples;
|
|
1729
|
+
if (overflow > 0) {
|
|
1730
|
+
requests.splice(0, overflow);
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
getResourceLimit(resource) {
|
|
1734
|
+
var _a;
|
|
1735
|
+
const config = (_a = this.resourceConfigs.get(resource)) != null ? _a : this.defaultConfig;
|
|
1736
|
+
return config.limit;
|
|
1737
|
+
}
|
|
1738
|
+
getDefaultCapacity(resource) {
|
|
1739
|
+
const limit = this.getResourceLimit(resource);
|
|
1740
|
+
const userReserved = Math.floor(limit * 0.3);
|
|
1741
|
+
const backgroundMax = Math.max(0, limit - userReserved);
|
|
1742
|
+
return {
|
|
1743
|
+
userReserved,
|
|
1744
|
+
backgroundMax,
|
|
1745
|
+
backgroundPaused: false,
|
|
1746
|
+
reason: "Default capacity allocation"
|
|
1747
|
+
};
|
|
1748
|
+
}
|
|
1749
|
+
deleteResourceItems(resource) {
|
|
1750
|
+
return __async(this, null, function* () {
|
|
1751
|
+
var _a;
|
|
1752
|
+
const partitionKeys = [
|
|
1753
|
+
`RATELIMIT#${resource}`,
|
|
1754
|
+
`RATELIMIT_SLOT#${resource}#user`,
|
|
1755
|
+
`RATELIMIT_SLOT#${resource}#background`
|
|
1756
|
+
];
|
|
1757
|
+
for (const pk of partitionKeys) {
|
|
1758
|
+
let lastEvaluatedKey;
|
|
1759
|
+
do {
|
|
1760
|
+
let queryResult;
|
|
1761
|
+
try {
|
|
1762
|
+
queryResult = yield this.docClient.send(
|
|
1763
|
+
new QueryCommand({
|
|
1764
|
+
TableName: this.tableName,
|
|
1765
|
+
KeyConditionExpression: "pk = :pk",
|
|
1766
|
+
ExpressionAttributeValues: { ":pk": pk },
|
|
1767
|
+
ProjectionExpression: "pk, sk",
|
|
1768
|
+
ExclusiveStartKey: lastEvaluatedKey
|
|
1769
|
+
})
|
|
1770
|
+
);
|
|
1771
|
+
} catch (error) {
|
|
1772
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1773
|
+
throw error;
|
|
1774
|
+
}
|
|
1775
|
+
const items = (_a = queryResult.Items) != null ? _a : [];
|
|
1776
|
+
if (items.length > 0) {
|
|
1777
|
+
try {
|
|
1778
|
+
yield batchDeleteWithRetries(
|
|
1779
|
+
this.docClient,
|
|
1780
|
+
this.tableName,
|
|
1781
|
+
items.map((item) => ({ pk: item["pk"], sk: item["sk"] }))
|
|
1782
|
+
);
|
|
1783
|
+
} catch (error) {
|
|
1784
|
+
throwIfDynamoTableMissing(error, this.tableName);
|
|
1785
|
+
throw error;
|
|
1786
|
+
}
|
|
1787
|
+
}
|
|
1788
|
+
lastEvaluatedKey = queryResult.LastEvaluatedKey;
|
|
1789
|
+
} while (lastEvaluatedKey);
|
|
1790
|
+
}
|
|
1791
|
+
});
|
|
1792
|
+
}
|
|
1793
|
+
assertValidResource(resource) {
|
|
1794
|
+
assertDynamoKeyPart(resource, "resource");
|
|
1795
|
+
}
|
|
1796
|
+
};
|
|
1797
|
+
|
|
1798
|
+
export { DEFAULT_TABLE_NAME, DynamoDBAdaptiveRateLimitStore, DynamoDBCacheStore, DynamoDBDedupeStore, DynamoDBRateLimitStore, TABLE_SCHEMA };
|
|
1799
|
+
//# sourceMappingURL=index.js.map
|
|
1800
|
+
//# sourceMappingURL=index.js.map
|