@serve.zone/dcrouter 32.1.1 → 32.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno.json +1 -1
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.dcrouter.js +2 -2
- package/dist_ts/config/classes.gateway-client-manager.js +1 -1
- package/dist_ts/db/documents/classes.cached.email.d.ts +0 -1
- package/dist_ts/db/documents/classes.cached.email.js +45 -48
- package/dist_ts/db/documents/classes.email-traffic-bucket.doc.d.ts +0 -1
- package/dist_ts/db/documents/classes.email-traffic-bucket.doc.js +61 -50
- package/dist_ts/db/documents/classes.gateway-client.doc.d.ts +9 -2
- package/dist_ts/db/documents/classes.gateway-client.doc.js +9 -11
- package/dist_ts/db/documents/classes.ip-intelligence.doc.d.ts +4 -3
- package/dist_ts/db/documents/classes.ip-intelligence.doc.js +27 -31
- package/dist_ts/db/documents/classes.ops-config-event.doc.d.ts +0 -2
- package/dist_ts/db/documents/classes.ops-config-event.doc.js +70 -68
- package/dist_ts/db/documents/classes.smartmta-storage.doc.d.ts +14 -0
- package/dist_ts/db/documents/classes.smartmta-storage.doc.js +17 -3
- package/dist_ts/db/documents/classes.webpush-admission.doc.d.ts +0 -1
- package/dist_ts/db/documents/classes.webpush-admission.doc.js +14 -5
- package/dist_ts/db/documents/classes.webpush-binding.doc.d.ts +1 -1
- package/dist_ts/db/documents/classes.webpush-binding.doc.js +45 -7
- package/dist_ts/db/documents/classes.webpush-spool.doc.d.ts +0 -1
- package/dist_ts/db/documents/classes.webpush-spool.doc.js +33 -5
- package/dist_ts/dns/manager.dns.js +7 -3
- package/dist_ts/email/classes.smartmta-storage-manager.d.ts +15 -3
- package/dist_ts/email/classes.smartmta-storage-manager.js +62 -36
- package/dist_ts/security/classes.security-policy-manager.js +11 -9
- package/dist_ts/webpush/classes.webpush-manager.d.ts +5 -1
- package/dist_ts/webpush/classes.webpush-manager.js +162 -232
- package/dist_ts_web/00_commitinfo_data.js +1 -1
- package/package.json +3 -3
- package/readme.md +1 -0
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.dcrouter.ts +1 -1
- package/ts/config/classes.gateway-client-manager.ts +4 -2
- package/ts/db/documents/classes.cached.email.ts +51 -66
- package/ts/db/documents/classes.email-traffic-bucket.doc.ts +65 -59
- package/ts/db/documents/classes.gateway-client.doc.ts +38 -18
- package/ts/db/documents/classes.ip-intelligence.doc.ts +37 -40
- package/ts/db/documents/classes.ops-config-event.doc.ts +80 -77
- package/ts/db/documents/classes.smartmta-storage.doc.ts +16 -1
- package/ts/db/documents/classes.webpush-admission.doc.ts +14 -5
- package/ts/db/documents/classes.webpush-binding.doc.ts +53 -7
- package/ts/db/documents/classes.webpush-spool.doc.ts +39 -5
- package/ts/dns/manager.dns.ts +6 -2
- package/ts/email/classes.smartmta-storage-manager.ts +66 -50
- package/ts/security/classes.security-policy-manager.ts +10 -8
- package/ts/webpush/classes.webpush-manager.ts +175 -280
- package/ts_web/00_commitinfo_data.ts +1 -1
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export const commitinfo = {
|
|
5
5
|
name: '@serve.zone/dcrouter',
|
|
6
|
-
version: '32.1.
|
|
6
|
+
version: '32.1.2',
|
|
7
7
|
description: 'A multifaceted routing service handling mail and SMS delivery functions.'
|
|
8
8
|
};
|
|
9
9
|
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vdHNfd2ViLzAwX2NvbW1pdGluZm9fZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7R0FFRztBQUNILE1BQU0sQ0FBQyxNQUFNLFVBQVUsR0FBRztJQUN4QixJQUFJLEVBQUUsc0JBQXNCO0lBQzVCLE9BQU8sRUFBRSxRQUFRO0lBQ2pCLFdBQVcsRUFBRSwwRUFBMEU7Q0FDeEYsQ0FBQSJ9
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@serve.zone/dcrouter",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "32.1.
|
|
4
|
+
"version": "32.1.2",
|
|
5
5
|
"description": "A multifaceted routing service handling mail and SMS delivery functions.",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"bin": {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"@git.zone/tsbuild": "^4.5.0",
|
|
19
19
|
"@git.zone/tsbundle": "^2.13.0",
|
|
20
20
|
"@git.zone/tsdeno": "^1.8.0",
|
|
21
|
-
"@git.zone/tsdocker": "^3.
|
|
21
|
+
"@git.zone/tsdocker": "^3.6.0",
|
|
22
22
|
"@git.zone/tsrun": "^2.0.6",
|
|
23
23
|
"@git.zone/tstest": "^6.1.1",
|
|
24
24
|
"@git.zone/tswatch": "^3.3.5",
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
"@design.estate/dees-catalog": "^10.0.0",
|
|
37
37
|
"@design.estate/dees-element": "^3.1.1",
|
|
38
38
|
"@idp.global/sdk": "18.0.0",
|
|
39
|
-
"@lossless.org/client": "1.
|
|
39
|
+
"@lossless.org/client": "1.1.0",
|
|
40
40
|
"@push.rocks/lik": "^6.4.1",
|
|
41
41
|
"@push.rocks/projectinfo": "^5.1.0",
|
|
42
42
|
"@push.rocks/qenv": "^6.1.4",
|
package/readme.md
CHANGED
|
@@ -350,6 +350,7 @@ Operational behavior:
|
|
|
350
350
|
- Payloads are limited to 3,500 UTF-8 bytes, TTL must be from 1 through 86,400 seconds, delivery makes at most eight attempts, and each retry sends the remaining TTL. Terminal rows retain only non-secret delivery metadata for seven days.
|
|
351
351
|
- Subscription endpoints must be public HTTPS hostnames on port 443. DNS is bounded and pinned for the request; IP literals, credentials, fragments, and any answer containing a non-global address are rejected.
|
|
352
352
|
- App credential rotation has a 15-minute overlap. VAPID rotation retains retiring keys for 90 days and, if necessary, until old queued items have terminally drained, with four VAPID keys maximum. Rotate before reaching that bound.
|
|
353
|
+
- Startup asserts the index topology of `web_push_binding`, `web_push_admission` and `web_push_spool` against the declared model and requires an exact match: a declared index that is missing is re-created on bind, while a renamed or altered index, or an index nobody declared, fails startup closed. Remedy for a hand-added index: drop it (`db.<collection>.dropIndex('<name>')`) or add it to the `web-push-provider-schema` migration and the model declaration.
|
|
353
354
|
- Upgrading to 32.0.0 reseals every stored Web Push envelope under new associated data, so `DCROUTER_WEB_PUSH_MASTER_KEY_RING` must hold the key each existing envelope names before the first 32.0.0 start; otherwise the `webpush-encryption-context` migration refuses by name and dcrouter does not start. The push payload also loses its `schemaVersion` member with `@serve.zone/interfaces` 32, so a service worker that reads it must stop expecting the field.
|
|
354
355
|
- For a key-ring rotation, add new unique key material, set `currentKeyId`, restart, and keep old encryption keys until every VAPID, delivery, and credential-recovery envelope using them has been rotated or purged. Keep each old HMAC key only until no current or overlap credential references its key id; terminal and idempotency retention does not require the old HMAC key. Remove old keys only after those conditions are true.
|
|
355
356
|
- `deleteWebPushBinding` is a destructive tombstone operation: it revokes and removes all credential hashes, prior credentials, credential-recovery envelopes, and encrypted VAPID private keys; cancels and scrubs queued work; and clears admission state. Queued work is terminalized immediately, but an already sending request remains `sending` until the remote response wins or its lease expires. An expired lease after cancellation is recorded as a failed delivery with an unknown remote outcome, never as a confirmed cancellation. Delete/disable reports success only after that lifecycle drains; a bounded drain timeout returns a retryable failure, and the same controller operation resumes cleanup. Re-enabling the same owner creates a new lifecycle, credential, and VAPID key. Clients must recreate browser push subscriptions and must not reuse deleted credentials.
|
package/ts/00_commitinfo_data.ts
CHANGED
package/ts/classes.dcrouter.ts
CHANGED
|
@@ -1353,7 +1353,7 @@ export class DcRouter {
|
|
|
1353
1353
|
logger.log('info', `smartmigration: fresh install stamped to ${migrationResult.currentVersionAfter}`);
|
|
1354
1354
|
}
|
|
1355
1355
|
|
|
1356
|
-
this.storageManager = new SmartMtaStorageManager(
|
|
1356
|
+
this.storageManager = new SmartMtaStorageManager();
|
|
1357
1357
|
|
|
1358
1358
|
// Start the cache cleaner for TTL-based document cleanup
|
|
1359
1359
|
const cleanupIntervalMs = (dbConfig.cleanupIntervalHours || 1) * 60 * 60 * 1000;
|
|
@@ -2,6 +2,8 @@ import * as plugins from '../plugins.js';
|
|
|
2
2
|
import {
|
|
3
3
|
GatewayClientDoc,
|
|
4
4
|
type IGatewayClientPendingProvisioning,
|
|
5
|
+
type TGatewayClientMutableSet,
|
|
6
|
+
type TGatewayClientUnsettableField,
|
|
5
7
|
} from '../db/index.js';
|
|
6
8
|
import type { IGatewayClient } from '../../ts_interfaces/data/workhoster.js';
|
|
7
9
|
import type { IGatewayCredentialLifecycle } from '../../ts_interfaces/data/route-management.js';
|
|
@@ -293,12 +295,12 @@ export class GatewayClientManager {
|
|
|
293
295
|
stagedAt: now,
|
|
294
296
|
stagedBy,
|
|
295
297
|
};
|
|
296
|
-
const set:
|
|
298
|
+
const set: TGatewayClientMutableSet = {
|
|
297
299
|
credentialSequence,
|
|
298
300
|
pendingProvisioning,
|
|
299
301
|
updatedAt: now,
|
|
300
302
|
};
|
|
301
|
-
const unset:
|
|
303
|
+
const unset: TGatewayClientUnsettableField[] = [];
|
|
302
304
|
if (policyUnchanged) {
|
|
303
305
|
set.name = normalized.name;
|
|
304
306
|
if (normalized.description) {
|
|
@@ -401,25 +401,19 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
|
|
|
401
401
|
}
|
|
402
402
|
|
|
403
403
|
private static async findLimited(
|
|
404
|
-
filter:
|
|
404
|
+
filter: plugins.smartdata.MongoFilter<CachedEmail>,
|
|
405
405
|
limit: number,
|
|
406
|
-
sort?:
|
|
406
|
+
sort?: plugins.smartdata.TSmartdataSort<CachedEmail>,
|
|
407
407
|
): Promise<CachedEmail[]> {
|
|
408
408
|
const safeLimit = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 25;
|
|
409
409
|
if (safeLimit === 0) {
|
|
410
410
|
return [];
|
|
411
411
|
}
|
|
412
|
-
const
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
CachedEmail as unknown as typeof plugins.smartdata.SmartDataDbDoc,
|
|
418
|
-
);
|
|
419
|
-
if (sort) {
|
|
420
|
-
cursor.mongodbCursor.sort(sort);
|
|
421
|
-
}
|
|
422
|
-
cursor.mongodbCursor.limit(safeLimit);
|
|
412
|
+
const cursor = await CachedEmail.getCursor<CachedEmail>(filter, {
|
|
413
|
+
...(sort ? { sort } : {}),
|
|
414
|
+
limit: safeLimit,
|
|
415
|
+
maxTimeMS: DB_OPERATION_TIMEOUT_MS,
|
|
416
|
+
});
|
|
423
417
|
try {
|
|
424
418
|
return await cursor.toArray();
|
|
425
419
|
} finally {
|
|
@@ -427,10 +421,6 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
|
|
|
427
421
|
}
|
|
428
422
|
}
|
|
429
423
|
|
|
430
|
-
private static getNativeCollection() {
|
|
431
|
-
return getDb().mongoDb.collection('CachedEmail');
|
|
432
|
-
}
|
|
433
|
-
|
|
434
424
|
private static buildListQuery(filterArg: ICachedEmailListFilter): Record<string, any> {
|
|
435
425
|
const query: Record<string, any> = {};
|
|
436
426
|
if (filterArg.direction) {
|
|
@@ -487,16 +477,21 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
|
|
|
487
477
|
}
|
|
488
478
|
const query = CachedEmail.buildListQuery(filterArg);
|
|
489
479
|
query.acceptedAt = { $gte: cutoffArg };
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
{
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
480
|
+
// The oldest retained message is the first row of an ascending, single-field scan;
|
|
481
|
+
// the projection keeps the read off the message bodies.
|
|
482
|
+
const cursor = await CachedEmail.getCursor<CachedEmail>(query, {
|
|
483
|
+
projection: { acceptedAt: 1 },
|
|
484
|
+
sort: { acceptedAt: 1 },
|
|
485
|
+
limit: 1,
|
|
486
|
+
maxTimeMS: DB_OPERATION_TIMEOUT_MS,
|
|
487
|
+
});
|
|
488
|
+
let oldest: CachedEmail | null;
|
|
489
|
+
try {
|
|
490
|
+
oldest = await cursor.next(false);
|
|
491
|
+
} finally {
|
|
492
|
+
await cursor.close();
|
|
493
|
+
}
|
|
494
|
+
const acceptedAt = Number(oldest?.acceptedAt);
|
|
500
495
|
return Number.isSafeInteger(acceptedAt) && acceptedAt >= 0 ? acceptedAt : null;
|
|
501
496
|
}
|
|
502
497
|
|
|
@@ -518,46 +513,36 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
|
|
|
518
513
|
}
|
|
519
514
|
const query = CachedEmail.buildListQuery(filterArg);
|
|
520
515
|
query.acceptedAt = { $gte: windowStartArg, $lte: windowEndArg };
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
{
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
)
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
const row = await cursor.next();
|
|
534
|
-
if (!row) {
|
|
535
|
-
continue;
|
|
536
|
-
}
|
|
537
|
-
const acceptedAt = Number(row.acceptedAt);
|
|
538
|
-
if (!Number.isSafeInteger(acceptedAt)) {
|
|
539
|
-
continue;
|
|
540
|
-
}
|
|
541
|
-
const bucketStart = Math.floor(acceptedAt / bucketSizeMsArg) * bucketSizeMsArg;
|
|
542
|
-
let bucket = buckets.get(bucketStart);
|
|
543
|
-
if (!bucket) {
|
|
544
|
-
bucket = { bucketStart, sent: 0, received: 0, failed: 0 };
|
|
545
|
-
buckets.set(bucketStart, bucket);
|
|
546
|
-
}
|
|
547
|
-
if (row.status === 'failed' || row.status === 'rejected') {
|
|
548
|
-
bucket.failed++;
|
|
549
|
-
} else if (row.direction === 'outbound') {
|
|
550
|
-
bucket.sent++;
|
|
551
|
-
} else if (row.direction === 'inbound') {
|
|
552
|
-
bucket.received++;
|
|
553
|
-
}
|
|
516
|
+
// Time buckets are derived per row, which no grouped-totals aggregation expresses,
|
|
517
|
+
// so the rows are streamed with a projection instead of being materialized.
|
|
518
|
+
const cursor = await CachedEmail.getCursor<CachedEmail>(query, {
|
|
519
|
+
projection: { acceptedAt: 1, direction: 1, status: 1 },
|
|
520
|
+
batchSize: 1000,
|
|
521
|
+
maxTimeMS: DB_OPERATION_TIMEOUT_MS,
|
|
522
|
+
});
|
|
523
|
+
const buckets = new Map<number, ICachedEmailTrafficBucket>();
|
|
524
|
+
await cursor.forEach(async (rowArg) => {
|
|
525
|
+
const acceptedAt = Number(rowArg.acceptedAt);
|
|
526
|
+
if (!Number.isSafeInteger(acceptedAt)) {
|
|
527
|
+
return;
|
|
554
528
|
}
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
)
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
529
|
+
const bucketStart = Math.floor(acceptedAt / bucketSizeMsArg) * bucketSizeMsArg;
|
|
530
|
+
let bucket = buckets.get(bucketStart);
|
|
531
|
+
if (!bucket) {
|
|
532
|
+
bucket = { bucketStart, sent: 0, received: 0, failed: 0 };
|
|
533
|
+
buckets.set(bucketStart, bucket);
|
|
534
|
+
}
|
|
535
|
+
if (rowArg.status === 'failed' || rowArg.status === 'rejected') {
|
|
536
|
+
bucket.failed++;
|
|
537
|
+
} else if (rowArg.direction === 'outbound') {
|
|
538
|
+
bucket.sent++;
|
|
539
|
+
} else if (rowArg.direction === 'inbound') {
|
|
540
|
+
bucket.received++;
|
|
541
|
+
}
|
|
542
|
+
});
|
|
543
|
+
return [...buckets.values()].sort((firstArg, secondArg) => (
|
|
544
|
+
firstArg.bucketStart - secondArg.bucketStart
|
|
545
|
+
));
|
|
561
546
|
}
|
|
562
547
|
|
|
563
548
|
public static async findRecentSecurityCandidates(
|
|
@@ -2,6 +2,9 @@ import * as plugins from '../../plugins.js';
|
|
|
2
2
|
import { DcRouterDb } from '../classes.dcrouter-db.js';
|
|
3
3
|
|
|
4
4
|
const DB_OPERATION_TIMEOUT_MS = 5_000;
|
|
5
|
+
const PRUNE_BATCH_SIZE = 500;
|
|
6
|
+
/** One bucket per minute; retention is 24 hours, so this is a wide safety bound. */
|
|
7
|
+
const MAX_LOADED_BUCKETS = 10_000;
|
|
5
8
|
|
|
6
9
|
const getDb = () => DcRouterDb.getInstance().getDb();
|
|
7
10
|
|
|
@@ -37,17 +40,6 @@ export class EmailTrafficBucketDoc extends plugins.smartdata.SmartDataDbDoc<
|
|
|
37
40
|
super();
|
|
38
41
|
}
|
|
39
42
|
|
|
40
|
-
private static async getNativeCollection() {
|
|
41
|
-
const smartdataCollection = (EmailTrafficBucketDoc as typeof EmailTrafficBucketDoc & {
|
|
42
|
-
collection: plugins.smartdata.SmartdataCollection<EmailTrafficBucketDoc>;
|
|
43
|
-
}).collection;
|
|
44
|
-
await smartdataCollection.init();
|
|
45
|
-
const probe = new EmailTrafficBucketDoc();
|
|
46
|
-
await smartdataCollection.markUniqueIndexes(probe.uniqueIndexes || []);
|
|
47
|
-
await smartdataCollection.createRegularIndexes(probe.regularIndexes || []);
|
|
48
|
-
return smartdataCollection.mongoDbCollection;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
43
|
private static validateSnapshot(snapshotArg: IEmailTrafficBucketSnapshot): void {
|
|
52
44
|
const counts = [snapshotArg.sent, snapshotArg.received, snapshotArg.failed];
|
|
53
45
|
if (
|
|
@@ -69,34 +61,33 @@ export class EmailTrafficBucketDoc extends plugins.smartdata.SmartDataDbDoc<
|
|
|
69
61
|
for (const snapshot of snapshotsArg) {
|
|
70
62
|
EmailTrafficBucketDoc.validateSnapshot(snapshot);
|
|
71
63
|
}
|
|
72
|
-
const collection = await EmailTrafficBucketDoc.getNativeCollection();
|
|
73
64
|
const now = Date.now();
|
|
74
|
-
|
|
75
|
-
|
|
65
|
+
// One bounded unordered batch (the client accepts 1..1000 operations and 16 MiB, so a
|
|
66
|
+
// 500-bucket flush always fits). Each operation is anchored on `bucketStart`, whose
|
|
67
|
+
// declared unique index makes the upsert idempotent under a retry, and `$max` is the
|
|
68
|
+
// server-side compare-and-write that lets a replayed absolute snapshot never lower a
|
|
69
|
+
// counter. `$setOnInsert` seeds the `@unI()` identity on the insert branch only; the
|
|
70
|
+
// client refuses an identity seed that contradicts an identity anchor in the filter.
|
|
71
|
+
await EmailTrafficBucketDoc.atomicUpsertMany<EmailTrafficBucketDoc>(
|
|
76
72
|
snapshotsArg.map((snapshot) => ({
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
bucketStart: snapshot.bucketStart,
|
|
92
|
-
createdAt: now,
|
|
93
|
-
_createdAt: updatedAt,
|
|
94
|
-
},
|
|
73
|
+
filter: { bucketStart: snapshot.bucketStart },
|
|
74
|
+
update: {
|
|
75
|
+
$max: {
|
|
76
|
+
sent: snapshot.sent,
|
|
77
|
+
received: snapshot.received,
|
|
78
|
+
failed: snapshot.failed,
|
|
79
|
+
},
|
|
80
|
+
$set: {
|
|
81
|
+
updatedAt: now,
|
|
82
|
+
},
|
|
83
|
+
$setOnInsert: {
|
|
84
|
+
id: `email-traffic-${snapshot.bucketStart}`,
|
|
85
|
+
bucketStart: snapshot.bucketStart,
|
|
86
|
+
createdAt: now,
|
|
95
87
|
},
|
|
96
|
-
upsert: true,
|
|
97
88
|
},
|
|
98
89
|
})),
|
|
99
|
-
{
|
|
90
|
+
{ timeoutMS: DB_OPERATION_TIMEOUT_MS },
|
|
100
91
|
);
|
|
101
92
|
}
|
|
102
93
|
|
|
@@ -104,23 +95,24 @@ export class EmailTrafficBucketDoc extends plugins.smartdata.SmartDataDbDoc<
|
|
|
104
95
|
if (!Number.isSafeInteger(cutoffArg) || cutoffArg < 0) {
|
|
105
96
|
throw new Error('EmailTrafficBucketDoc.loadSince requires a valid cutoff');
|
|
106
97
|
}
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
98
|
+
const cursor = await EmailTrafficBucketDoc.getCursor<EmailTrafficBucketDoc>(
|
|
99
|
+
{ bucketStart: { $gte: cutoffArg } },
|
|
100
|
+
{
|
|
101
|
+
sort: { bucketStart: 1 },
|
|
102
|
+
limit: MAX_LOADED_BUCKETS,
|
|
103
|
+
maxTimeMS: DB_OPERATION_TIMEOUT_MS,
|
|
104
|
+
},
|
|
105
|
+
);
|
|
114
106
|
try {
|
|
115
107
|
const rows = await cursor.toArray();
|
|
116
|
-
return rows.map((
|
|
117
|
-
bucketStart: Number(
|
|
118
|
-
sent: Number(
|
|
119
|
-
received: Number(
|
|
120
|
-
failed: Number(
|
|
108
|
+
return rows.map((rowArg) => ({
|
|
109
|
+
bucketStart: Number(rowArg.bucketStart),
|
|
110
|
+
sent: Number(rowArg.sent || 0),
|
|
111
|
+
received: Number(rowArg.received || 0),
|
|
112
|
+
failed: Number(rowArg.failed || 0),
|
|
121
113
|
}));
|
|
122
114
|
} finally {
|
|
123
|
-
await cursor.close(
|
|
115
|
+
await cursor.close();
|
|
124
116
|
}
|
|
125
117
|
}
|
|
126
118
|
|
|
@@ -128,24 +120,38 @@ export class EmailTrafficBucketDoc extends plugins.smartdata.SmartDataDbDoc<
|
|
|
128
120
|
if (!Number.isSafeInteger(cutoffArg) || cutoffArg < 0) {
|
|
129
121
|
throw new Error('EmailTrafficBucketDoc.pruneBefore requires a valid cutoff');
|
|
130
122
|
}
|
|
131
|
-
const collection = await EmailTrafficBucketDoc.getNativeCollection();
|
|
132
123
|
let deletedCount = 0;
|
|
133
124
|
while (true) {
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
125
|
+
const cursor = await EmailTrafficBucketDoc.getCursor<EmailTrafficBucketDoc>(
|
|
126
|
+
{ bucketStart: { $lt: cutoffArg } },
|
|
127
|
+
{
|
|
128
|
+
projection: { id: 1 },
|
|
129
|
+
sort: { bucketStart: 1, id: 1 },
|
|
130
|
+
limit: PRUNE_BATCH_SIZE,
|
|
131
|
+
maxTimeMS: DB_OPERATION_TIMEOUT_MS,
|
|
132
|
+
},
|
|
133
|
+
);
|
|
134
|
+
let rows: EmailTrafficBucketDoc[];
|
|
135
|
+
try {
|
|
136
|
+
rows = await cursor.toArray();
|
|
137
|
+
} finally {
|
|
138
|
+
await cursor.close();
|
|
139
|
+
}
|
|
142
140
|
if (rows.length === 0) break;
|
|
143
|
-
|
|
144
|
-
|
|
141
|
+
// The identity `$or` is the equality anchor a plural delete requires; the range
|
|
142
|
+
// guard is re-evaluated server-side so a bucket written after the scan survives.
|
|
143
|
+
const result = await EmailTrafficBucketDoc.atomicDeleteMany<EmailTrafficBucketDoc>(
|
|
144
|
+
{
|
|
145
|
+
$or: rows.map((rowArg) => ({ id: rowArg.id })),
|
|
146
|
+
bucketStart: { $lt: cutoffArg },
|
|
147
|
+
},
|
|
145
148
|
{ timeoutMS: DB_OPERATION_TIMEOUT_MS },
|
|
146
149
|
);
|
|
150
|
+
// A batch the scan matched but the server-side guard did not would otherwise be
|
|
151
|
+
// rescanned forever; no progress ends the sweep the same way it does for events.
|
|
152
|
+
if (result.deletedCount === 0) break;
|
|
147
153
|
deletedCount += result.deletedCount;
|
|
148
|
-
if (rows.length <
|
|
154
|
+
if (rows.length < PRUNE_BATCH_SIZE) break;
|
|
149
155
|
}
|
|
150
156
|
return deletedCount;
|
|
151
157
|
}
|
|
@@ -19,6 +19,26 @@ export interface IGatewayClientPendingProvisioning {
|
|
|
19
19
|
|
|
20
20
|
const getDb = () => DcRouterDb.getInstance().getDb();
|
|
21
21
|
|
|
22
|
+
/** Fields a compare-and-set mutation may write; the revision is owned by the fence. */
|
|
23
|
+
export type TGatewayClientMutableSet = Partial<Pick<
|
|
24
|
+
GatewayClientDoc,
|
|
25
|
+
| 'type'
|
|
26
|
+
| 'name'
|
|
27
|
+
| 'description'
|
|
28
|
+
| 'hostnamePatterns'
|
|
29
|
+
| 'allowedRouteTargets'
|
|
30
|
+
| 'capabilities'
|
|
31
|
+
| 'enabled'
|
|
32
|
+
| 'policyGeneration'
|
|
33
|
+
| 'credentialSequence'
|
|
34
|
+
| 'activeCredentialSequence'
|
|
35
|
+
| 'pendingProvisioning'
|
|
36
|
+
| 'updatedAt'
|
|
37
|
+
>>;
|
|
38
|
+
|
|
39
|
+
/** Optional fields a compare-and-set mutation may remove from the stored document. */
|
|
40
|
+
export type TGatewayClientUnsettableField = 'description' | 'pendingProvisioning';
|
|
41
|
+
|
|
22
42
|
@plugins.smartdata.Collection(() => getDb())
|
|
23
43
|
export class GatewayClientDoc extends plugins.smartdata.SmartDataDbDoc<GatewayClientDoc, GatewayClientDoc> {
|
|
24
44
|
@plugins.smartdata.unI()
|
|
@@ -80,15 +100,14 @@ export class GatewayClientDoc extends plugins.smartdata.SmartDataDbDoc<GatewayCl
|
|
|
80
100
|
return await GatewayClientDoc.getInstance({ id });
|
|
81
101
|
}
|
|
82
102
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
private static buildRevisionSelector(idArg: string, revisionArg: number): Record<string, unknown> {
|
|
103
|
+
/**
|
|
104
|
+
* Compare-and-set selector. Revision 0 also matches documents written before the
|
|
105
|
+
* revision field existed, so a first mutation fences them exactly once.
|
|
106
|
+
*/
|
|
107
|
+
private static buildRevisionSelector(
|
|
108
|
+
idArg: string,
|
|
109
|
+
revisionArg: number,
|
|
110
|
+
): plugins.smartdata.TSmartdataAtomicFilter<GatewayClientDoc> {
|
|
92
111
|
if (revisionArg === 0) {
|
|
93
112
|
return {
|
|
94
113
|
id: idArg,
|
|
@@ -104,21 +123,23 @@ export class GatewayClientDoc extends plugins.smartdata.SmartDataDbDoc<GatewayCl
|
|
|
104
123
|
public static async compareAndSet(
|
|
105
124
|
idArg: string,
|
|
106
125
|
expectedRevisionArg: number,
|
|
107
|
-
setArg:
|
|
108
|
-
unsetArg:
|
|
126
|
+
setArg: TGatewayClientMutableSet,
|
|
127
|
+
unsetArg: TGatewayClientUnsettableField[] = [],
|
|
109
128
|
): Promise<GatewayClientDoc | null> {
|
|
110
|
-
const
|
|
111
|
-
const update: Record<string, unknown> = {
|
|
129
|
+
const update: plugins.smartdata.ISmartdataAtomicUpdate<GatewayClientDoc> = {
|
|
112
130
|
$set: {
|
|
113
131
|
...setArg,
|
|
114
132
|
mutationRevision: expectedRevisionArg + 1,
|
|
115
|
-
_updatedAt: new Date().toISOString(),
|
|
116
133
|
},
|
|
117
134
|
};
|
|
118
135
|
if (unsetArg.length > 0) {
|
|
119
|
-
update.$unset = Object.fromEntries(
|
|
136
|
+
update.$unset = Object.fromEntries(
|
|
137
|
+
unsetArg.map((fieldArg) => [fieldArg, '' as const]),
|
|
138
|
+
);
|
|
120
139
|
}
|
|
121
|
-
|
|
140
|
+
// One fenced write: a mismatched revision matches nothing and the caller learns the
|
|
141
|
+
// document changed concurrently.
|
|
142
|
+
const result = await GatewayClientDoc.atomicUpdate<GatewayClientDoc>(
|
|
122
143
|
GatewayClientDoc.buildRevisionSelector(idArg, expectedRevisionArg),
|
|
123
144
|
update,
|
|
124
145
|
);
|
|
@@ -127,8 +148,7 @@ export class GatewayClientDoc extends plugins.smartdata.SmartDataDbDoc<GatewayCl
|
|
|
127
148
|
}
|
|
128
149
|
|
|
129
150
|
public static async deleteIfRevision(idArg: string, expectedRevisionArg: number): Promise<boolean> {
|
|
130
|
-
const
|
|
131
|
-
const result = await collection.deleteOne(
|
|
151
|
+
const result = await GatewayClientDoc.atomicDelete<GatewayClientDoc>(
|
|
132
152
|
GatewayClientDoc.buildRevisionSelector(idArg, expectedRevisionArg),
|
|
133
153
|
);
|
|
134
154
|
return result.deletedCount === 1;
|
|
@@ -7,6 +7,8 @@ import type {
|
|
|
7
7
|
|
|
8
8
|
const getDb = () => DcRouterDb.getInstance().getDb();
|
|
9
9
|
|
|
10
|
+
const MAX_DELETE_BATCH_SIZE = 500;
|
|
11
|
+
|
|
10
12
|
@plugins.smartdata.Collection(() => getDb())
|
|
11
13
|
export class IpIntelligenceDoc extends plugins.smartdata.SmartDataDbDoc<IpIntelligenceDoc, IpIntelligenceDoc> implements IIpIntelligenceRecord {
|
|
12
14
|
@plugins.smartdata.unI()
|
|
@@ -111,18 +113,14 @@ export class IpIntelligenceDoc extends plugins.smartdata.SmartDataDbDoc<IpIntell
|
|
|
111
113
|
throw new Error('IpIntelligenceDoc.findRecent requires a positive integer limit');
|
|
112
114
|
}
|
|
113
115
|
|
|
114
|
-
const filter = options.ipAddresses
|
|
116
|
+
const filter: plugins.smartdata.MongoFilter<IpIntelligenceDoc> = options.ipAddresses
|
|
115
117
|
? { ipAddress: { $in: options.ipAddresses } }
|
|
116
118
|
: {};
|
|
117
|
-
const
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
IpIntelligenceDoc as unknown as typeof plugins.smartdata.SmartDataDbDoc,
|
|
123
|
-
);
|
|
124
|
-
cursor.mongodbCursor.sort({ lastSeenAt: -1, _id: -1 });
|
|
125
|
-
cursor.mongodbCursor.limit(options.limit);
|
|
119
|
+
const cursor = await IpIntelligenceDoc.getCursor<IpIntelligenceDoc>(filter, {
|
|
120
|
+
// ipAddress is the immutable identity, so it is the deterministic tiebreaker.
|
|
121
|
+
sort: { lastSeenAt: -1, ipAddress: -1 },
|
|
122
|
+
limit: options.limit,
|
|
123
|
+
});
|
|
126
124
|
try {
|
|
127
125
|
return await cursor.toArray();
|
|
128
126
|
} finally {
|
|
@@ -131,50 +129,49 @@ export class IpIntelligenceDoc extends plugins.smartdata.SmartDataDbDoc<IpIntell
|
|
|
131
129
|
}
|
|
132
130
|
|
|
133
131
|
public static async countAll(): Promise<number> {
|
|
134
|
-
return await
|
|
135
|
-
.getDb()
|
|
136
|
-
.mongoDb
|
|
137
|
-
.collection('IpIntelligenceDoc')
|
|
138
|
-
.countDocuments({});
|
|
132
|
+
return await IpIntelligenceDoc.getCount<IpIntelligenceDoc>({});
|
|
139
133
|
}
|
|
140
134
|
|
|
141
|
-
|
|
135
|
+
/** Least recently seen records first; the identity is the deterministic tiebreaker. */
|
|
136
|
+
public static async findOldestIpAddresses(options: {
|
|
142
137
|
lastSeenBefore?: number;
|
|
143
138
|
limit: number;
|
|
144
|
-
}): Promise<
|
|
139
|
+
}): Promise<string[]> {
|
|
145
140
|
if (!Number.isSafeInteger(options.limit) || options.limit <= 0) {
|
|
146
|
-
throw new Error('IpIntelligenceDoc.
|
|
141
|
+
throw new Error('IpIntelligenceDoc.findOldestIpAddresses requires a positive integer limit');
|
|
147
142
|
}
|
|
148
|
-
const
|
|
149
|
-
.
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
const
|
|
153
|
-
|
|
154
|
-
: { lastSeenAt:
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
.sort({ lastSeenAt: 1, _id: 1 })
|
|
158
|
-
.limit(options.limit);
|
|
143
|
+
const filter: plugins.smartdata.MongoFilter<IpIntelligenceDoc> =
|
|
144
|
+
options.lastSeenBefore === undefined
|
|
145
|
+
? {}
|
|
146
|
+
: { lastSeenAt: { $lt: options.lastSeenBefore } };
|
|
147
|
+
const cursor = await IpIntelligenceDoc.getCursor<IpIntelligenceDoc>(filter, {
|
|
148
|
+
projection: { ipAddress: 1 },
|
|
149
|
+
sort: { lastSeenAt: 1, ipAddress: 1 },
|
|
150
|
+
limit: options.limit,
|
|
151
|
+
});
|
|
159
152
|
try {
|
|
160
153
|
const rows = await cursor.toArray();
|
|
161
|
-
return rows.map((
|
|
154
|
+
return rows.map((rowArg) => rowArg.ipAddress);
|
|
162
155
|
} finally {
|
|
163
156
|
await cursor.close();
|
|
164
157
|
}
|
|
165
158
|
}
|
|
166
159
|
|
|
167
|
-
public static async
|
|
168
|
-
if (
|
|
169
|
-
if (
|
|
170
|
-
|
|
160
|
+
public static async deleteByIpAddresses(ipAddressesArg: string[]): Promise<number> {
|
|
161
|
+
if (ipAddressesArg.length === 0) return 0;
|
|
162
|
+
if (
|
|
163
|
+
ipAddressesArg.length > MAX_DELETE_BATCH_SIZE
|
|
164
|
+
|| ipAddressesArg.some((ipArg) => typeof ipArg !== 'string' || !ipArg)
|
|
165
|
+
) {
|
|
166
|
+
throw new Error(
|
|
167
|
+
`IpIntelligenceDoc.deleteByIpAddresses requires 1 to ${MAX_DELETE_BATCH_SIZE} ip addresses`,
|
|
168
|
+
);
|
|
171
169
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
.
|
|
176
|
-
|
|
177
|
-
.deleteMany(deleteFilter);
|
|
170
|
+
// Each `$or` branch pins the immutable identity, which is the equality anchor a
|
|
171
|
+
// plural atomic delete requires.
|
|
172
|
+
const result = await IpIntelligenceDoc.atomicDeleteMany<IpIntelligenceDoc>({
|
|
173
|
+
$or: ipAddressesArg.map((ipArg) => ({ ipAddress: ipArg })),
|
|
174
|
+
});
|
|
178
175
|
return result.deletedCount;
|
|
179
176
|
}
|
|
180
177
|
}
|