@serve.zone/dcrouter 18.1.0 → 18.1.1

package/ts/db/documents/classes.authentication-event.doc.ts

@@ -0,0 +1,248 @@
+import * as plugins from '../../plugins.js';
+import { DcRouterDb } from '../classes.dcrouter-db.js';
+import type {
+  IAuthenticationEvent,
+  TAuthenticationFailureReason,
+  TAuthenticationSource,
+  TResolvedAuthenticationSource,
+} from '../../../ts_interfaces/data/stats.js';
+
+const DB_OPERATION_TIMEOUT_MS = 5_000;
+
+const getDb = () => DcRouterDb.getInstance().getDb();
+
+@plugins.smartdata.Collection(() => getDb())
+export class AuthenticationEventDoc extends plugins.smartdata.SmartDataDbDoc<
+  AuthenticationEventDoc,
+  AuthenticationEventDoc
+> implements IAuthenticationEvent {
+  @plugins.smartdata.unI()
+  @plugins.smartdata.svDb()
+  public id!: string;
+
+  @plugins.smartdata.svDb()
+  public timestamp!: number;
+
+  @plugins.smartdata.svDb()
+  public username!: string;
+
+  @plugins.smartdata.svDb()
+  public userId?: string;
+
+  @plugins.smartdata.svDb()
+  public success!: boolean;
+
+  @plugins.smartdata.svDb()
+  public requestedAuthSource!: TAuthenticationSource;
+
+  @plugins.smartdata.svDb()
+  public resolvedAuthSource?: TResolvedAuthenticationSource;
+
+  @plugins.smartdata.svDb()
+  public failureReason?: TAuthenticationFailureReason;
+
+  constructor() {
+    super();
+  }
+
+  public toApiObject(): IAuthenticationEvent {
+    return {
+      id: this.id,
+      timestamp: this.timestamp,
+      username: this.username,
+      success: this.success,
+      requestedAuthSource: this.requestedAuthSource,
+      ...(this.userId ? { userId: this.userId } : {}),
+      ...(this.resolvedAuthSource ? { resolvedAuthSource: this.resolvedAuthSource } : {}),
+      ...(this.failureReason ? { failureReason: this.failureReason } : {}),
+    };
+  }
+
+  private static async getNativeCollection() {
+    const smartdataCollection = (AuthenticationEventDoc as typeof AuthenticationEventDoc & {
+      collection: plugins.smartdata.SmartdataCollection<AuthenticationEventDoc>;
+    }).collection;
+    await smartdataCollection.init();
+    const probe = new AuthenticationEventDoc();
+    await smartdataCollection.markUniqueIndexes(probe.uniqueIndexes || []);
+    await smartdataCollection.createRegularIndexes(probe.regularIndexes || []);
+    return smartdataCollection.mongoDbCollection;
+  }
+
+  private static toPersistedFields(eventArg: IAuthenticationEvent): IAuthenticationEvent {
+    return {
+      id: eventArg.id,
+      timestamp: eventArg.timestamp,
+      username: eventArg.username,
+      success: eventArg.success,
+      requestedAuthSource: eventArg.requestedAuthSource,
+      ...(eventArg.userId !== undefined ? { userId: eventArg.userId } : {}),
+      ...(eventArg.resolvedAuthSource !== undefined
+        ? { resolvedAuthSource: eventArg.resolvedAuthSource }
+        : {}),
+      ...(eventArg.failureReason !== undefined ? { failureReason: eventArg.failureReason } : {}),
+    };
+  }
+
+  private static validateEvent(eventArg: IAuthenticationEvent): void {
+    if (
+      typeof eventArg.id !== 'string'
+      || !eventArg.id.trim()
+      || !Number.isSafeInteger(eventArg.timestamp)
+      || eventArg.timestamp < 0
+      || typeof eventArg.username !== 'string'
+      || !eventArg.username.trim()
+      || typeof eventArg.success !== 'boolean'
+      || !['auto', 'local', 'idp.global'].includes(eventArg.requestedAuthSource)
+    ) {
+      throw new Error('Invalid authentication event');
+    }
+    if (
+      eventArg.userId !== undefined
+      && (typeof eventArg.userId !== 'string' || !eventArg.userId.trim())
+    ) {
+      throw new Error('Invalid authentication event user ID');
+    }
+    if (
+      eventArg.resolvedAuthSource !== undefined
+      && eventArg.resolvedAuthSource !== 'local'
+      && eventArg.resolvedAuthSource !== 'idp.global'
+    ) {
+      throw new Error('Invalid resolved authentication source');
+    }
+    if (
+      eventArg.failureReason !== undefined
+      && ![
+        'invalidCredentials',
+        'serviceUnavailable',
+        'identityIssuanceFailed',
+        'internalError',
+      ].includes(eventArg.failureReason)
+    ) {
+      throw new Error('Invalid authentication failure reason');
+    }
+    if (eventArg.success && eventArg.failureReason) {
+      throw new Error('Successful authentication events cannot carry a failure reason');
+    }
+    if (!eventArg.success && !eventArg.failureReason) {
+      throw new Error('Failed authentication events require a failure reason');
+    }
+  }
+
+  public static async upsertMany(eventsArg: IAuthenticationEvent[]): Promise<void> {
+    if (eventsArg.length === 0) return;
+    if (eventsArg.length > 500) {
+      throw new Error('AuthenticationEventDoc.upsertMany accepts at most 500 events');
+    }
+    for (const event of eventsArg) {
+      AuthenticationEventDoc.validateEvent(event);
+    }
+    const collection = await AuthenticationEventDoc.getNativeCollection();
+    await collection.bulkWrite(
+      eventsArg.map((event) => {
+        const timestamp = new Date(event.timestamp).toISOString();
+        const persistedEvent = AuthenticationEventDoc.toPersistedFields(event);
+        return {
+          updateOne: {
+            filter: { id: event.id },
+            update: {
+              $setOnInsert: {
+                ...persistedEvent,
+                _createdAt: timestamp,
+                _updatedAt: timestamp,
+              },
+            },
+            upsert: true,
+          },
+        };
+      }),
+      { ordered: false, timeoutMS: DB_OPERATION_TIMEOUT_MS },
+    );
+  }
+
+  public static async getWindowSummary(
+    cutoffArg: number,
+    limitArg = 100,
+  ): Promise<{
+    successes: number;
+    failures: number;
+    events: IAuthenticationEvent[];
+  }> {
+    if (!Number.isSafeInteger(cutoffArg) || cutoffArg < 0) {
+      throw new Error('AuthenticationEventDoc.getWindowSummary requires a valid cutoff');
+    }
+    if (!Number.isSafeInteger(limitArg) || limitArg <= 0 || limitArg > 500) {
+      throw new Error('AuthenticationEventDoc.getWindowSummary requires a limit from 1 to 500');
+    }
+    const collection = await AuthenticationEventDoc.getNativeCollection();
+    const selector = { timestamp: { $gte: cutoffArg } };
+    const cursor = collection
+      .find(selector, { timeoutMS: DB_OPERATION_TIMEOUT_MS })
+      .sort({ timestamp: -1, id: -1 })
+      .limit(limitArg);
+    try {
+      const [successes, failures, rows] = await Promise.all([
+        collection.countDocuments(
+          { ...selector, success: true },
+          { timeoutMS: DB_OPERATION_TIMEOUT_MS },
+        ),
+        collection.countDocuments(
+          { ...selector, success: false },
+          { timeoutMS: DB_OPERATION_TIMEOUT_MS },
+        ),
+        cursor.toArray(),
+      ]);
+      return {
+        successes,
+        failures,
+        events: rows.map((row) => AuthenticationEventDoc
+          .createInstanceFromMongoDbNativeDoc(row as any)
+          .toApiObject()),
+      };
+    } finally {
+      await cursor.close({ timeoutMS: DB_OPERATION_TIMEOUT_MS });
+    }
+  }
+
+  public static async findExistingIds(idsArg: string[]): Promise<Set<string>> {
+    const ids = [...new Set(idsArg)];
+    if (ids.length === 0) return new Set();
+    if (ids.length > 500) {
+      throw new Error('AuthenticationEventDoc.findExistingIds accepts at most 500 ids');
+    }
+    const collection = await AuthenticationEventDoc.getNativeCollection();
+    const rows = await collection
+      .find(
+        { id: { $in: ids } },
+        { projection: { id: 1 }, timeoutMS: DB_OPERATION_TIMEOUT_MS },
+      )
+      .toArray();
+    return new Set(rows.map((row) => String(row.id)));
+  }
+
+  public static async pruneBefore(cutoffArg: number): Promise<number> {
+    if (!Number.isSafeInteger(cutoffArg) || cutoffArg < 0) {
+      throw new Error('AuthenticationEventDoc.pruneBefore requires a valid cutoff');
+    }
+    const collection = await AuthenticationEventDoc.getNativeCollection();
+    let deletedCount = 0;
+    while (true) {
+      const rows = await collection
+        .find(
+          { timestamp: { $lt: cutoffArg } },
+          { projection: { _id: 1 }, timeoutMS: DB_OPERATION_TIMEOUT_MS },
+        )
+        .sort({ timestamp: 1, _id: 1 })
+        .limit(500)
+        .toArray();
+      if (rows.length === 0) break;
+      const result = await collection.deleteMany(
+        { _id: { $in: rows.map((row) => row._id) } },
+        { timeoutMS: DB_OPERATION_TIMEOUT_MS },
+      );
+      deletedCount += result.deletedCount;
+      if (rows.length < 500) break;
+    }
+    return deletedCount;
+  }
+}

package/ts/db/documents/classes.email-traffic-bucket.doc.ts

@@ -0,0 +1,161 @@
+import * as plugins from '../../plugins.js';
+import { DcRouterDb } from '../classes.dcrouter-db.js';
+
+const DB_OPERATION_TIMEOUT_MS = 5_000;
+
+const getDb = () => DcRouterDb.getInstance().getDb();
+
+export interface IEmailTrafficBucketCounts {
+  sent: number;
+  received: number;
+  failed: number;
+}
+
+export interface IEmailTrafficBucketSnapshot extends IEmailTrafficBucketCounts {
+  bucketStart: number;
+}
+
+@plugins.smartdata.Collection(() => getDb())
+export class EmailTrafficBucketDoc extends plugins.smartdata.SmartDataDbDoc<
+  EmailTrafficBucketDoc,
+  EmailTrafficBucketDoc
+> implements IEmailTrafficBucketSnapshot {
+  @plugins.smartdata.unI()
+  @plugins.smartdata.svDb()
+  public id!: string;
+
+  @plugins.smartdata.svDb()
+  public bucketStart!: number;
+
+  @plugins.smartdata.svDb()
+  public sent: number = 0;
+
+  @plugins.smartdata.svDb()
+  public received: number = 0;
+
+  @plugins.smartdata.svDb()
+  public failed: number = 0;
+
+  @plugins.smartdata.svDb()
+  public createdAt!: number;
+
+  @plugins.smartdata.svDb()
+  public updatedAt!: number;
+
+  constructor() {
+    super();
+  }
+
+  private static async getNativeCollection() {
+    const smartdataCollection = (EmailTrafficBucketDoc as typeof EmailTrafficBucketDoc & {
+      collection: plugins.smartdata.SmartdataCollection<EmailTrafficBucketDoc>;
+    }).collection;
+    await smartdataCollection.init();
+    const probe = new EmailTrafficBucketDoc();
+    await smartdataCollection.markUniqueIndexes(probe.uniqueIndexes || []);
+    await smartdataCollection.createRegularIndexes(probe.regularIndexes || []);
+    return smartdataCollection.mongoDbCollection;
+  }
+
+  private static validateSnapshot(snapshotArg: IEmailTrafficBucketSnapshot): void {
+    const counts = [snapshotArg.sent, snapshotArg.received, snapshotArg.failed];
+    if (
+      !Number.isSafeInteger(snapshotArg.bucketStart)
+      || snapshotArg.bucketStart < 0
+      || snapshotArg.bucketStart % 60_000 !== 0
+      || counts.some((count) => !Number.isSafeInteger(count) || count < 0)
+    ) {
+      throw new Error('Invalid email traffic bucket snapshot');
+    }
+  }
+
+  /** Persist retry-safe absolute monotonic counters. */
+  public static async persistAbsolute(snapshotsArg: IEmailTrafficBucketSnapshot[]): Promise<void> {
+    if (snapshotsArg.length === 0) return;
+    if (snapshotsArg.length > 500) {
+      throw new Error('EmailTrafficBucketDoc.persistAbsolute accepts at most 500 buckets');
+    }
+    for (const snapshot of snapshotsArg) {
+      EmailTrafficBucketDoc.validateSnapshot(snapshot);
+    }
+    const collection = await EmailTrafficBucketDoc.getNativeCollection();
+    const now = Date.now();
+    const updatedAt = new Date(now).toISOString();
+    await collection.bulkWrite(
+      snapshotsArg.map((snapshot) => ({
+        updateOne: {
+          filter: { bucketStart: snapshot.bucketStart },
+          update: {
+            $max: {
+              sent: snapshot.sent,
+              received: snapshot.received,
+              failed: snapshot.failed,
+            },
+            $set: {
+              updatedAt: now,
+              _updatedAt: updatedAt,
+            },
+            $setOnInsert: {
+              id: `email-traffic-${snapshot.bucketStart}`,
+              bucketStart: snapshot.bucketStart,
+              createdAt: now,
+              _createdAt: updatedAt,
+            },
+          },
+          upsert: true,
+        },
+      })),
+      { ordered: false, timeoutMS: DB_OPERATION_TIMEOUT_MS },
+    );
+  }
+
+  public static async loadSince(cutoffArg: number): Promise<IEmailTrafficBucketSnapshot[]> {
+    if (!Number.isSafeInteger(cutoffArg) || cutoffArg < 0) {
+      throw new Error('EmailTrafficBucketDoc.loadSince requires a valid cutoff');
+    }
+    const collection = await EmailTrafficBucketDoc.getNativeCollection();
+    const cursor = collection
+      .find(
+        { bucketStart: { $gte: cutoffArg } },
+        { timeoutMS: DB_OPERATION_TIMEOUT_MS },
+      )
+      .sort({ bucketStart: 1 });
+    try {
+      const rows = await cursor.toArray();
+      return rows.map((row) => ({
+        bucketStart: Number(row.bucketStart),
+        sent: Number(row.sent || 0),
+        received: Number(row.received || 0),
+        failed: Number(row.failed || 0),
+      }));
+    } finally {
+      await cursor.close({ timeoutMS: DB_OPERATION_TIMEOUT_MS });
+    }
+  }
+
+  public static async pruneBefore(cutoffArg: number): Promise<number> {
+    if (!Number.isSafeInteger(cutoffArg) || cutoffArg < 0) {
+      throw new Error('EmailTrafficBucketDoc.pruneBefore requires a valid cutoff');
+    }
+    const collection = await EmailTrafficBucketDoc.getNativeCollection();
+    let deletedCount = 0;
+    while (true) {
+      const rows = await collection
+        .find(
+          { bucketStart: { $lt: cutoffArg } },
+          { projection: { _id: 1 }, timeoutMS: DB_OPERATION_TIMEOUT_MS },
+        )
+        .sort({ bucketStart: 1, _id: 1 })
+        .limit(500)
+        .toArray();
+      if (rows.length === 0) break;
+      const result = await collection.deleteMany(
+        { _id: { $in: rows.map((row) => row._id) } },
+        { timeoutMS: DB_OPERATION_TIMEOUT_MS },
+      );
+      deletedCount += result.deletedCount;
+      if (rows.length < 500) break;
+    }
+    return deletedCount;
+  }
+}

package/ts/db/documents/index.ts

@@ -4,6 +4,8 @@ export * from './classes.cached.ip.reputation.js';
 export * from './classes.ip-intelligence.doc.js';
 export * from './classes.security-block-rule.doc.js';
 export * from './classes.security-policy-audit.doc.js';
+export * from './classes.authentication-event.doc.js';
+export * from './classes.email-traffic-bucket.doc.js';
 
 // Config document classes
 export * from './classes.route.doc.js';

package/ts/email/classes.accepted-email-spool.ts

@@ -256,6 +256,7 @@ export class AcceptedEmailSpool {
       await this.notifyEmailQueuePersisted(cachedEmail, 'email-accepted-without-queue');
     }
 
+    this.trackAcceptedInboundEmail(cachedEmail);
     return {
       accepted: true,
       smtpCode: 250,
@@ -319,6 +320,7 @@ export class AcceptedEmailSpool {
       await this.notifyEmailQueuePersisted(cachedEmail, 'raw-email-accepted-without-queue');
     }
 
+    this.trackAcceptedInboundEmail(cachedEmail);
     return {
       accepted: true,
       spoolItemId: cachedEmail.id,
@@ -480,6 +482,12 @@ export class AcceptedEmailSpool {
         logger.log('warn', `Durable envelope ${cachedEmail.id}: ${failed.length}/${dispatchResult.results.length} non-store recipients failed dispatch: ${failed.map((result) => `${result.recipient}=${result.message || result.status}`).join('; ')}`);
       }
     }
+    this.trackAcceptedInboundEmail(cachedEmail);
+  }
+
+  private trackAcceptedInboundEmail(cachedEmailArg: CachedEmail): void {
+    if (cachedEmailArg.direction !== 'inbound') return;
+    this.dcRouterRef.metricsManager?.trackEmailReceived(cachedEmailArg.from);
   }
 
   /**