@serve.zone/dcrouter 16.5.0 → 17.1.0

package/ts/email/classes.smartmta-blob-storage-manager.ts

@@ -0,0 +1,139 @@
+import * as plugins from '../plugins.js';
+import type { IBlobStorageManager } from '@push.rocks/smartmta';
+
+const SMARTMTA_BLOB_NAMESPACES = [
+  '/email/queue/',
+  '/email/attachments/',
+  '/email/messages/',
+] as const;
+const DEFAULT_OBJECT_PREFIX = 'dcrouter/smartmta/';
+const MAX_STORAGE_KEY_BYTES = 1024;
+
+type TBlobBucket = Pick<
+  plugins.smartbucket.Bucket,
+  'fastExists' | 'fastGet' | 'fastPut' | 'fastRemove' | 'listAllObjects'
+>;
+
+export interface ISmartMtaBlobStorageConfig {
+  storageDescriptor: plugins.tsclass.storage.IStorageDescriptor;
+  bucketName: string;
+  objectPrefix?: string;
+  createBucketIfMissing?: boolean;
+}
+
+function normalizeObjectPrefix(prefixArg = DEFAULT_OBJECT_PREFIX): string {
+  const prefix = prefixArg.trim().replace(/^\/+|\/+$/g, '');
+  if (!prefix || prefix.includes('\\') || prefix.includes('\0')) {
+    throw new Error('SmartMTA blob objectPrefix must be a non-empty object-key prefix');
+  }
+  const segments = prefix.split('/');
+  if (segments.some((segment) => !segment || segment === '.' || segment === '..')) {
+    throw new Error('SmartMTA blob objectPrefix contains an invalid path segment');
+  }
+  return `${prefix}/`;
+}
+
+function normalizeBlobKey(keyArg: string, allowNamespaceRoot = false): string {
+  if (
+    typeof keyArg !== 'string' ||
+    !keyArg.startsWith('/') ||
+    keyArg.includes('\\') ||
+    keyArg.includes('\0') ||
+    Buffer.byteLength(keyArg, 'utf8') > MAX_STORAGE_KEY_BYTES
+  ) {
+    throw new Error('Invalid SmartMTA blob storage key');
+  }
+  const namespace = SMARTMTA_BLOB_NAMESPACES.find((candidate) => keyArg.startsWith(candidate));
+  if (!namespace) {
+    throw new Error(`Unsupported SmartMTA blob storage namespace: ${keyArg}`);
+  }
+  const remainder = keyArg.slice(namespace.length);
+  if (!allowNamespaceRoot && !remainder) {
+    throw new Error('SmartMTA blob storage key must identify an object');
+  }
+  if (remainder && remainder.split('/').some((segment) => !segment || segment === '.' || segment === '..')) {
+    throw new Error('SmartMTA blob storage key contains an invalid path segment');
+  }
+  return keyArg;
+}
+
+/**
+ * SmartMTA blob persistence backed by a SmartBucket bucket.
+ *
+ * The adapter accepts only SmartMTA queue, attachment, and retained-message
+ * namespaces. Every key remains a SmartBucket object key.
+ */
+export class SmartMtaBlobStorageManager implements IBlobStorageManager {
+  private readonly objectPrefix: string;
+
+  public constructor(
+    private readonly bucketRef: TBlobBucket,
+    objectPrefix = DEFAULT_OBJECT_PREFIX,
+  ) {
+    this.objectPrefix = normalizeObjectPrefix(objectPrefix);
+  }
+
+  public static async create(
+    configArg: ISmartMtaBlobStorageConfig,
+  ): Promise<SmartMtaBlobStorageManager> {
+    const bucketName = configArg.bucketName?.trim();
+    if (!bucketName) {
+      throw new Error('SmartMTA blob storage requires a bucketName');
+    }
+
+    const smartBucket = new plugins.smartbucket.SmartBucket(configArg.storageDescriptor);
+    let bucket: plugins.smartbucket.Bucket;
+    if (configArg.createBucketIfMissing) {
+      bucket = await smartBucket.bucketExists(bucketName)
+        ? await smartBucket.getBucketByName(bucketName)
+        : await smartBucket.createBucket(bucketName);
+    } else {
+      bucket = await smartBucket.getBucketByName(bucketName);
+    }
+
+    return new SmartMtaBlobStorageManager(bucket, configArg.objectPrefix);
+  }
+
+  public async get(keyArg: string): Promise<Buffer | null> {
+    const objectKey = this.toObjectKey(normalizeBlobKey(keyArg));
+    if (!(await this.bucketRef.fastExists({ path: objectKey }))) {
+      return null;
+    }
+    return Buffer.from(await this.bucketRef.fastGet({ path: objectKey }));
+  }
+
+  public async set(keyArg: string, valueArg: Buffer): Promise<void> {
+    if (!Buffer.isBuffer(valueArg)) {
+      throw new TypeError('SmartMTA blob storage values must be Buffers');
+    }
+    const objectKey = this.toObjectKey(normalizeBlobKey(keyArg));
+    await this.bucketRef.fastPut({
+      path: objectKey,
+      contents: Buffer.from(valueArg),
+      overwrite: true,
+    });
+  }
+
+  public async list(prefixArg: string): Promise<string[]> {
+    const prefix = normalizeBlobKey(prefixArg, true);
+    const objectPrefix = this.toObjectKey(prefix);
+    const keys: string[] = [];
+    for await (const objectKey of this.bucketRef.listAllObjects(objectPrefix)) {
+      if (!objectKey.startsWith(this.objectPrefix)) {
+        throw new Error(`SmartBucket returned an object outside the configured prefix: ${objectKey}`);
+      }
+      const key = `/${objectKey.slice(this.objectPrefix.length)}`;
+      keys.push(normalizeBlobKey(key));
+    }
+    return keys.sort();
+  }
+
+  public async delete(keyArg: string): Promise<void> {
+    const objectKey = this.toObjectKey(normalizeBlobKey(keyArg));
+    await this.bucketRef.fastRemove({ path: objectKey });
+  }
+
+  private toObjectKey(keyArg: string): string {
+    return `${this.objectPrefix}${keyArg.slice(1)}`;
+  }
+}

package/ts/email/classes.smartmta-storage-manager.ts

@@ -1,108 +1,111 @@
 import * as plugins from '../plugins.js';
-import type { IStorageManagerLike } from '@push.rocks/smartmta';
+import type { IStorageManager } from '@push.rocks/smartmta';
+import {
+  getSmartMtaTextNamespace,
+  hashSmartMtaStorageKey,
+  normalizeSmartMtaStorageKey,
+  validateSmartMtaTextValue,
+} from './smartmta-storage-policy.js';
 
-export class SmartMtaStorageManager implements IStorageManagerLike {
-  private readonly resolvedRootDir: string;
+interface ISmartMtaStorageDocument {
+  _id: string;
+  id: string;
+  key: string;
+  value: string;
+  createdAt: string;
+  updatedAt: string;
+}
 
-  constructor(private rootDir: string) {
-    this.resolvedRootDir = plugins.path.resolve(rootDir);
-    plugins.fsUtils.ensureDirSync(this.resolvedRootDir);
-  }
+export class SmartMtaStorageManager implements IStorageManager {
+  private readonly collection: any;
 
-  private normalizeKey(key: string): string {
-    return key.replace(/^\/+/, '').replace(/\\/g, '/');
+  constructor(smartdataDb: plugins.smartdata.SmartdataDb) {
+    this.collection = smartdataDb.mongoDb.collection('SmartMtaStorageDoc');
   }
 
-  private resolvePathForKey(key: string): string {
-    const normalizedKey = this.normalizeKey(key);
-    const resolvedPath = plugins.path.resolve(this.resolvedRootDir, normalizedKey);
+  private validateDocument(
+    document: ISmartMtaStorageDocument,
+    expectedKey?: string,
+  ): ISmartMtaStorageDocument {
+    const normalizedKey = normalizeSmartMtaStorageKey(document.key);
+    const expectedId = hashSmartMtaStorageKey(normalizedKey);
     if (
-      resolvedPath !== this.resolvedRootDir
-      && !resolvedPath.startsWith(`${this.resolvedRootDir}${plugins.path.sep}`)
+      document._id !== expectedId
+      || document.id !== expectedId
+      || document.key !== normalizedKey
+      || (expectedKey && document.key !== expectedKey)
+      || typeof document.value !== 'string'
     ) {
-      throw new Error(`Storage key escapes root directory: ${key}`);
+      throw new Error(`Corrupt SmartMTA storage document: ${document._id}`);
     }
-    return resolvedPath;
-  }
-
-  private toStorageKey(filePath: string): string {
-    const relativePath = plugins.path.relative(this.resolvedRootDir, filePath).split(plugins.path.sep).join('/');
-    return `/${relativePath}`;
+    validateSmartMtaTextValue(document.key, document.value);
+    return document;
   }
 
   public async get(key: string): Promise<string | null> {
-    const filePath = this.resolvePathForKey(key);
-    try {
-      return await plugins.fs.promises.readFile(filePath, 'utf8');
-    } catch (error: unknown) {
-      if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
-        return null;
-      }
-      throw error;
-    }
+    const normalizedKey = normalizeSmartMtaStorageKey(key);
+    getSmartMtaTextNamespace(normalizedKey);
+    const id = hashSmartMtaStorageKey(normalizedKey);
+    const document = await this.collection.findOne({ _id: id });
+    return document ? this.validateDocument(document, normalizedKey).value : null;
   }
 
   public async set(key: string, value: string): Promise<void> {
-    const filePath = this.resolvePathForKey(key);
-    await plugins.fs.promises.mkdir(plugins.path.dirname(filePath), { recursive: true });
-    await plugins.fs.promises.writeFile(filePath, value, 'utf8');
-  }
+    const normalizedKey = normalizeSmartMtaStorageKey(key);
+    validateSmartMtaTextValue(normalizedKey, value);
+    const id = hashSmartMtaStorageKey(normalizedKey);
+    const existing = await this.collection.findOne({ _id: id });
+    if (existing) {
+      this.validateDocument(existing, normalizedKey);
+    }
 
-  public async list(prefix: string): Promise<string[]> {
-    const prefixPath = this.resolvePathForKey(prefix);
-    try {
-      const stat = await plugins.fs.promises.stat(prefixPath);
-      if (stat.isFile()) {
-        return [this.toStorageKey(prefixPath)];
-      }
-    } catch (error: unknown) {
-      if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
-        return [];
-      }
-      throw error;
+    const now = new Date().toISOString();
+    await this.collection.updateOne(
+      { _id: id },
+      {
+        $set: {
+          id,
+          key: normalizedKey,
+          value,
+          updatedAt: now,
+        },
+        $setOnInsert: {
+          createdAt: now,
+        },
+      },
+      { upsert: true },
+    );
+
+    const stored = await this.collection.findOne({ _id: id });
+    if (!stored || this.validateDocument(stored, normalizedKey).value !== value) {
+      throw new Error(`SmartMTA storage write verification failed: ${normalizedKey}`);
     }
+  }
 
-    const results: string[] = [];
-    const walk = async (currentPath: string): Promise<void> => {
-      const entries = await plugins.fs.promises.readdir(currentPath, { withFileTypes: true });
-      for (const entry of entries) {
-        const entryPath = plugins.path.join(currentPath, entry.name);
-        if (entry.isDirectory()) {
-          await walk(entryPath);
-        } else if (entry.isFile()) {
-          results.push(this.toStorageKey(entryPath));
-        }
-      }
-    };
+  public async list(prefix: string): Promise<string[]> {
+    const normalizedPrefix = normalizeSmartMtaStorageKey(prefix);
+    getSmartMtaTextNamespace(normalizedPrefix);
+    const documents = await this.collection.find({
+      key: {
+        $gte: normalizedPrefix,
+        $lt: `${normalizedPrefix}\uffff`,
+      },
+    }).toArray();
 
-    await walk(prefixPath);
-    return results.sort();
+    return documents
+      .map((document: ISmartMtaStorageDocument) => this.validateDocument(document).key)
+      .filter((key: string) => key.startsWith(normalizedPrefix))
+      .sort();
   }
 
   public async delete(key: string): Promise<void> {
-    const targetPath = this.resolvePathForKey(key);
-    try {
-      const stat = await plugins.fs.promises.stat(targetPath);
-      if (stat.isDirectory()) {
-        await plugins.fs.promises.rm(targetPath, { recursive: true, force: true });
-      } else {
-        await plugins.fs.promises.unlink(targetPath);
-      }
-    } catch (error: unknown) {
-      if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
-        return;
-      }
-      throw error;
-    }
-
-    let currentDir = plugins.path.dirname(targetPath);
-    while (currentDir.startsWith(this.resolvedRootDir) && currentDir !== this.resolvedRootDir) {
-      const entries = await plugins.fs.promises.readdir(currentDir);
-      if (entries.length > 0) {
-        break;
-      }
-      await plugins.fs.promises.rmdir(currentDir);
-      currentDir = plugins.path.dirname(currentDir);
+    const normalizedKey = normalizeSmartMtaStorageKey(key);
+    getSmartMtaTextNamespace(normalizedKey);
+    const id = hashSmartMtaStorageKey(normalizedKey);
+    const existing = await this.collection.findOne({ _id: id });
+    if (existing) {
+      this.validateDocument(existing, normalizedKey);
+      await this.collection.deleteOne({ _id: id });
     }
   }
 }

package/ts/email/classes.workapp-mail-manager.ts

@@ -4,10 +4,12 @@ import type {
   IMessageAcceptanceContext,
   IMessageAcceptanceDecision,
   IUnifiedEmailServerOptions,
+  IStorageManagerLike,
 } from '@push.rocks/smartmta';
 import type { Buffer } from 'node:buffer';
 import * as plugins from '../plugins.js';
 import { CachedEmail } from '../db/index.js';
+import type { ICachedEmailSmtpTransaction } from '../db/documents/classes.cached.email.js';
 import type { DcRouter } from '../classes.dcrouter.js';
 import type * as interfaces from '../../ts_interfaces/index.js';
 
@@ -42,6 +44,7 @@ type TSmartMtaStatusQueueItem = {
   attempts?: number;
   nextAttempt?: Date;
   lastError?: string;
+  smtpTransactions?: ICachedEmailSmtpTransaction[];
   processingResult?: {
     headers?: Record<string, string>;
     email?: {
@@ -611,11 +614,20 @@ export class WorkAppMailManager {
     await storageManager.set(this.storageKey, JSON.stringify(state, null, 2));
   }
 
-  private getRequiredStorageManager() {
-    if (!plugins.smartmta.hasStorageManagerMethods(this.dcRouterRef.storageManager, ['get', 'set'])) {
+  private getRequiredStorageManager(): IStorageManagerLike & Required<
+    Pick<IStorageManagerLike, 'get' | 'set'>
+  > {
+    const storageManager = this.dcRouterRef.storageManager;
+    if (
+      !storageManager
+      || typeof storageManager.get !== 'function'
+      || typeof storageManager.set !== 'function'
+    ) {
       throw new Error('DcRouter storage manager does not support managed mail identity persistence');
     }
-    return this.dcRouterRef.storageManager;
+    return storageManager as IStorageManagerLike & Required<
+      Pick<IStorageManagerLike, 'get' | 'set'>
+    >;
   }
 
   private mergeIdentitiesIntoEmailConfig<TConfig extends IUnifiedEmailServerOptions>(
@@ -1044,10 +1056,41 @@ export class WorkAppMailManager {
         createdAt,
         updatedAt,
       },
-      attempts: [],
+      attempts: this.mapDeliveryAttempts(
+        queueItem?.smtpTransactions || cachedEmailArg.smtpTransactions || [],
+        cachedEmailArg.id,
+      ),
     };
   }
 
+  private mapDeliveryAttempts(
+    transactionsArg: ICachedEmailSmtpTransaction[],
+    spoolItemIdArg: string,
+  ): plugins.servezoneInterfaces.data.IMailDeliveryAttempt[] {
+    return transactionsArg.map((transaction) => ({
+      id: transaction.id,
+      spoolItemId: spoolItemIdArg,
+      target: {
+        type: 'mtaRelay',
+        label: transaction.recipientDomain || `${transaction.targetHost}:${transaction.targetPort}`,
+        host: transaction.targetHost,
+        port: transaction.targetPort,
+      },
+      status: transaction.outcome === 'succeeded'
+        ? 'delivered'
+        : transaction.retryable
+          ? 'deferred'
+          : 'failed',
+      startedAt: this.toTimestamp(transaction.startedAt) || 0,
+      completedAt: this.toTimestamp(transaction.completedAt),
+      smtpCode: transaction.smtpCode,
+      response: transaction.recipientResults?.map((result) => {
+        return `${result.recipient}: ${result.responseCode} ${result.accepted ? 'accepted' : 'rejected'}`;
+      }).join('; '),
+      error: transaction.error,
+    }));
+  }
+
   private findQueueItemForCachedEmail(cachedEmailIdArg: string): TSmartMtaStatusQueueItem | undefined {
     const emailServer = this.dcRouterRef.emailServer;
     if (!emailServer?.getQueueItems) {

package/ts/email/index.ts

@@ -2,9 +2,13 @@ export * from './classes.accepted-email-spool.js';
 export * from './classes.email-domain.manager.js';
 export * from './classes.email-route-builder.js';
 export * from './classes.mail-dns-sync.js';
+export * from './mail-dns-txt-policy.js';
+export * from './reconcile-mail-dns-singleton.js';
 export * from './classes.mail-egress-coordinator.js';
 export * from './classes.mail-edge-eligibility.js';
 export * from './classes.email-settings.manager.js';
+export * from './classes.smartmta-blob-storage-manager.js';
 export * from './classes.smartmta-storage-manager.js';
+export * from './smartmta-storage-policy.js';
 export * from './classes.workapp-mail-manager.js';
 export * from './email-dns-records.js';