@serve.zone/dcrouter 17.1.3 → 17.1.5

package/dist_ts_web/00_commitinfo_data.js

@@ -3,7 +3,7 @@
  */
 export const commitinfo = {
     name: '@serve.zone/dcrouter',
-    version: '17.1.3',
+    version: '17.1.5',
     description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 };
 //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vdHNfd2ViLzAwX2NvbW1pdGluZm9fZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7R0FFRztBQUNILE1BQU0sQ0FBQyxNQUFNLFVBQVUsR0FBRztJQUN4QixJQUFJLEVBQUUsc0JBQXNCO0lBQzVCLE9BQU8sRUFBRSxRQUFRO0lBQ2pCLFdBQVcsRUFBRSwwRUFBMEU7Q0FDeEYsQ0FBQSJ9

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@serve.zone/dcrouter",
   "private": false,
-  "version": "17.1.3",
+  "version": "17.1.5",
   "description": "A multifaceted routing service handling mail and SMS delivery functions.",
   "type": "module",
   "bin": {
@@ -40,7 +40,7 @@
     "@push.rocks/smartbucket": "^4.6.1",
     "@push.rocks/smartchallenge": "^1.1.0",
     "@push.rocks/smartdata": "^7.3.1",
-    "@push.rocks/smartdb": "^2.13.1",
+    "@push.rocks/smartdb": "^2.13.2",
     "@push.rocks/smartdns": "^7.12.1",
     "@push.rocks/smartfs": "^1.6.0",
     "@push.rocks/smartguard": "^3.1.2",

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '17.1.3',
+  version: '17.1.5',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }

package/ts/classes.dcrouter.ts

@@ -816,10 +816,17 @@ export class DcRouter {
       // Mail DNS reconciliation starts last: it needs persistence, provider
       // state, the SMTP DKIM creator, and live tunnel topology. Earlier events
       // only mark the reconciler dirty and are consumed here.
+      const mailDnsSyncDependencies = [
+        'DnsManager', 'EmailDomainManager', 'EmailServer', 'RemoteIngress',
+      ];
+      if (this.options.dnsNsDomains?.length && this.options.dnsScopes?.length) {
+        mailDnsSyncDependencies.push('DnsServer');
+      }
+
       this.serviceManager.addService(
         new plugins.taskbuffer.Service('MailDnsSync')
           .optional()
-          .dependsOn('DnsManager', 'EmailDomainManager', 'EmailServer', 'RemoteIngress')
+          .dependsOn(...mailDnsSyncDependencies)
           .withStart(async () => {
             await this.mailDnsSync.start();
           })

package/ts/db/documents/classes.ip-intelligence.doc.ts

@@ -1,6 +1,9 @@
 import * as plugins from '../../plugins.js';
 import { DcRouterDb } from '../classes.dcrouter-db.js';
-import type { IIpIntelligenceRecord } from '../../../ts_interfaces/data/security-policy.js';
+import type {
+  IIpIntelligenceRecord,
+  TSecurityBlockRuleMatchMode,
+} from '../../../ts_interfaces/data/security-policy.js';
 
 const getDb = () => DcRouterDb.getInstance().getDb();
 
@@ -72,7 +75,106 @@ export class IpIntelligenceDoc extends plugins.smartdata.SmartDataDbDoc<IpIntell
     return await IpIntelligenceDoc.getInstance({ ipAddress });
   }
 
+  public static async findByAsn(asn: number): Promise<IpIntelligenceDoc[]> {
+    return await IpIntelligenceDoc.getInstances({ asn });
+  }
+
+  public static async findByOrganization(
+    organization: string,
+    matchMode: TSecurityBlockRuleMatchMode,
+  ): Promise<IpIntelligenceDoc[]> {
+    const normalizedOrganization = organization.trim();
+    if (!normalizedOrganization) return [];
+
+    const escapedOrganization = normalizedOrganization.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const regex = matchMode === 'exact'
+      ? `^${escapedOrganization}$`
+      : escapedOrganization;
+
+    return await IpIntelligenceDoc.getInstances({
+      $or: [
+        { asnOrg: { $regex: regex, $options: 'i' } },
+        { registrantOrg: { $regex: regex, $options: 'i' } },
+      ],
+    });
+  }
+
   public static async findAll(): Promise<IpIntelligenceDoc[]> {
     return await IpIntelligenceDoc.getInstances({});
   }
+
+  public static async findRecent(options: {
+    ipAddresses?: string[];
+    limit: number;
+  }): Promise<IpIntelligenceDoc[]> {
+    if (!Number.isSafeInteger(options.limit) || options.limit <= 0) {
+      throw new Error('IpIntelligenceDoc.findRecent requires a positive integer limit');
+    }
+
+    const filter = options.ipAddresses
+      ? { ipAddress: { $in: options.ipAddresses } }
+      : {};
+    const collection = (IpIntelligenceDoc as typeof IpIntelligenceDoc & {
+      collection: plugins.smartdata.SmartdataCollection<IpIntelligenceDoc>;
+    }).collection;
+    const cursor = await collection.getCursor(
+      filter,
+      IpIntelligenceDoc as unknown as typeof plugins.smartdata.SmartDataDbDoc,
+    );
+    cursor.mongodbCursor.sort({ lastSeenAt: -1, _id: -1 });
+    cursor.mongodbCursor.limit(options.limit);
+    try {
+      return await cursor.toArray();
+    } finally {
+      await cursor.close();
+    }
+  }
+
+  public static async countAll(): Promise<number> {
+    return await DcRouterDb.getInstance()
+      .getDb()
+      .mongoDb
+      .collection('IpIntelligenceDoc')
+      .countDocuments({});
+  }
+
+  public static async findOldestIds(options: {
+    lastSeenBefore?: number;
+    limit: number;
+  }): Promise<unknown[]> {
+    if (!Number.isSafeInteger(options.limit) || options.limit <= 0) {
+      throw new Error('IpIntelligenceDoc.findOldestIds requires a positive integer limit');
+    }
+    const collection = DcRouterDb.getInstance()
+      .getDb()
+      .mongoDb
+      .collection('IpIntelligenceDoc');
+    const filter = options.lastSeenBefore === undefined
+      ? {}
+      : { lastSeenAt: { $lt: options.lastSeenBefore } };
+    const cursor = collection
+      .find(filter, { projection: { _id: 1 } })
+      .sort({ lastSeenAt: 1, _id: 1 })
+      .limit(options.limit);
+    try {
+      const rows = await cursor.toArray();
+      return rows.map((row) => row._id);
+    } finally {
+      await cursor.close();
+    }
+  }
+
+  public static async deleteByDocumentIds(documentIds: unknown[]): Promise<number> {
+    if (documentIds.length === 0) return 0;
+    if (documentIds.length > 500 || documentIds.some((id) => id === undefined || id === null)) {
+      throw new Error('IpIntelligenceDoc.deleteByDocumentIds requires 1 to 500 document ids');
+    }
+    const deleteFilter: Record<string, any> = { _id: { $in: documentIds } };
+    const result = await DcRouterDb.getInstance()
+      .getDb()
+      .mongoDb
+      .collection('IpIntelligenceDoc')
+      .deleteMany(deleteFilter);
+    return result.deletedCount;
+  }
 }

package/ts/dns/classes.dns-server-runtime.ts

@@ -150,9 +150,7 @@ export class DnsServerRuntime {
 
       // Hand the DnsServer to DnsManager so DB-backed local records on
       // dcrouter-hosted domains get registered too.
-      if (this.dcRouterRef.dnsManager) {
-        await this.dcRouterRef.dnsManager.attachDnsServer(dnsServer);
-      }
+      await this.attachDnsServer(dnsServer);
     } catch (error) {
       dnsServer.removeAllListeners();
       await dnsServer.stop().catch((stopError: Error) => {
@@ -166,6 +164,14 @@ export class DnsServerRuntime {
     }
   }
 
+  private async attachDnsServer(
+    dnsServer: plugins.smartdns.dnsServerMod.DnsServer,
+  ): Promise<void> {
+    if (!this.dcRouterRef.dnsManager) return;
+    await this.dcRouterRef.dnsManager.attachDnsServer(dnsServer);
+    this.dcRouterRef.mailDnsSync?.requestSync('DNS server attached');
+  }
+
   /**
    * Create the DoH socket handler SmartProxy routes hand TLS sockets to.
    */

package/ts/email/classes.email-domain.manager.ts

@@ -16,6 +16,7 @@ import {
   MAIL_DNS_MAX_RETRY_INTERVAL_MS,
   MAIL_DNS_RETRY_INTERVAL_MS,
 } from './classes.mail-dns-sync.js';
+import { projectMailDnsIntentStatus } from './mail-dns-status.js';
 
 export interface IEmailDomainManagerActionResult {
   success: boolean;
@@ -325,33 +326,39 @@ export class EmailDomainManager {
       type: intent.type,
       name: intent.name,
       value: intent.value,
-      status: intent.status === 'valid'
-        ? 'valid'
-        : intent.status === 'conflict' || intent.status === 'failed'
-          ? 'invalid'
-          : 'missing',
+      status: projectMailDnsIntentStatus(intent),
     }));
   }
 
   public async provisionDnsRecords(id: string): Promise<IEmailDomainManagerActionResult> {
-    const before = new Set(
-      (await DnsRecordDoc.findAll()).filter((record) => record.managedBy === 'mail-dns-reconciler').map((record) => record.id),
-    );
-    await this.dcRouter.mailDnsSync?.sync(`manual provision: ${id}`);
-    const doc = await EmailDomainDoc.findById(id);
+    const current = await EmailDomainDoc.findById(id);
+    if (!current) throw new Error(`Email domain not found: ${id}`);
+    const before = new Set((await DnsRecordDoc.findByDomainId(current.linkedDomainId))
+      .filter((record) => record.managedBy === 'mail-dns-reconciler').map((record) => record.id));
+    const mailDnsSync = this.dcRouter.mailDnsSync;
+    if (!mailDnsSync) throw new Error('MailDnsSync is unavailable');
+    const ticket = await mailDnsSync.syncDomain(id, `manual provision: ${id}`);
+    const doc = ticket.domain;
     if (!doc) throw new Error(`Email domain not found: ${id}`);
-    const provisioned = (await DnsRecordDoc.findAll())
+    const provisioned = ticket.records
       .filter((record) => record.managedBy === 'mail-dns-reconciler' && !before.has(record.id)).length;
-    return { ...this.resultForDoc(doc, !this.hasDnsMutationFailure(doc)), provisioned };
+    return { ...this.resultForDoc(doc, true), provisioned };
   }
 
   public async validateDns(id: string): Promise<IEmailDomainManagerActionResult> {
-    await this.dcRouter.mailDnsSync?.sync(`manual validation: ${id}`);
-    const doc = await EmailDomainDoc.findById(id);
+    const mailDnsSync = this.dcRouter.mailDnsSync;
+    if (!mailDnsSync) throw new Error('MailDnsSync is unavailable');
+    const ticket = await mailDnsSync.syncDomain(id, `manual validation: ${id}`);
+    const doc = ticket.domain;
     if (!doc) throw new Error(`Email domain not found: ${id}`);
-    const records = await this.getRequiredDnsRecords(id);
+    const records = (doc.reconciliation?.intents || []).map((intent) => ({
+      type: intent.type,
+      name: intent.name,
+      value: intent.value,
+      status: projectMailDnsIntentStatus(intent),
+    }));
     return {
-      ...this.resultForDoc(doc, doc.reconciliation?.lifecycleStatus === 'active'),
+      ...this.resultForDoc(doc, true),
       domain: this.docToInterface(doc),
       records,
     };

package/ts/email/classes.mail-dns-sync.ts

@@ -27,14 +27,15 @@ import {
   applyMailDnsVerificationResult,
   reconcileMailDnsSingleton,
 } from './reconcile-mail-dns-singleton.js';
+import { projectMailDnsIntentAggregateStatus } from './mail-dns-status.js';
 
 export const MAIL_DNS_SYNC_CREATED_BY = 'mail-dns-reconciler';
 export const MAIL_DNS_MANAGED_BY = 'mail-dns-reconciler';
 export const MAIL_DNS_RETRY_INTERVAL_MS = 5 * 60_000;
 export const MAIL_DNS_MAX_RETRY_INTERVAL_MS = 60 * 60_000;
 export const MAIL_DNS_PUBLIC_RESOLVER_OPTIONS = {
-  strategy: 'doh' as const,
-  allowDohFallback: false,
+  strategy: 'prefer-udp' as const,
+  allowDohFallback: true,
   timeoutMs: 5_000,
 };
 
@@ -175,7 +176,25 @@ export class SmartDnsMailVerifier implements IMailDnsVerifier {
       };
     }
     if (result.status === 'missing') {
-      return { status: 'missing', reason: result.missingReason };
+      const evidence = result.attempts.map((attempt) => {
+        if (attempt.status === 'missing') {
+          return `${attempt.transport}:missing:${attempt.missingReason}`;
+        }
+        if (attempt.status === 'error') {
+          return `${attempt.transport}:error:${attempt.error.code}`;
+        }
+        return `${attempt.transport}:success`;
+      }).join(', ');
+      const corroboratedMissing = (
+        result.attempts.length === 2
+        && result.attempts[0].transport === 'udp'
+        && result.attempts[0].status === 'missing'
+        && result.attempts[1].transport === 'doh'
+        && result.attempts[1].status === 'missing'
+      );
+      return corroboratedMissing
+        ? { status: 'missing', reason: `${result.missingReason}; attempts=${evidence}` }
+        : { status: 'error', reason: `unconfirmed ${result.missingReason}; attempts=${evidence || 'none'}` };
     }
     return {
       status: 'error',
@@ -190,7 +209,7 @@ export class SmartDnsMailVerifier implements IMailDnsVerifier {
         return { valid: false, outcome: 'lookup-error', reason: `DNS lookup failed: ${query.reason}` };
       }
       if (query.status === 'missing') {
-        return { valid: false, outcome: 'missing', reason: `${intent.type} ${intent.name} is missing` };
+        return { valid: false, outcome: 'missing', reason: `${intent.type} ${intent.name} is missing (${query.reason})` };
       }
       const values = query.values;
       if (intent.type === 'MX') {
@@ -255,7 +274,7 @@ export class SmartDnsMailVerifier implements IMailDnsVerifier {
         return { valid: false, outcome: 'lookup-error', reason: `MX lookup failed: ${query.reason}` };
       }
       if (query.status === 'missing') {
-        return { valid: false, outcome: 'missing', reason: `MX ${intents[0].name} is missing` };
+        return { valid: false, outcome: 'missing', reason: `MX ${intents[0].name} is missing (${query.reason})` };
       }
       return evaluateMailMxValues(intents, query.values, previousValues);
     } catch (error: unknown) {
@@ -319,6 +338,12 @@ interface IPlannedDomain {
   selectorCorrectSigningSupported: boolean;
 }
 
+interface IZoneReconciliationResult {
+  errors: Map<string, IEmailDomainOperationError>;
+  provenProviderZoneIds: Set<string>;
+  readyZoneIds: Set<string>;
+}
+
 /**
  * Sole owner of managed mail DNS. API, topology, timer, and startup events only
  * mark this globally serialized reconciler dirty.
@@ -520,7 +545,7 @@ export class MailDnsSync {
         if (outcome) Object.assign(intent, outcome);
         if (intent.error) errors.push(intent.error);
       }
-      await this.validateAndActivate(plan, errors, zoneResult.errors);
+      await this.validateAndActivate(plan, errors, zoneResult, zones);
     }
 
     const cleanupProof = await this.refreshProviderZonesForCleanup(
@@ -900,12 +925,10 @@ export class MailDnsSync {
   private async reconcileZones(
     intents: IEmailDnsRecordIntent[],
     zones: DomainDoc[],
-  ): Promise<{
-    errors: Map<string, IEmailDomainOperationError>;
-    provenProviderZoneIds: Set<string>;
-  }> {
+  ): Promise<IZoneReconciliationResult> {
     const errors = new Map<string, IEmailDomainOperationError>();
     const provenProviderZoneIds = new Set<string>();
+    const readyZoneIds = new Set<string>();
     const byZone = new Map<string, IEmailDnsRecordIntent[]>();
     for (const intent of intents) {
       if (!intent.domainId || intent.error) continue;
@@ -921,6 +944,14 @@ export class MailDnsSync {
           const result = await this.dcRouterRef.dnsManager!.syncDomain(zone.id);
           if (!result.success) throw new Error(result.message || `Failed to refresh ${zone.name}`);
           provenProviderZoneIds.add(zone.id);
+          readyZoneIds.add(zone.id);
+        }
+        if (
+          zone.source === 'dcrouter'
+          && this.dcRouterRef.dnsServer
+          && this.dcRouterRef.dnsManager?.dnsServer === this.dcRouterRef.dnsServer
+        ) {
+          readyZoneIds.add(zone.id);
         }
         for (const intent of this.sortIntents(zoneIntents)) {
           await this.reconcileIntent(intent, zoneIntents);
@@ -939,7 +970,7 @@ export class MailDnsSync {
         }
       }
     }
-    return { errors, provenProviderZoneIds };
+    return { errors, provenProviderZoneIds, readyZoneIds };
   }
 
   private sortIntents(intents: IEmailDnsRecordIntent[]): IEmailDnsRecordIntent[] {
@@ -1104,10 +1135,29 @@ export class MailDnsSync {
   private async validateAndActivate(
     plan: IPlannedDomain,
     errors: IEmailDomainOperationError[],
-    zoneErrors: Map<string, IEmailDomainOperationError>,
+    zoneResult: IZoneReconciliationResult,
+    zones: DomainDoc[],
   ): Promise<void> {
+    for (const intent of plan.intents) {
+      if (intent.error || zoneResult.readyZoneIds.has(intent.domainId)) continue;
+      const zone = zones.find((candidate) => candidate.id === intent.domainId);
+      const providerManaged = zone?.source === 'provider';
+      const outcome: IEmailDomainOperationError = {
+        code: providerManaged ? 'DNS_PROVIDER_ZONE_UNPROVEN' : 'DNS_SERVER_NOT_READY',
+        message: providerManaged
+          ? `DNS provider zone ${zone?.name || intent.domainId} was not proven by this reconciliation pass`
+          : `Authoritative DNS runtime is not ready for ${zone?.name || intent.domainId}`,
+        retryable: true,
+      };
+      intent.status = 'pending';
+      intent.verificationOutcome = 'lookup-error';
+      intent.error = { ...outcome, recordKey: intent.key };
+      if (!errors.some((error) => error.code === outcome.code)) errors.push(outcome);
+    }
     const verifiable = plan.intents.filter((intent) => (
-      !intent.error && !zoneErrors.has(intent.domainId)
+      !intent.error
+      && !zoneResult.errors.has(intent.domainId)
+      && zoneResult.readyZoneIds.has(intent.domainId)
     ));
     const markResult = (intent: IEmailDnsRecordIntent, result: IMailDnsVerificationResult) => {
       const outcome = result.outcome || (result.valid ? 'valid' : 'propagation');
@@ -1265,13 +1315,13 @@ export class MailDnsSync {
       && plan.revision.mxSlots.length > 0;
     plan.doc.dnsStatus = {
       mx: mailPublicationReady
-        ? this.statusForIntent(plan.intents, (intent) => intent.type === 'MX')
+        ? projectMailDnsIntentAggregateStatus(plan.intents, (intent) => intent.type === 'MX')
         : 'unchecked',
       spf: mailPublicationReady
-        ? this.statusForIntent(plan.intents, (intent) => intent.type === 'TXT' && intent.value.startsWith('v=spf1'))
+        ? projectMailDnsIntentAggregateStatus(plan.intents, (intent) => intent.type === 'TXT' && intent.value.startsWith('v=spf1'))
         : 'unchecked',
-      dkim: this.statusForIntent(plan.intents, (intent) => intent.type === 'TXT' && intent.name.includes('._domainkey.')),
-      dmarc: this.statusForIntent(plan.intents, (intent) => intent.type === 'TXT' && intent.value.startsWith('v=DMARC1')),
+      dkim: projectMailDnsIntentAggregateStatus(plan.intents, (intent) => intent.type === 'TXT' && intent.name.includes('._domainkey.')),
+      dmarc: projectMailDnsIntentAggregateStatus(plan.intents, (intent) => intent.type === 'TXT' && intent.value.startsWith('v=DMARC1')),
       lastCheckedAt: now,
     };
     plan.doc.updatedAt = now;
@@ -1307,26 +1357,6 @@ export class MailDnsSync {
     doc.dkim.lastRotatedAt = now;
   }
 
-  private statusForIntent(
-    intents: IEmailDnsRecordIntent[],
-    predicate: (intent: IEmailDnsRecordIntent) => boolean,
-  ): 'valid' | 'missing' | 'invalid' | 'unchecked' {
-    const selected = intents.filter(predicate);
-    if (selected.length === 0) return 'missing';
-    if (selected.every((intent) => intent.status === 'valid')) return 'valid';
-    if (selected.some((intent) => (
-      intent.verificationOutcome === 'duplicate'
-      || intent.verificationOutcome === 'conflict'
-      || intent.verificationOutcome === 'invalid'
-      || intent.status === 'conflict'
-      || intent.status === 'failed'
-    ))) return 'invalid';
-    if (selected.some((intent) => intent.verificationOutcome === 'lookup-error')) {
-      return 'unchecked';
-    }
-    return 'missing';
-  }
-
   private async refreshProviderZonesForCleanup(
     docs: EmailDomainDoc[],
     zones: DomainDoc[],

package/ts/email/index.ts

@@ -3,6 +3,7 @@ export * from './classes.email-domain.manager.js';
 export * from './classes.email-route-builder.js';
 export * from './classes.mail-dns-sync.js';
 export * from './mail-dns-txt-policy.js';
+export * from './mail-dns-status.js';
 export * from './reconcile-mail-dns-singleton.js';
 export * from './classes.mail-egress-coordinator.js';
 export * from './classes.mail-edge-eligibility.js';

package/ts/email/mail-dns-status.ts

@@ -0,0 +1,35 @@
+import type {
+  IEmailDnsRecordIntent,
+  TDnsRecordStatus,
+} from '../../ts_interfaces/data/email-domain.js';
+
+/** Project one durable reconciliation intent into the operator-facing DNS status. */
+export function projectMailDnsIntentStatus(
+  intent: IEmailDnsRecordIntent | undefined,
+): TDnsRecordStatus {
+  if (!intent) return 'unchecked';
+  if (
+    intent.verificationOutcome === 'duplicate'
+    || intent.verificationOutcome === 'conflict'
+    || intent.verificationOutcome === 'invalid'
+    || intent.status === 'conflict'
+    || intent.status === 'failed'
+  ) return 'invalid';
+  if (intent.verificationOutcome === 'missing') return 'missing';
+  if (intent.verificationOutcome === 'valid' && intent.status === 'valid') return 'valid';
+  return 'unchecked';
+}
+
+/** Aggregate matching intents with deterministic operator-facing precedence. */
+export function projectMailDnsIntentAggregateStatus(
+  intents: IEmailDnsRecordIntent[],
+  predicate: (intent: IEmailDnsRecordIntent) => boolean,
+): TDnsRecordStatus {
+  const selected = intents.filter(predicate);
+  if (selected.length === 0) return 'unchecked';
+  const statuses = selected.map(projectMailDnsIntentStatus);
+  if (statuses.includes('invalid')) return 'invalid';
+  if (statuses.includes('missing')) return 'missing';
+  if (statuses.includes('unchecked')) return 'unchecked';
+  return 'valid';
+}