@serve.zone/dcrouter 34.7.1 → 34.7.2

package/dist_ts_web/00_commitinfo_data.js

@@ -3,7 +3,7 @@
  */
 export const commitinfo = {
     name: '@serve.zone/dcrouter',
-    version: '34.7.1',
+    version: '34.7.2',
     description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 };
 //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vdHNfd2ViLzAwX2NvbW1pdGluZm9fZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7R0FFRztBQUNILE1BQU0sQ0FBQyxNQUFNLFVBQVUsR0FBRztJQUN4QixJQUFJLEVBQUUsc0JBQXNCO0lBQzVCLE9BQU8sRUFBRSxRQUFRO0lBQ2pCLFdBQVcsRUFBRSwwRUFBMEU7Q0FDeEYsQ0FBQSJ9

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@serve.zone/dcrouter",
   "private": false,
-  "version": "34.7.1",
+  "version": "34.7.2",
   "description": "A multifaceted routing service handling mail and SMS delivery functions.",
   "type": "module",
   "bin": {
@@ -45,7 +45,7 @@
     "@push.rocks/lik": "^6.4.6",
     "@push.rocks/projectinfo": "^5.1.0",
     "@push.rocks/qenv": "8.0.0",
-    "@push.rocks/smartacme": "^9.8.1",
+    "@push.rocks/smartacme": "^9.8.3",
     "@push.rocks/smartchallenge": "^1.1.0",
     "@push.rocks/smartdb": "5.7.1",
     "@push.rocks/smartdns": "^11.2.1",

package/readme.md

@@ -75,10 +75,11 @@ dcrouter adopt --controller <https origin> --code <adoption code>
 
 `dcrouter` starts the router only without arguments; from 34.2.0 on it refuses any argument it does not know (exit code 2) and starts nothing. Releases up to 34.1.0 started the router on an unknown argument, so check `dcrouter --version` before running a command a later release added.
 
-`dcrouter migrate` walks the data of a **stopped** dcrouter back to an earlier data version through the migration steps' down handlers; forward migrations happen when dcrouter starts. A dcrouter refuses to start on a data version a newer release wrote, so a rollback runs the newer release's `dcrouter migrate` first. Two downgrades are supported:
+`dcrouter migrate` walks the data of a **stopped** dcrouter back to an earlier data version through the migration steps' down handlers; forward migrations happen when dcrouter starts. A dcrouter refuses to start on a data version a newer release wrote, so a rollback runs the newer release's `dcrouter migrate` first. The data version of this release is `34.7.2`. Three downgrades are supported:
 
+- data version `34.7.2` → `34.5.0`, the data dcrouter 34.5.0 through 34.7.1 read: the `floor-exact-certificate-renew-after` step changes nothing on the way back. On the way up it rounds down the renewal time of exact certificates that were stored with a fraction of a millisecond; the rounded value is valid in every release, so the way back keeps it.
 - data version `34.5.0` → `34.0.0`, the data dcrouter 34.0.0 through 34.4.0 read: the `gateway-route-request-headers` step deletes the request headers gateway clients set on their routes (`headers.request` of every gateway-client route), because those releases would list the values. It logs each route and the field names, never a value. Nothing keeps the values: after upgrading again, each gateway client's next full sync sets them again. Operator routes keep their headers.
-- data version `34.0.0` → `32.6.0`, the data dcrouter 33.0.0 reads, which returns the route DNS ownership the `release-annexed-gateway-route-dns-records` step released. From 34.5.0 data, `--to 32.6.0` runs both steps, newest first; each commits with its own ledger stamp, so a refusal of the second leaves the data at `34.0.0`.
+- data version `34.0.0` → `32.6.0`, the data dcrouter 33.0.0 reads, which returns the route DNS ownership the `release-annexed-gateway-route-dns-records` step released. From 34.7.2 data, `--to 32.6.0` runs all three steps, newest first; each commits with its own ledger stamp, so a refusal of the last leaves the data at `34.0.0`.
 
 ```bash
 # the embedded database's default root is ~/.serve.zone/dcrouter/tsmdb (dbConfig.storagePath overrides it)
@@ -359,6 +360,12 @@ registration. At most 128 access requests enter/wait for the local admission
 boundary. ACME has a separate global cap of 256 unresolved TXT resources.
 Revocation cleanup retains one captured record operation per grant so a full
 client quota cannot prevent cleanup. Maintenance uses bounded batches of 25.
+A revoked grant whose cleanup cannot go on (an unresolved certificate job or DNS
+mutation still holds the hostname, a provider delete is not confirmed, or the
+cleanup fails) is tried again after a wait that starts at 2 seconds and doubles
+up to 5 minutes while the reason stays the same. After the blocking operation
+is resolved, `cleanupState` becomes `completed` once the current wait is over,
+so poll it for more than 5 minutes before treating the cleanup as stuck.
 
 Resolved jobs and request receipts are retained for 30 days, then explicitly
 removed with their capacity counters. Known expired references return `expired`;

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '34.7.1',
+  version: '34.7.2',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }

package/ts/testing/classes.testing-certificate-manager.ts

@@ -1,4 +1,5 @@
 import * as plugins from '../plugins.js';
+import { logger } from '../logger.js';
 import type { DnsManager } from '../dns/manager.dns.js';
 import { CertificateIssuance, testingCertificateNamespace } from '../acme/classes.certificate-issuance.js';
 import { ExactIssuanceStore } from '../acme/classes.exact-issuance-store.js';
@@ -12,6 +13,7 @@ import { testingLimits, type TTestingCertificateJob, type ITestingCertificateMet
   type ITestingCertificateDownload, type ITestingOperationRecovery, type TTestingRecoveryAction,
   type ITestingPageRequest, type ITestingPage, type TTestingErrorCode } from '../../ts_interfaces/data/testing.js';
 import { TestingError } from './errors.js';
+import { testingErrorLogFields } from './error-log-fields.js';
 import { testingCheck as check, testingIdCheck } from './validation.js';
 import { requireTestingInput, testingId, testingKey, testingPage, testingRequestDigest } from './request-validation.js';
 
@@ -268,21 +270,36 @@ export class TestingCertificateManager {
       // that a dispatched CA order did not happen.
       let status: plugins.smartacme.IExactIssuanceInfo | null | undefined;
       try { status = await this.issuance.getExactStatus(testingCertificateNamespace(snapshot.data.grantId), snapshot.data.hostname); } catch { /* Unavailable owner is unknown. */ }
+      const claimed = snapshot;
       await this.dns.runDnsMutationExclusive(async () => {
         const current = await TestingCertificateJobDoc.exact.findStoredOne({ id, 'data.state': 'running', 'data.leaseOwner': leaseOwner });
-        if (!current) return;
         const code = error instanceof TestingError ? error.publicError.code : 'issuance_failed';
-        const unapplied = current.data.evidence === 'intent_persisted' || status === null || status?.status === 'prepared';
+        const unapplied = current?.data.evidence === 'intent_persisted' || status === null || status?.status === 'prepared';
+        const cancelled = unapplied && !['acme_unavailable', 'service_unavailable', 'issuance_failed'].includes(code);
+        const state = status?.status === 'abandoned_unverified' ? 'abandoned_unverified' : unapplied ? 'retry_wait' : 'indeterminate';
+        const outcome = !current ? 'claim_lost' : cancelled ? 'cancelled' : state;
+        const job = current ?? claimed;
+        // One line per failed pass, before the first state write, so a write that throws keeps it. A grant
+        // revoked before anything was ordered cancels its job by design; every other outcome needs a look.
+        // Fields only: never the error, its stack or cause, the status object or the job record.
+        logger.log(outcome === 'cancelled' ? 'info' : 'warn', 'Testing certificate job pass ended with an error', {
+          jobId: job.id, grantId: job.data.grantId, hostname: job.data.hostname, attempt: job.data.attempts,
+          outcome, ...testingErrorLogFields(error),
+          issuanceStatus: status === undefined ? 'unavailable' : status === null ? 'none' : status.status,
+          ...(status ? { issuanceRevision: status.revision } : {}),
+          ...(status?.errorCode ? { issuanceErrorCode: status.errorCode } : {}),
+        });
+        if (!current) return;
         const changed = await this.change(current, (data) => {
           delete data.recoveryAction;
           if (status) { data.issuanceId = status.issuanceId; data.issuanceRevision = status.revision; }
           data.evidence = unapplied ? 'effect_proven_unapplied' : status?.errorCode === 'challenge_cleanup_pending' ? 'cleanup_pending' : 'effect_unacknowledged';
         });
-        if (unapplied && !['acme_unavailable', 'service_unavailable', 'issuance_failed'].includes(code)) {
+        if (cancelled) {
           await this.finish(changed, 'cancelled', undefined, code);
         } else await this.change(changed, (data) => {
-          data.state = status?.status === 'abandoned_unverified' ? 'abandoned_unverified' : unapplied ? 'retry_wait' : 'indeterminate';
-          if (data.state === 'abandoned_unverified') data.evidence = 'abandoned_unverified';
+          data.state = state;
+          if (state === 'abandoned_unverified') data.evidence = 'abandoned_unverified';
           data.errorCode = unapplied ? code : 'indeterminate';
           if (unapplied) data.retryAt = Date.now() + testingLimits.pollAfterMs;
         });

package/ts/testing/classes.testing-dns-cleanup-manager.ts

@@ -1,4 +1,5 @@
 import * as plugins from '../plugins.js';
+import { logger } from '../logger.js';
 import type { DnsManager } from '../dns/manager.dns.js';
 import { ManagedDnsEffect, type IManagedDnsEffect } from '../dns/providers/classes.managed-dns-effect.js';
 import { TestingDomainGrantDoc, type ITestingDomainGrantRecord } from '../db/documents/classes.testing-domain-grant.doc.js';
@@ -13,13 +14,32 @@ import type { ITestingDnsCleanup } from './dns-cleanup.js';
 import { testingLimits, type ITestingOperationRecovery, type ITestingPageRequest, type ITestingPage,
   type TTestingRecoveryAction } from '../../ts_interfaces/data/testing.js';
 import { TestingError } from './errors.js';
+import { testingErrorLogFields } from './error-log-fields.js';
 import { testingCheck as check, testingIdCheck } from './validation.js';
 import { requireTestingInput, testingId, testingPage, testingKey, testingRequestDigest } from './request-validation.js';
 
 type TGrant = plugins.smartdata.TStoredDocument<ITestingDomainGrantRecord>;
 
+const sweepLimit = 25;
+/** The longest wait between two visits to a grant whose cleanup makes no progress. */
+const stallDelayCapMs = 5 * 60 * 1000;
+
+interface ICleanupStall {
+  /** What held the cleanup on the last visit. */
+  key: string;
+  delayMs: number;
+  /** The `cleanupRetryAt` written with it. */
+  retryAt: number;
+}
+
 /** Revocation reuses one grant-owned intent slot; a full client quota cannot prevent cleanup. */
 export class TestingDnsCleanupManager {
+  /**
+   * How this process paces a stalled grant: grant id to the reason and the current wait. Only
+   * `cleanupRetryAt` is stored, so a restart begins the waits again and logs each stall once more.
+   */
+  private stalls = new Map<string, ICleanupStall>();
+
   constructor(private dns: DnsManager, private mutations: TestingDnsManager, private certificates: TestingCertificateManager) {}
 
   private async change(current: TGrant, change: (data: ITestingDomainGrantRecord['data']) => void): Promise<TGrant> {
@@ -36,30 +56,87 @@ export class TestingDnsCleanupManager {
       previous: { ...intended, providerRecordId: operation.providerRecordId! }, siblings: [] };
   }
 
-  private async blocked(grantId: string): Promise<boolean> {
-    return Boolean(await TestingDnsMutationDoc.exact.findStoredOne({ 'data.grantId': grantId, 'data.activeConflictKey': { $exists: true } })
-      || await TestingCertificateJobDoc.exact.findStoredOne({ 'data.grantId': grantId, 'data.activeConflictKey': { $exists: true } }));
+  /** The unresolved operation that still holds the grant's hostname, the lowest id of its kind first. */
+  private async blocker(grantId: string): Promise<{ kind: 'dns' | 'certificate'; operationId: string } | null> {
+    const options = { filter: { 'data.grantId': grantId, 'data.activeConflictKey': { $exists: true } }, sort: { id: 1 as const }, limit: 1 };
+    const [mutation] = await TestingDnsMutationDoc.exact.findStored(options);
+    if (mutation) return { kind: 'dns', operationId: mutation.id };
+    const [job] = await TestingCertificateJobDoc.exact.findStored(options);
+    return job ? { kind: 'certificate', operationId: job.id } : null;
+  }
+
+  /** The grant as the sweep may work on it: revoked, cleanup unfinished and not abandoned by an operator. */
+  private async sweepable(grantId: string): Promise<TGrant | null> {
+    const current = await TestingDomainGrantDoc.exact.findStoredOne({ id: grantId, 'data.state': 'revoked' });
+    return !current || current.data.cleanupState === 'completed' || current.data.cleanupOperation?.phase === 'abandoned_unverified'
+      ? null : current;
+  }
+
+  /**
+   * The one stall rule: a visit that got the cleanup no further defers the grant, and nothing else writes
+   * `cleanupRetryAt`. The same reason as on the last visit doubles the wait, up to the cap. A new reason
+   * starts at the poll interval and is logged once. Between two visits the sweep does not select the grant,
+   * so a stalled grant is written once per wait. `stateChange` goes into that one write.
+   */
+  private async stall(current: TGrant, key: string, context: Record<string, string | number>,
+    stateChange?: (data: ITestingDomainGrantRecord['data']) => void): Promise<void> {
+    const known = this.stalls.get(current.id);
+    const repeated = known?.key === key;
+    const delayMs = repeated ? Math.min(known.delayMs * 2, stallDelayCapMs) : testingLimits.pollAfterMs;
+    const retryAt = Date.now() + delayMs;
+    await this.change(current, (data) => { stateChange?.(data); data.cleanupRetryAt = retryAt; });
+    this.stalls.set(current.id, { key, delayMs, retryAt });
+    if (!repeated) logger.log('warn', 'Testing DNS cleanup of a revoked grant is stalled', {
+      grantId: current.id, hostname: current.data.hostname, ...context });
+  }
+
+  /** One grant under DNS exclusion. It ends finished, a record further, or stalled. */
+  private async visit(grantId: string): Promise<void> {
+    const current = await this.sweepable(grantId);
+    if (!current) { this.stalls.delete(grantId); return; }
+    await this.mutations.cancelForRevokedGrant(current.id);
+    await this.certificates.cancelForRevokedGrant(current.id);
+    const blocker = await this.blocker(current.id);
+    if (blocker) {
+      await this.stall(current, `${blocker.kind}:${blocker.operationId}`,
+        { stalledBy: 'operation', blockingKind: blocker.kind, blockingOperationId: blocker.operationId },
+        (data) => { data.cleanupState = 'indeterminate'; });
+      return;
+    }
+    const advanced = await this.advance(current, false);
+    const operation = advanced.data.cleanupOperation;
+    // Finished, or a record is gone and more remain: nothing is written and the next pass takes the grant.
+    if (!operation) { this.stalls.delete(grantId); return; }
+    await this.stall(advanced, `${operation.recordId}:${operation.phase}:${advanced.data.cleanupState}`,
+      { stalledBy: 'record', recordId: operation.recordId, phase: operation.phase, cleanupState: advanced.data.cleanupState });
+  }
+
+  /** A visit that threw is deferred from a fresh read: it may have written the grant before it threw. */
+  private async deferThrown(grantId: string, error: unknown): Promise<void> {
+    const current = await this.sweepable(grantId);
+    if (!current) { this.stalls.delete(grantId); return; }
+    const fields = testingErrorLogFields(error);
+    await this.stall(current, `error:${fields.errorCode ?? fields.errorName}`, { stalledBy: 'error', ...fields });
   }
 
   /** Bounded maintenance, with no provider retry after an unacknowledged delete. */
   public async sweep(): Promise<void> {
+    const now = Date.now();
     const grants = await TestingDomainGrantDoc.exact.findStored({ filter: {
       'data.state': 'revoked', 'data.cleanupState': { $in: ['pending', 'indeterminate'] },
       'data.cleanupOperation.phase': { $ne: 'abandoned_unverified' },
-      $or: [{ 'data.cleanupRetryAt': { $exists: false } }, { 'data.cleanupRetryAt': { $lte: Date.now() } }],
-    }, sort: { 'data.cleanupRetryAt': 1, id: 1 }, limit: 25 });
+      $or: [{ 'data.cleanupRetryAt': { $exists: false } }, { 'data.cleanupRetryAt': { $lte: now } }],
+    }, sort: { 'data.cleanupRetryAt': 1, id: 1 }, limit: sweepLimit });
+    // A grant an operator abandoned, or one that is gone, leaves the sweep without a visit: its wait is over
+    // and it is not listed. A full page does not list every due grant, so it says nothing about the others.
+    if (grants.length < sweepLimit) {
+      const due = new Set(grants.map((grant) => grant.id));
+      for (const [grantId, stall] of this.stalls) if (stall.retryAt <= now && !due.has(grantId)) this.stalls.delete(grantId);
+    }
     for (const grant of grants) {
-      await this.dns.runDnsMutationExclusive(async () => {
-        let current = await TestingDomainGrantDoc.exact.findStoredOne({ id: grant.id, 'data.state': 'revoked' });
-        if (!current || current.data.cleanupState === 'completed' || current.data.cleanupOperation?.phase === 'abandoned_unverified') return;
-        current = await this.change(current, (data) => { data.cleanupRetryAt = Date.now() + testingLimits.pollAfterMs; });
-        await this.mutations.cancelForRevokedGrant(current.id);
-        await this.certificates.cancelForRevokedGrant(current.id);
-        if (await this.blocked(current.id)) {
-          await this.change(current, (data) => { data.cleanupState = 'indeterminate'; }); return;
-        }
-        await this.advance(current, false);
-      });
+      // One grant that throws must not keep the pass from the grants behind it.
+      try { await this.dns.runDnsMutationExclusive(() => this.visit(grant.id)); }
+      catch (error) { await this.dns.runDnsMutationExclusive(() => this.deferThrown(grant.id, error)); }
     }
   }
 
@@ -117,7 +194,7 @@ export class TestingDnsCleanupManager {
   }
 
   private async complete(current: TGrant): Promise<TGrant> {
-    if (await this.blocked(current.id)) return this.change(current, (data) => { data.cleanupState = 'indeterminate'; });
+    if (await this.blocker(current.id)) return this.change(current, (data) => { data.cleanupState = 'indeterminate'; });
     const session = DcRouterDb.getInstance().getDb().createSession();
     try {
       const result = await session.withTransaction(async (transaction) => {
@@ -139,7 +216,10 @@ export class TestingDnsCleanupManager {
         if (result.status === 'concurrent_change') throw new TestingError('revision_conflict');
         return result.document;
       });
-      if (!result) throw new TestingError('service_unavailable'); return result;
+      if (!result) throw new TestingError('service_unavailable');
+      // Every way to completion ends here, an operator's recheck included.
+      this.stalls.delete(current.id);
+      return result;
     } finally { await session.close(); }
   }
 

package/ts/testing/error-log-fields.ts

@@ -0,0 +1,35 @@
+import { TestingError } from './errors.js';
+
+export interface ITestingErrorLogFields {
+  errorName: string;
+  errorCode?: string | number;
+  errorMessage: string;
+}
+
+const withheldMessage = 'withheld: the message may hold key material or a token';
+/** A PEM armor line, or an unbroken run long enough to be a token, a key or a digest. */
+const secretShape = /-----BEGIN|[A-Za-z0-9+_=-]{40,}/;
+/** Shorter than the run `secretShape` withholds, so a code never carries what a message may not. */
+const codeShape = /^[A-Za-z0-9_.-]{1,39}$/;
+
+/**
+ * What a log line may say about a caught error. A caught error is foreign text: it can quote a provider
+ * response, a certificate or a credential. The logger never gets the error itself, its stack or its cause.
+ */
+export function testingErrorLogFields(error: unknown): ITestingErrorLogFields {
+  // An error's `message` and `name` are whatever its thrower assigned: only a string is text to log.
+  const foreignMessage: unknown = error instanceof Error ? error.message : error;
+  const message = typeof foreignMessage === 'string' ? foreignMessage : '';
+  const foreignName: unknown = error instanceof Error ? error.name : undefined;
+  const name = typeof foreignName === 'string' ? foreignName : typeof error;
+  const code = error instanceof TestingError ? error.publicError.code
+    : typeof error === 'object' && error !== null && 'code' in error ? error.code : undefined;
+  const loggable = typeof code === 'string' ? codeShape.test(code) : Number.isSafeInteger(code);
+  return {
+    errorName: name.slice(0, 64),
+    ...(loggable ? { errorCode: code as string | number } : {}),
+    // The whole message is judged before it is cut: a secret behind the first line still withholds it.
+    errorMessage: secretShape.test(message) ? withheldMessage
+      : message.split(/[\r\n\u2028\u2029]/, 1)[0].replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ').slice(0, 300),
+  };
+}

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '34.7.1',
+  version: '34.7.2',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }