@serve.zone/dcrouter 34.7.0 → 34.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.dcrouter.d.ts +7 -1
- package/dist_ts/classes.dcrouter.js +1 -1
- package/dist_ts/email/classes.coremail-gateway.manager.js +11 -2
- package/dist_ts/testing/classes.testing-certificate-manager.js +23 -6
- package/dist_ts/testing/classes.testing-dns-cleanup-manager.d.ts +20 -1
- package/dist_ts/testing/classes.testing-dns-cleanup-manager.js +97 -20
- package/dist_ts/testing/error-log-fields.d.ts +10 -0
- package/dist_ts/testing/error-log-fields.js +28 -0
- package/dist_ts_migrations/index.d.ts +1 -1
- package/dist_ts_migrations/index.js +59 -3
- package/dist_ts_web/00_commitinfo_data.js +1 -1
- package/package.json +2 -2
- package/readme.md +10 -3
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.dcrouter.ts +7 -1
- package/ts/email/classes.coremail-gateway.manager.ts +15 -1
- package/ts/testing/classes.testing-certificate-manager.ts +22 -5
- package/ts/testing/classes.testing-dns-cleanup-manager.ts +98 -18
- package/ts/testing/error-log-fields.ts +35 -0
- package/ts_web/00_commitinfo_data.ts +1 -1
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export const commitinfo = {
|
|
5
5
|
name: '@serve.zone/dcrouter',
|
|
6
|
-
version: '34.7.
|
|
6
|
+
version: '34.7.2',
|
|
7
7
|
description: 'A multifaceted routing service handling mail and SMS delivery functions.'
|
|
8
8
|
};
|
|
9
9
|
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vdHNfd2ViLzAwX2NvbW1pdGluZm9fZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7R0FFRztBQUNILE1BQU0sQ0FBQyxNQUFNLFVBQVUsR0FBRztJQUN4QixJQUFJLEVBQUUsc0JBQXNCO0lBQzVCLE9BQU8sRUFBRSxRQUFRO0lBQ2pCLFdBQVcsRUFBRSwwRUFBMEU7Q0FDeEYsQ0FBQSJ9
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@serve.zone/dcrouter",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "34.7.
|
|
4
|
+
"version": "34.7.2",
|
|
5
5
|
"description": "A multifaceted routing service handling mail and SMS delivery functions.",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"bin": {
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"@push.rocks/lik": "^6.4.6",
|
|
46
46
|
"@push.rocks/projectinfo": "^5.1.0",
|
|
47
47
|
"@push.rocks/qenv": "8.0.0",
|
|
48
|
-
"@push.rocks/smartacme": "^9.8.
|
|
48
|
+
"@push.rocks/smartacme": "^9.8.3",
|
|
49
49
|
"@push.rocks/smartchallenge": "^1.1.0",
|
|
50
50
|
"@push.rocks/smartdb": "5.7.1",
|
|
51
51
|
"@push.rocks/smartdns": "^11.2.1",
|
package/readme.md
CHANGED
|
@@ -75,10 +75,11 @@ dcrouter adopt --controller <https origin> --code <adoption code>
|
|
|
75
75
|
|
|
76
76
|
`dcrouter` starts the router only without arguments; from 34.2.0 on it refuses any argument it does not know (exit code 2) and starts nothing. Releases up to 34.1.0 started the router on an unknown argument, so check `dcrouter --version` before running a command a later release added.
|
|
77
77
|
|
|
78
|
-
`dcrouter migrate` walks the data of a **stopped** dcrouter back to an earlier data version through the migration steps' down handlers; forward migrations happen when dcrouter starts. A dcrouter refuses to start on a data version a newer release wrote, so a rollback runs the newer release's `dcrouter migrate` first.
|
|
78
|
+
`dcrouter migrate` walks the data of a **stopped** dcrouter back to an earlier data version through the migration steps' down handlers; forward migrations happen when dcrouter starts. A dcrouter refuses to start on a data version a newer release wrote, so a rollback runs the newer release's `dcrouter migrate` first. The data version of this release is `34.7.2`. Three downgrades are supported:
|
|
79
79
|
|
|
80
|
+
- data version `34.7.2` → `34.5.0`, the data dcrouter 34.5.0 through 34.7.1 read: the `floor-exact-certificate-renew-after` step changes nothing on the way back. On the way up it rounds down the renewal time of exact certificates that were stored with a fraction of a millisecond; the rounded value is valid in every release, so the way back keeps it.
|
|
80
81
|
- data version `34.5.0` → `34.0.0`, the data dcrouter 34.0.0 through 34.4.0 read: the `gateway-route-request-headers` step deletes the request headers gateway clients set on their routes (`headers.request` of every gateway-client route), because those releases would list the values. It logs each route and the field names, never a value. Nothing keeps the values: after upgrading again, each gateway client's next full sync sets them again. Operator routes keep their headers.
|
|
81
|
-
- data version `34.0.0` → `32.6.0`, the data dcrouter 33.0.0 reads, which returns the route DNS ownership the `release-annexed-gateway-route-dns-records` step released. From 34.
|
|
82
|
+
- data version `34.0.0` → `32.6.0`, the data dcrouter 33.0.0 reads, which returns the route DNS ownership the `release-annexed-gateway-route-dns-records` step released. From 34.7.2 data, `--to 32.6.0` runs all three steps, newest first; each commits with its own ledger stamp, so a refusal of the last leaves the data at `34.0.0`.
|
|
82
83
|
|
|
83
84
|
```bash
|
|
84
85
|
# the embedded database's default root is ~/.serve.zone/dcrouter/tsmdb (dbConfig.storagePath overrides it)
|
|
@@ -161,7 +162,7 @@ Bootstrap behavior:
|
|
|
161
162
|
| `baseDir` | Root directory for dcrouter runtime data. Defaults to `~/.serve.zone/dcrouter`. |
|
|
162
163
|
| `coreTrafficConfig` | Main CoreTraffic route configuration for HTTP/HTTPS/TCP/SNI traffic. `smartProxyConfig` remains accepted as a legacy alias. |
|
|
163
164
|
| `emailConfig` | UnifiedEmailServer configuration: hostname, ports, domains, and mail routes. |
|
|
164
|
-
| `emailOutboundMode` | Outbound SMTP mode
|
|
165
|
+
| `emailOutboundMode` | Outbound SMTP mode; explicit `direct` mode lets unmanaged/static senders dial from the hub. With the database enabled it seeds the stored email settings on first start only; after that the stored settings are authoritative, they default to `remoteIngress`, and the mode changes through the email settings API or Ops UI. Without the database it is read directly on every delivery and has no default: while it is unset, outbound mail defers. |
|
|
165
166
|
| `emailPortConfig` | External-to-internal email port mapping and received-email storage path. |
|
|
166
167
|
| `tls` | Legacy/static TLS and ACME contact settings used to seed certificate config. |
|
|
167
168
|
| `dnsNsDomains` | Nameserver hostnames used for generated NS records and DoH routes. A zone becomes authoritative by having its public NS records observed naming one of these. |
|
|
@@ -359,6 +360,12 @@ registration. At most 128 access requests enter/wait for the local admission
|
|
|
359
360
|
boundary. ACME has a separate global cap of 256 unresolved TXT resources.
|
|
360
361
|
Revocation cleanup retains one captured record operation per grant so a full
|
|
361
362
|
client quota cannot prevent cleanup. Maintenance uses bounded batches of 25.
|
|
363
|
+
A revoked grant whose cleanup cannot go on (an unresolved certificate job or DNS
|
|
364
|
+
mutation still holds the hostname, a provider delete is not confirmed, or the
|
|
365
|
+
cleanup fails) is tried again after a wait that starts at 2 seconds and doubles
|
|
366
|
+
up to 5 minutes while the reason stays the same. After the blocking operation
|
|
367
|
+
is resolved, `cleanupState` becomes `completed` once the current wait is over,
|
|
368
|
+
so poll it for more than 5 minutes before treating the cleanup as stuck.
|
|
362
369
|
|
|
363
370
|
Resolved jobs and request receipts are retained for 30 days, then explicitly
|
|
364
371
|
removed with their capacity counters. Known expired references return `expired`;
|
package/ts/00_commitinfo_data.ts
CHANGED
package/ts/classes.dcrouter.ts
CHANGED
|
@@ -116,7 +116,13 @@ export interface IDcRouterOptions {
|
|
|
116
116
|
/** SmartBucket configuration for durable SmartMTA queue and attachment blobs. */
|
|
117
117
|
emailBlobStorage?: ISmartMtaBlobStorageConfig;
|
|
118
118
|
|
|
119
|
-
/**
|
|
119
|
+
/**
|
|
120
|
+
* Outbound SMTP connection mode. With the database enabled it seeds the
|
|
121
|
+
* stored email settings on first start only; after that the stored settings
|
|
122
|
+
* are authoritative, and they default to `remoteIngress`. Without the
|
|
123
|
+
* database it is read directly on every delivery and has no default: while
|
|
124
|
+
* it is unset, outbound mail defers.
|
|
125
|
+
*/
|
|
120
126
|
emailOutboundMode?: TEmailOutboundMode;
|
|
121
127
|
|
|
122
128
|
/**
|
|
@@ -149,6 +149,20 @@ const contractError = (
|
|
|
149
149
|
) satisfies plugins.servezoneInterfaces.data.ICoreMailErrorData,
|
|
150
150
|
);
|
|
151
151
|
|
|
152
|
+
/**
|
|
153
|
+
* Whether a replayed envelope is the one the stored handoff was prepared with.
|
|
154
|
+
* Compares values, never encodings: the stored envelope is whatever the
|
|
155
|
+
* document store returns, and its key order is not part of the data. The
|
|
156
|
+
* recipient order is, as it is in the submission digest.
|
|
157
|
+
*/
|
|
158
|
+
const isSameCoreMailEnvelope = (
|
|
159
|
+
storedArg: plugins.servezoneInterfaces.data.ICoreMailEnvelope,
|
|
160
|
+
replayedArg: plugins.servezoneInterfaces.data.ICoreMailEnvelope,
|
|
161
|
+
): boolean =>
|
|
162
|
+
storedArg.mailFrom === replayedArg.mailFrom
|
|
163
|
+
&& storedArg.rcptTo.length === replayedArg.rcptTo.length
|
|
164
|
+
&& storedArg.rcptTo.every((recipientArg, indexArg) => recipientArg === replayedArg.rcptTo[indexArg]);
|
|
165
|
+
|
|
152
166
|
/**
|
|
153
167
|
* Answers a wire-9 request with the handler's own contract refusal: TypedRequest 9
|
|
154
168
|
* answers an error of another class, TypedRequest 8's `TypedResponseError`
|
|
@@ -580,7 +594,7 @@ export class CoreMailGatewayManager {
|
|
|
580
594
|
|| existing.submissionDigest !== submissionDigest
|
|
581
595
|
|| existing.rawMimeSha256 !== message.rawMime.sha256
|
|
582
596
|
|| existing.rawMimeLengthBytes !== message.rawMime.lengthBytes
|
|
583
|
-
||
|
|
597
|
+
|| !isSameCoreMailEnvelope(existing.envelope, message.envelope)
|
|
584
598
|
) {
|
|
585
599
|
throw contractError('IDEMPOTENCY_CONFLICT');
|
|
586
600
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as plugins from '../plugins.js';
|
|
2
|
+
import { logger } from '../logger.js';
|
|
2
3
|
import type { DnsManager } from '../dns/manager.dns.js';
|
|
3
4
|
import { CertificateIssuance, testingCertificateNamespace } from '../acme/classes.certificate-issuance.js';
|
|
4
5
|
import { ExactIssuanceStore } from '../acme/classes.exact-issuance-store.js';
|
|
@@ -12,6 +13,7 @@ import { testingLimits, type TTestingCertificateJob, type ITestingCertificateMet
|
|
|
12
13
|
type ITestingCertificateDownload, type ITestingOperationRecovery, type TTestingRecoveryAction,
|
|
13
14
|
type ITestingPageRequest, type ITestingPage, type TTestingErrorCode } from '../../ts_interfaces/data/testing.js';
|
|
14
15
|
import { TestingError } from './errors.js';
|
|
16
|
+
import { testingErrorLogFields } from './error-log-fields.js';
|
|
15
17
|
import { testingCheck as check, testingIdCheck } from './validation.js';
|
|
16
18
|
import { requireTestingInput, testingId, testingKey, testingPage, testingRequestDigest } from './request-validation.js';
|
|
17
19
|
|
|
@@ -268,21 +270,36 @@ export class TestingCertificateManager {
|
|
|
268
270
|
// that a dispatched CA order did not happen.
|
|
269
271
|
let status: plugins.smartacme.IExactIssuanceInfo | null | undefined;
|
|
270
272
|
try { status = await this.issuance.getExactStatus(testingCertificateNamespace(snapshot.data.grantId), snapshot.data.hostname); } catch { /* Unavailable owner is unknown. */ }
|
|
273
|
+
const claimed = snapshot;
|
|
271
274
|
await this.dns.runDnsMutationExclusive(async () => {
|
|
272
275
|
const current = await TestingCertificateJobDoc.exact.findStoredOne({ id, 'data.state': 'running', 'data.leaseOwner': leaseOwner });
|
|
273
|
-
if (!current) return;
|
|
274
276
|
const code = error instanceof TestingError ? error.publicError.code : 'issuance_failed';
|
|
275
|
-
const unapplied = current
|
|
277
|
+
const unapplied = current?.data.evidence === 'intent_persisted' || status === null || status?.status === 'prepared';
|
|
278
|
+
const cancelled = unapplied && !['acme_unavailable', 'service_unavailable', 'issuance_failed'].includes(code);
|
|
279
|
+
const state = status?.status === 'abandoned_unverified' ? 'abandoned_unverified' : unapplied ? 'retry_wait' : 'indeterminate';
|
|
280
|
+
const outcome = !current ? 'claim_lost' : cancelled ? 'cancelled' : state;
|
|
281
|
+
const job = current ?? claimed;
|
|
282
|
+
// One line per failed pass, before the first state write, so a write that throws keeps it. A grant
|
|
283
|
+
// revoked before anything was ordered cancels its job by design; every other outcome needs a look.
|
|
284
|
+
// Fields only: never the error, its stack or cause, the status object or the job record.
|
|
285
|
+
logger.log(outcome === 'cancelled' ? 'info' : 'warn', 'Testing certificate job pass ended with an error', {
|
|
286
|
+
jobId: job.id, grantId: job.data.grantId, hostname: job.data.hostname, attempt: job.data.attempts,
|
|
287
|
+
outcome, ...testingErrorLogFields(error),
|
|
288
|
+
issuanceStatus: status === undefined ? 'unavailable' : status === null ? 'none' : status.status,
|
|
289
|
+
...(status ? { issuanceRevision: status.revision } : {}),
|
|
290
|
+
...(status?.errorCode ? { issuanceErrorCode: status.errorCode } : {}),
|
|
291
|
+
});
|
|
292
|
+
if (!current) return;
|
|
276
293
|
const changed = await this.change(current, (data) => {
|
|
277
294
|
delete data.recoveryAction;
|
|
278
295
|
if (status) { data.issuanceId = status.issuanceId; data.issuanceRevision = status.revision; }
|
|
279
296
|
data.evidence = unapplied ? 'effect_proven_unapplied' : status?.errorCode === 'challenge_cleanup_pending' ? 'cleanup_pending' : 'effect_unacknowledged';
|
|
280
297
|
});
|
|
281
|
-
if (
|
|
298
|
+
if (cancelled) {
|
|
282
299
|
await this.finish(changed, 'cancelled', undefined, code);
|
|
283
300
|
} else await this.change(changed, (data) => {
|
|
284
|
-
data.state =
|
|
285
|
-
if (
|
|
301
|
+
data.state = state;
|
|
302
|
+
if (state === 'abandoned_unverified') data.evidence = 'abandoned_unverified';
|
|
286
303
|
data.errorCode = unapplied ? code : 'indeterminate';
|
|
287
304
|
if (unapplied) data.retryAt = Date.now() + testingLimits.pollAfterMs;
|
|
288
305
|
});
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as plugins from '../plugins.js';
|
|
2
|
+
import { logger } from '../logger.js';
|
|
2
3
|
import type { DnsManager } from '../dns/manager.dns.js';
|
|
3
4
|
import { ManagedDnsEffect, type IManagedDnsEffect } from '../dns/providers/classes.managed-dns-effect.js';
|
|
4
5
|
import { TestingDomainGrantDoc, type ITestingDomainGrantRecord } from '../db/documents/classes.testing-domain-grant.doc.js';
|
|
@@ -13,13 +14,32 @@ import type { ITestingDnsCleanup } from './dns-cleanup.js';
|
|
|
13
14
|
import { testingLimits, type ITestingOperationRecovery, type ITestingPageRequest, type ITestingPage,
|
|
14
15
|
type TTestingRecoveryAction } from '../../ts_interfaces/data/testing.js';
|
|
15
16
|
import { TestingError } from './errors.js';
|
|
17
|
+
import { testingErrorLogFields } from './error-log-fields.js';
|
|
16
18
|
import { testingCheck as check, testingIdCheck } from './validation.js';
|
|
17
19
|
import { requireTestingInput, testingId, testingPage, testingKey, testingRequestDigest } from './request-validation.js';
|
|
18
20
|
|
|
19
21
|
type TGrant = plugins.smartdata.TStoredDocument<ITestingDomainGrantRecord>;
|
|
20
22
|
|
|
23
|
+
const sweepLimit = 25;
|
|
24
|
+
/** The longest wait between two visits to a grant whose cleanup makes no progress. */
|
|
25
|
+
const stallDelayCapMs = 5 * 60 * 1000;
|
|
26
|
+
|
|
27
|
+
interface ICleanupStall {
|
|
28
|
+
/** What held the cleanup on the last visit. */
|
|
29
|
+
key: string;
|
|
30
|
+
delayMs: number;
|
|
31
|
+
/** The `cleanupRetryAt` written with it. */
|
|
32
|
+
retryAt: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
21
35
|
/** Revocation reuses one grant-owned intent slot; a full client quota cannot prevent cleanup. */
|
|
22
36
|
export class TestingDnsCleanupManager {
|
|
37
|
+
/**
|
|
38
|
+
* How this process paces a stalled grant: grant id to the reason and the current wait. Only
|
|
39
|
+
* `cleanupRetryAt` is stored, so a restart begins the waits again and logs each stall once more.
|
|
40
|
+
*/
|
|
41
|
+
private stalls = new Map<string, ICleanupStall>();
|
|
42
|
+
|
|
23
43
|
constructor(private dns: DnsManager, private mutations: TestingDnsManager, private certificates: TestingCertificateManager) {}
|
|
24
44
|
|
|
25
45
|
private async change(current: TGrant, change: (data: ITestingDomainGrantRecord['data']) => void): Promise<TGrant> {
|
|
@@ -36,30 +56,87 @@ export class TestingDnsCleanupManager {
|
|
|
36
56
|
previous: { ...intended, providerRecordId: operation.providerRecordId! }, siblings: [] };
|
|
37
57
|
}
|
|
38
58
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
59
|
+
/** The unresolved operation that still holds the grant's hostname, the lowest id of its kind first. */
|
|
60
|
+
private async blocker(grantId: string): Promise<{ kind: 'dns' | 'certificate'; operationId: string } | null> {
|
|
61
|
+
const options = { filter: { 'data.grantId': grantId, 'data.activeConflictKey': { $exists: true } }, sort: { id: 1 as const }, limit: 1 };
|
|
62
|
+
const [mutation] = await TestingDnsMutationDoc.exact.findStored(options);
|
|
63
|
+
if (mutation) return { kind: 'dns', operationId: mutation.id };
|
|
64
|
+
const [job] = await TestingCertificateJobDoc.exact.findStored(options);
|
|
65
|
+
return job ? { kind: 'certificate', operationId: job.id } : null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The grant as the sweep may work on it: revoked, cleanup unfinished and not abandoned by an operator. */
|
|
69
|
+
private async sweepable(grantId: string): Promise<TGrant | null> {
|
|
70
|
+
const current = await TestingDomainGrantDoc.exact.findStoredOne({ id: grantId, 'data.state': 'revoked' });
|
|
71
|
+
return !current || current.data.cleanupState === 'completed' || current.data.cleanupOperation?.phase === 'abandoned_unverified'
|
|
72
|
+
? null : current;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The one stall rule: a visit that got the cleanup no further defers the grant, and nothing else writes
|
|
77
|
+
* `cleanupRetryAt`. The same reason as on the last visit doubles the wait, up to the cap. A new reason
|
|
78
|
+
* starts at the poll interval and is logged once. Between two visits the sweep does not select the grant,
|
|
79
|
+
* so a stalled grant is written once per wait. `stateChange` goes into that one write.
|
|
80
|
+
*/
|
|
81
|
+
private async stall(current: TGrant, key: string, context: Record<string, string | number>,
|
|
82
|
+
stateChange?: (data: ITestingDomainGrantRecord['data']) => void): Promise<void> {
|
|
83
|
+
const known = this.stalls.get(current.id);
|
|
84
|
+
const repeated = known?.key === key;
|
|
85
|
+
const delayMs = repeated ? Math.min(known.delayMs * 2, stallDelayCapMs) : testingLimits.pollAfterMs;
|
|
86
|
+
const retryAt = Date.now() + delayMs;
|
|
87
|
+
await this.change(current, (data) => { stateChange?.(data); data.cleanupRetryAt = retryAt; });
|
|
88
|
+
this.stalls.set(current.id, { key, delayMs, retryAt });
|
|
89
|
+
if (!repeated) logger.log('warn', 'Testing DNS cleanup of a revoked grant is stalled', {
|
|
90
|
+
grantId: current.id, hostname: current.data.hostname, ...context });
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** One grant under DNS exclusion. It ends finished, a record further, or stalled. */
|
|
94
|
+
private async visit(grantId: string): Promise<void> {
|
|
95
|
+
const current = await this.sweepable(grantId);
|
|
96
|
+
if (!current) { this.stalls.delete(grantId); return; }
|
|
97
|
+
await this.mutations.cancelForRevokedGrant(current.id);
|
|
98
|
+
await this.certificates.cancelForRevokedGrant(current.id);
|
|
99
|
+
const blocker = await this.blocker(current.id);
|
|
100
|
+
if (blocker) {
|
|
101
|
+
await this.stall(current, `${blocker.kind}:${blocker.operationId}`,
|
|
102
|
+
{ stalledBy: 'operation', blockingKind: blocker.kind, blockingOperationId: blocker.operationId },
|
|
103
|
+
(data) => { data.cleanupState = 'indeterminate'; });
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const advanced = await this.advance(current, false);
|
|
107
|
+
const operation = advanced.data.cleanupOperation;
|
|
108
|
+
// Finished, or a record is gone and more remain: nothing is written and the next pass takes the grant.
|
|
109
|
+
if (!operation) { this.stalls.delete(grantId); return; }
|
|
110
|
+
await this.stall(advanced, `${operation.recordId}:${operation.phase}:${advanced.data.cleanupState}`,
|
|
111
|
+
{ stalledBy: 'record', recordId: operation.recordId, phase: operation.phase, cleanupState: advanced.data.cleanupState });
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** A visit that threw is deferred from a fresh read: it may have written the grant before it threw. */
|
|
115
|
+
private async deferThrown(grantId: string, error: unknown): Promise<void> {
|
|
116
|
+
const current = await this.sweepable(grantId);
|
|
117
|
+
if (!current) { this.stalls.delete(grantId); return; }
|
|
118
|
+
const fields = testingErrorLogFields(error);
|
|
119
|
+
await this.stall(current, `error:${fields.errorCode ?? fields.errorName}`, { stalledBy: 'error', ...fields });
|
|
42
120
|
}
|
|
43
121
|
|
|
44
122
|
/** Bounded maintenance, with no provider retry after an unacknowledged delete. */
|
|
45
123
|
public async sweep(): Promise<void> {
|
|
124
|
+
const now = Date.now();
|
|
46
125
|
const grants = await TestingDomainGrantDoc.exact.findStored({ filter: {
|
|
47
126
|
'data.state': 'revoked', 'data.cleanupState': { $in: ['pending', 'indeterminate'] },
|
|
48
127
|
'data.cleanupOperation.phase': { $ne: 'abandoned_unverified' },
|
|
49
|
-
$or: [{ 'data.cleanupRetryAt': { $exists: false } }, { 'data.cleanupRetryAt': { $lte:
|
|
50
|
-
}, sort: { 'data.cleanupRetryAt': 1, id: 1 }, limit:
|
|
128
|
+
$or: [{ 'data.cleanupRetryAt': { $exists: false } }, { 'data.cleanupRetryAt': { $lte: now } }],
|
|
129
|
+
}, sort: { 'data.cleanupRetryAt': 1, id: 1 }, limit: sweepLimit });
|
|
130
|
+
// A grant an operator abandoned, or one that is gone, leaves the sweep without a visit: its wait is over
|
|
131
|
+
// and it is not listed. A full page does not list every due grant, so it says nothing about the others.
|
|
132
|
+
if (grants.length < sweepLimit) {
|
|
133
|
+
const due = new Set(grants.map((grant) => grant.id));
|
|
134
|
+
for (const [grantId, stall] of this.stalls) if (stall.retryAt <= now && !due.has(grantId)) this.stalls.delete(grantId);
|
|
135
|
+
}
|
|
51
136
|
for (const grant of grants) {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
current = await this.change(current, (data) => { data.cleanupRetryAt = Date.now() + testingLimits.pollAfterMs; });
|
|
56
|
-
await this.mutations.cancelForRevokedGrant(current.id);
|
|
57
|
-
await this.certificates.cancelForRevokedGrant(current.id);
|
|
58
|
-
if (await this.blocked(current.id)) {
|
|
59
|
-
await this.change(current, (data) => { data.cleanupState = 'indeterminate'; }); return;
|
|
60
|
-
}
|
|
61
|
-
await this.advance(current, false);
|
|
62
|
-
});
|
|
137
|
+
// One grant that throws must not keep the pass from the grants behind it.
|
|
138
|
+
try { await this.dns.runDnsMutationExclusive(() => this.visit(grant.id)); }
|
|
139
|
+
catch (error) { await this.dns.runDnsMutationExclusive(() => this.deferThrown(grant.id, error)); }
|
|
63
140
|
}
|
|
64
141
|
}
|
|
65
142
|
|
|
@@ -117,7 +194,7 @@ export class TestingDnsCleanupManager {
|
|
|
117
194
|
}
|
|
118
195
|
|
|
119
196
|
private async complete(current: TGrant): Promise<TGrant> {
|
|
120
|
-
if (await this.
|
|
197
|
+
if (await this.blocker(current.id)) return this.change(current, (data) => { data.cleanupState = 'indeterminate'; });
|
|
121
198
|
const session = DcRouterDb.getInstance().getDb().createSession();
|
|
122
199
|
try {
|
|
123
200
|
const result = await session.withTransaction(async (transaction) => {
|
|
@@ -139,7 +216,10 @@ export class TestingDnsCleanupManager {
|
|
|
139
216
|
if (result.status === 'concurrent_change') throw new TestingError('revision_conflict');
|
|
140
217
|
return result.document;
|
|
141
218
|
});
|
|
142
|
-
if (!result) throw new TestingError('service_unavailable');
|
|
219
|
+
if (!result) throw new TestingError('service_unavailable');
|
|
220
|
+
// Every way to completion ends here, an operator's recheck included.
|
|
221
|
+
this.stalls.delete(current.id);
|
|
222
|
+
return result;
|
|
143
223
|
} finally { await session.close(); }
|
|
144
224
|
}
|
|
145
225
|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { TestingError } from './errors.js';
|
|
2
|
+
|
|
3
|
+
export interface ITestingErrorLogFields {
|
|
4
|
+
errorName: string;
|
|
5
|
+
errorCode?: string | number;
|
|
6
|
+
errorMessage: string;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
const withheldMessage = 'withheld: the message may hold key material or a token';
|
|
10
|
+
/** A PEM armor line, or an unbroken run long enough to be a token, a key or a digest. */
|
|
11
|
+
const secretShape = /-----BEGIN|[A-Za-z0-9+_=-]{40,}/;
|
|
12
|
+
/** Shorter than the run `secretShape` withholds, so a code never carries what a message may not. */
|
|
13
|
+
const codeShape = /^[A-Za-z0-9_.-]{1,39}$/;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* What a log line may say about a caught error. A caught error is foreign text: it can quote a provider
|
|
17
|
+
* response, a certificate or a credential. The logger never gets the error itself, its stack or its cause.
|
|
18
|
+
*/
|
|
19
|
+
export function testingErrorLogFields(error: unknown): ITestingErrorLogFields {
|
|
20
|
+
// An error's `message` and `name` are whatever its thrower assigned: only a string is text to log.
|
|
21
|
+
const foreignMessage: unknown = error instanceof Error ? error.message : error;
|
|
22
|
+
const message = typeof foreignMessage === 'string' ? foreignMessage : '';
|
|
23
|
+
const foreignName: unknown = error instanceof Error ? error.name : undefined;
|
|
24
|
+
const name = typeof foreignName === 'string' ? foreignName : typeof error;
|
|
25
|
+
const code = error instanceof TestingError ? error.publicError.code
|
|
26
|
+
: typeof error === 'object' && error !== null && 'code' in error ? error.code : undefined;
|
|
27
|
+
const loggable = typeof code === 'string' ? codeShape.test(code) : Number.isSafeInteger(code);
|
|
28
|
+
return {
|
|
29
|
+
errorName: name.slice(0, 64),
|
|
30
|
+
...(loggable ? { errorCode: code as string | number } : {}),
|
|
31
|
+
// The whole message is judged before it is cut: a secret behind the first line still withholds it.
|
|
32
|
+
errorMessage: secretShape.test(message) ? withheldMessage
|
|
33
|
+
: message.split(/[\r\n\u2028\u2029]/, 1)[0].replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ').slice(0, 300),
|
|
34
|
+
};
|
|
35
|
+
}
|