@enbox/agent 0.7.7 → 0.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.mjs +9 -9
- package/dist/browser.mjs.map +4 -4
- package/dist/esm/dwn-api.js +3 -2
- package/dist/esm/dwn-api.js.map +1 -1
- package/dist/esm/enbox-connect-protocol.js +5 -5
- package/dist/esm/enbox-connect-protocol.js.map +1 -1
- package/dist/esm/index.js +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/permissions-api.js +7 -34
- package/dist/esm/permissions-api.js.map +1 -1
- package/dist/esm/sync-closure-resolver.js +229 -110
- package/dist/esm/sync-closure-resolver.js.map +1 -1
- package/dist/esm/sync-closure-types.js +39 -7
- package/dist/esm/sync-closure-types.js.map +1 -1
- package/dist/esm/sync-engine-level.js +2242 -797
- package/dist/esm/sync-engine-level.js.map +1 -1
- package/dist/esm/sync-link-id.js +4 -13
- package/dist/esm/sync-link-id.js.map +1 -1
- package/dist/esm/sync-link-reconciler.js +26 -8
- package/dist/esm/sync-link-reconciler.js.map +1 -1
- package/dist/esm/sync-messages.js +218 -154
- package/dist/esm/sync-messages.js.map +1 -1
- package/dist/esm/sync-permission-grants.js +208 -0
- package/dist/esm/sync-permission-grants.js.map +1 -0
- package/dist/esm/sync-replication-ledger.js +23 -40
- package/dist/esm/sync-replication-ledger.js.map +1 -1
- package/dist/esm/sync-scope-acceptance.js +126 -0
- package/dist/esm/sync-scope-acceptance.js.map +1 -0
- package/dist/esm/sync-topological-sort.js +57 -15
- package/dist/esm/sync-topological-sort.js.map +1 -1
- package/dist/esm/types/sync.js +130 -22
- package/dist/esm/types/sync.js.map +1 -1
- package/dist/types/dwn-api.d.ts.map +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/permissions-api.d.ts +1 -2
- package/dist/types/permissions-api.d.ts.map +1 -1
- package/dist/types/sync-closure-resolver.d.ts.map +1 -1
- package/dist/types/sync-closure-types.d.ts +14 -3
- package/dist/types/sync-closure-types.d.ts.map +1 -1
- package/dist/types/sync-engine-level.d.ts +144 -31
- package/dist/types/sync-engine-level.d.ts.map +1 -1
- package/dist/types/sync-link-id.d.ts +3 -9
- package/dist/types/sync-link-id.d.ts.map +1 -1
- package/dist/types/sync-link-reconciler.d.ts +12 -2
- package/dist/types/sync-link-reconciler.d.ts.map +1 -1
- package/dist/types/sync-messages.d.ts +16 -13
- package/dist/types/sync-messages.d.ts.map +1 -1
- package/dist/types/sync-permission-grants.d.ts +52 -0
- package/dist/types/sync-permission-grants.d.ts.map +1 -0
- package/dist/types/sync-replication-ledger.d.ts +5 -13
- package/dist/types/sync-replication-ledger.d.ts.map +1 -1
- package/dist/types/sync-scope-acceptance.d.ts +28 -0
- package/dist/types/sync-scope-acceptance.d.ts.map +1 -0
- package/dist/types/sync-topological-sort.d.ts +2 -1
- package/dist/types/sync-topological-sort.d.ts.map +1 -1
- package/dist/types/types/permissions.d.ts +2 -0
- package/dist/types/types/permissions.d.ts.map +1 -1
- package/dist/types/types/sync.d.ts +137 -75
- package/dist/types/types/sync.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/dwn-api.ts +3 -2
- package/src/enbox-connect-protocol.ts +5 -5
- package/src/index.ts +10 -1
- package/src/permissions-api.ts +11 -42
- package/src/sync-closure-resolver.ts +306 -126
- package/src/sync-closure-types.ts +54 -9
- package/src/sync-engine-level.ts +3051 -967
- package/src/sync-link-id.ts +9 -14
- package/src/sync-link-reconciler.ts +43 -10
- package/src/sync-messages.ts +263 -159
- package/src/sync-permission-grants.ts +297 -0
- package/src/sync-replication-ledger.ts +55 -50
- package/src/sync-scope-acceptance.ts +186 -0
- package/src/sync-topological-sort.ts +89 -21
- package/src/types/permissions.ts +2 -0
- package/src/types/sync.ts +235 -62
package/src/sync-engine-level.ts
CHANGED
|
@@ -1,31 +1,36 @@
|
|
|
1
1
|
import type { AbstractLevel } from 'abstract-level';
|
|
2
2
|
|
|
3
3
|
import type { DwnSubscriptionHandler, ResubscribeFactory } from '@enbox/dwn-clients';
|
|
4
|
-
import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncDiffEntry, MessagesSyncReply, ProgressToken, StateIndex, SubscriptionMessage } from '@enbox/dwn-sdk-js';
|
|
4
|
+
import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncDependencyEntry, MessagesSyncDiffEntry, MessagesSyncReply, ProgressToken, ProtocolsConfigureMessage, ProtocolsQueryReply, RecordsProjectionScope, RecordsWriteMessage, StateIndex, SubscriptionMessage } from '@enbox/dwn-sdk-js';
|
|
5
5
|
|
|
6
6
|
import ms from 'ms';
|
|
7
7
|
|
|
8
8
|
import { Level } from 'level';
|
|
9
9
|
import { sleep } from '@enbox/common';
|
|
10
|
-
import { Encoder, hashToHex, initDefaultHashes, Message } from '@enbox/dwn-sdk-js';
|
|
10
|
+
import { authenticate, DwnInterfaceName, DwnMethodName, Encoder, hashToHex, initDefaultHashes, Message, ProtocolsConfigure, RECORDS_PROJECTION_ROOT_VERSION, RecordsProjection, RecordsWrite } from '@enbox/dwn-sdk-js';
|
|
11
11
|
|
|
12
|
-
import type {
|
|
12
|
+
import type { DwnMessageParams } from './types/dwn.js';
|
|
13
13
|
import type { EnboxPlatformAgent } from './types/agent.js';
|
|
14
14
|
import type { PermissionsApi } from './types/permissions.js';
|
|
15
|
-
import type {
|
|
16
|
-
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import { ReplicationLedger } from './sync-replication-ledger.js';
|
|
20
|
-
import { createClosureContext, invalidateClosureCache } from './sync-closure-types.js';
|
|
15
|
+
import type { SyncMessageEntry } from './sync-messages.js';
|
|
16
|
+
import type { SyncScopeClassification } from './sync-scope-acceptance.js';
|
|
17
|
+
import type { ClosureEvaluationContext, ClosureResult } from './sync-closure-types.js';
|
|
18
|
+
import type { DeadLetterCategory, DeadLetterEntry, NonEmptyStringArray, PushResult, ReplicationLinkState, StartSyncParams, SyncAuthorization, SyncConnectivityState, SyncEngine, SyncEvent, SyncEventListener, SyncEventScope, SyncHealthSummary, SyncIdentityOptions, SyncMode, SyncScope } from './types/sync.js';
|
|
21
19
|
|
|
22
20
|
import { AgentPermissionsApi } from './permissions-api.js';
|
|
21
|
+
|
|
22
|
+
import { buildLinkId } from './sync-link-id.js';
|
|
23
23
|
import { DwnInterface } from './types/dwn.js';
|
|
24
|
+
import { evaluateClosure } from './sync-closure-resolver.js';
|
|
24
25
|
import { isRecordsWrite } from './utils.js';
|
|
25
|
-
import {
|
|
26
|
+
import { ReplicationLedger } from './sync-replication-ledger.js';
|
|
26
27
|
import { topologicalSort } from './sync-topological-sort.js';
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
28
|
+
import { classifySyncEventScope, classifySyncMessageScope } from './sync-scope-acceptance.js';
|
|
29
|
+
import { ClosureFailureCode, createClosureContext, invalidateClosureCache, isTerminalClosureFailureCode } from './sync-closure-types.js';
|
|
30
|
+
import { computeAuthorizationEpoch, computeProjectionId, lexicographicalCompare, MAX_PENDING_TOKENS, protocolsForSyncScope, singleProtocolForSyncScope, syncScopeFromProtocols } from './types/sync.js';
|
|
31
|
+
import { fetchRemoteMessages, getMessageCid, pullMessages, pushMessages, SyncPullAbortedError } from './sync-messages.js';
|
|
32
|
+
import { partitionRemoteEntries, SyncLinkReconciler } from './sync-link-reconciler.js';
|
|
33
|
+
import { permissionGrantIdsFromEntries, resolveMessagesSyncScopes, toMessagesPermissionGrantIds, toSyncAuthorizationGrants } from './sync-permission-grants.js';
|
|
29
34
|
|
|
30
35
|
export type SyncEngineLevelParams = {
|
|
31
36
|
agent?: EnboxPlatformAgent;
|
|
@@ -34,7 +39,7 @@ export type SyncEngineLevelParams = {
|
|
|
34
39
|
};
|
|
35
40
|
|
|
36
41
|
/**
|
|
37
|
-
* Maximum bit prefix depth for the per-node tree walk
|
|
42
|
+
* Maximum bit prefix depth for the per-node tree walk fallback.
|
|
38
43
|
* At depth 16, each subtree covers ~1/65536 of the key space.
|
|
39
44
|
*/
|
|
40
45
|
const MAX_DIFF_DEPTH = 16;
|
|
@@ -62,7 +67,6 @@ type LiveSubscription = {
|
|
|
62
67
|
did: string;
|
|
63
68
|
dwnUrl: string;
|
|
64
69
|
delegateDid?: string;
|
|
65
|
-
protocol?: string;
|
|
66
70
|
close: () => Promise<void>;
|
|
67
71
|
};
|
|
68
72
|
|
|
@@ -72,10 +76,103 @@ type LocalSubscription = {
|
|
|
72
76
|
did: string;
|
|
73
77
|
dwnUrl: string;
|
|
74
78
|
delegateDid?: string;
|
|
75
|
-
protocol?: string;
|
|
76
79
|
close: () => Promise<void>;
|
|
77
80
|
};
|
|
78
81
|
|
|
82
|
+
type SyncTarget = {
|
|
83
|
+
did: string;
|
|
84
|
+
dwnUrl: string;
|
|
85
|
+
delegateDid?: string;
|
|
86
|
+
scope: SyncScope;
|
|
87
|
+
authorization: SyncAuthorization;
|
|
88
|
+
authorizationEpoch: string;
|
|
89
|
+
permissionGrantIds?: NonEmptyStringArray;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
type SyncTargetResolution = Pick<SyncTarget, 'authorization' | 'authorizationEpoch' | 'delegateDid' | 'permissionGrantIds' | 'scope'>;
|
|
93
|
+
|
|
94
|
+
type LinkSyncTarget = SyncTarget & { linkKey: string };
|
|
95
|
+
|
|
96
|
+
enum LinkSubscriptionOpenResult {
|
|
97
|
+
ReadyForLive = 'readyForLive',
|
|
98
|
+
Polling = 'polling',
|
|
99
|
+
Repairing = 'repairing',
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
enum LinkInitializationStatus {
|
|
103
|
+
Active = 'active',
|
|
104
|
+
Failed = 'failed',
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
type LinkInitializationResult =
|
|
108
|
+
| { status: LinkInitializationStatus.Active; durableLinkIdentityKey: string }
|
|
109
|
+
| { status: LinkInitializationStatus.Failed };
|
|
110
|
+
|
|
111
|
+
type PullAcceptanceResult =
|
|
112
|
+
| { accepted: true }
|
|
113
|
+
| { accepted: false; classification: Exclude<SyncScopeClassification, 'in-scope'> };
|
|
114
|
+
|
|
115
|
+
type RecordsWriteProtocolDescriptor = GenericMessage['descriptor'] & { protocol?: unknown };
|
|
116
|
+
type RecordsWriteProtocolMessage = GenericMessage & { descriptor: RecordsWriteProtocolDescriptor };
|
|
117
|
+
type ProtocolsConfigureDefinition = {
|
|
118
|
+
protocol: string;
|
|
119
|
+
uses?: Record<string, unknown>;
|
|
120
|
+
};
|
|
121
|
+
type ProtocolsConfigureDefinitionDescriptor = GenericMessage['descriptor'] & {
|
|
122
|
+
definition: ProtocolsConfigureDefinition;
|
|
123
|
+
};
|
|
124
|
+
type MaybeProtocolsConfigureDefinitionDescriptor = GenericMessage['descriptor'] & {
|
|
125
|
+
definition?: {
|
|
126
|
+
protocol?: string;
|
|
127
|
+
uses?: Record<string, unknown>;
|
|
128
|
+
};
|
|
129
|
+
};
|
|
130
|
+
type SyncDiffEntryWithMessage = MessagesSyncDiffEntry & { message: GenericMessage };
|
|
131
|
+
type SyncDependencyEntryWithMessage = MessagesSyncDependencyEntry & { message: GenericMessage };
|
|
132
|
+
type VerifiedProtocolConfigCandidate = {
|
|
133
|
+
rootMessageCid: string;
|
|
134
|
+
dependency: AuthenticatedProtocolConfigDependency;
|
|
135
|
+
};
|
|
136
|
+
type AuthenticatedProtocolConfigDependency = SyncDependencyEntryWithMessage & { message: ProtocolsConfigureMessage };
|
|
137
|
+
type VerifiedRecordsInitialWriteCandidate = {
|
|
138
|
+
rootMessageCid: string;
|
|
139
|
+
dependency: AuthenticatedRecordsInitialWriteDependency;
|
|
140
|
+
};
|
|
141
|
+
type AuthenticatedRecordsInitialWriteDependency = SyncDependencyEntryWithMessage & { message: RecordsWriteMessage };
|
|
142
|
+
|
|
143
|
+
type ProjectionReconcileTarget = {
|
|
144
|
+
did: string;
|
|
145
|
+
dwnUrl: string;
|
|
146
|
+
delegateDid?: string;
|
|
147
|
+
scope: SyncScope;
|
|
148
|
+
authorization: SyncAuthorization;
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
type ProjectionReconcileOptions = {
|
|
152
|
+
direction?: 'push' | 'pull';
|
|
153
|
+
verifyConvergence?: boolean;
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
type ProjectionReconcileResult = {
|
|
157
|
+
aborted?: boolean;
|
|
158
|
+
converged?: boolean;
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
type ProjectionDiffResult = {
|
|
162
|
+
dependencies?: MessagesSyncDependencyEntry[];
|
|
163
|
+
onlyRemote: MessagesSyncDiffEntry[];
|
|
164
|
+
onlyLocal: string[];
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
type ProtocolSetScope = Extract<SyncScope, { kind: 'protocolSet' }>;
|
|
168
|
+
type RecordsProjectionSyncScope = Extract<SyncScope, { kind: 'recordsProjection' }>;
|
|
169
|
+
|
|
170
|
+
type ProtocolSetDiffPlan = {
|
|
171
|
+
changedProtocols: string[];
|
|
172
|
+
onlyRemote: MessagesSyncDiffEntry[];
|
|
173
|
+
onlyLocal: string[];
|
|
174
|
+
};
|
|
175
|
+
|
|
79
176
|
// ---------------------------------------------------------------------------
|
|
80
177
|
// Per-link in-memory delivery-order tracking (not persisted to ledger)
|
|
81
178
|
// ---------------------------------------------------------------------------
|
|
@@ -95,48 +192,6 @@ type InFlightCommit = {
|
|
|
95
192
|
committed: boolean;
|
|
96
193
|
};
|
|
97
194
|
|
|
98
|
-
/**
|
|
99
|
-
* Checks whether a message's protocolPath and contextId match the link's
|
|
100
|
-
* subset scope prefixes. Returns true if the message is in scope.
|
|
101
|
-
*
|
|
102
|
-
* When the scope has no prefixes (or is kind:'full'), all messages match.
|
|
103
|
-
* When protocolPathPrefixes or contextIdPrefixes are specified, the message
|
|
104
|
-
* must match at least one prefix in each specified set.
|
|
105
|
-
*
|
|
106
|
-
* This is agent-side filtering for subset scopes. The underlying
|
|
107
|
-
* MessagesSubscribe filter only supports protocol-level scoping today —
|
|
108
|
-
* protocolPath/contextId prefix filtering at the EventLog level is a
|
|
109
|
-
* follow-up (requires dwn-sdk-js MessagesFilter extension).
|
|
110
|
-
*/
|
|
111
|
-
function isEventInScope(message: GenericMessage, scope: SyncScope): boolean {
|
|
112
|
-
if (scope.kind === 'full') { return true; }
|
|
113
|
-
if (!scope.protocolPathPrefixes && !scope.contextIdPrefixes) { return true; }
|
|
114
|
-
|
|
115
|
-
const desc = message.descriptor as Record<string, unknown>;
|
|
116
|
-
|
|
117
|
-
// Check protocolPath prefix.
|
|
118
|
-
if (scope.protocolPathPrefixes && scope.protocolPathPrefixes.length > 0) {
|
|
119
|
-
const protocolPath = desc.protocolPath as string | undefined;
|
|
120
|
-
if (!protocolPath) { return false; }
|
|
121
|
-
const matches = scope.protocolPathPrefixes.some(
|
|
122
|
-
prefix => protocolPath === prefix || protocolPath.startsWith(prefix + '/')
|
|
123
|
-
);
|
|
124
|
-
if (!matches) { return false; }
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
// Check contextId prefix.
|
|
128
|
-
if (scope.contextIdPrefixes && scope.contextIdPrefixes.length > 0) {
|
|
129
|
-
const contextId = (message as any).contextId as string | undefined;
|
|
130
|
-
if (!contextId) { return false; }
|
|
131
|
-
const matches = scope.contextIdPrefixes.some(
|
|
132
|
-
prefix => contextId === prefix || contextId.startsWith(prefix + '/')
|
|
133
|
-
);
|
|
134
|
-
if (!matches) { return false; }
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
return true;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
195
|
/**
|
|
141
196
|
* Per-link runtime state held in memory. Not persisted — on crash,
|
|
142
197
|
* replay restarts from `contiguousAppliedToken` (idempotent apply).
|
|
@@ -150,18 +205,66 @@ type LinkRuntimeState = {
|
|
|
150
205
|
inflight: Map<number, InFlightCommit>;
|
|
151
206
|
};
|
|
152
207
|
|
|
208
|
+
type PushRuntimeEntry = { cid: string };
|
|
209
|
+
|
|
153
210
|
type PushRuntimeState = {
|
|
154
211
|
did: string;
|
|
155
212
|
dwnUrl: string;
|
|
156
213
|
delegateDid?: string;
|
|
157
214
|
protocol?: string;
|
|
158
|
-
|
|
215
|
+
permissionGrantIds?: NonEmptyStringArray;
|
|
216
|
+
entries: PushRuntimeEntry[];
|
|
159
217
|
retryCount: number;
|
|
160
218
|
timer?: ReturnType<typeof setTimeout>;
|
|
161
219
|
/** True while a push HTTP request is in flight for this link. */
|
|
162
220
|
flushing?: boolean;
|
|
163
221
|
};
|
|
164
222
|
|
|
223
|
+
type PushFlushBatch = {
|
|
224
|
+
pushRuntime: PushRuntimeState;
|
|
225
|
+
pushEntries: PushRuntimeEntry[];
|
|
226
|
+
isStale: () => boolean;
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
type LivePullContext = {
|
|
230
|
+
did: string;
|
|
231
|
+
dwnUrl: string;
|
|
232
|
+
delegateDid?: string;
|
|
233
|
+
eventScope: SyncEventScope;
|
|
234
|
+
linkKey: string;
|
|
235
|
+
link?: ReplicationLinkState;
|
|
236
|
+
permissionGrantIds?: NonEmptyStringArray;
|
|
237
|
+
isStale: () => boolean;
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
type PullDelivery = {
|
|
241
|
+
runtime?: LinkRuntimeState;
|
|
242
|
+
ordinal: number;
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
type LivePullDataStreamFactory = () => Promise<ReadableStream<Uint8Array> | undefined>;
|
|
246
|
+
|
|
247
|
+
type ApplyStatus = {
|
|
248
|
+
code: number;
|
|
249
|
+
detail?: string;
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
function syncEventScope(scope: SyncScope | undefined): SyncEventScope {
|
|
253
|
+
if (scope === undefined) {
|
|
254
|
+
return {};
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const coveredProtocols = protocolsForSyncScope(scope);
|
|
258
|
+
if (coveredProtocols === undefined) {
|
|
259
|
+
return {};
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const protocols = [...coveredProtocols] as NonEmptyStringArray;
|
|
263
|
+
return protocols.length === 1
|
|
264
|
+
? { protocol: protocols[0], protocols }
|
|
265
|
+
: { protocols };
|
|
266
|
+
}
|
|
267
|
+
|
|
165
268
|
export class SyncEngineLevel implements SyncEngine {
|
|
166
269
|
/**
|
|
167
270
|
* Holds the instance of a `EnboxPlatformAgent` that represents the current execution context for
|
|
@@ -257,6 +360,9 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
257
360
|
*/
|
|
258
361
|
private readonly _closureContexts: Map<string, ClosureEvaluationContext> = new Map();
|
|
259
362
|
|
|
363
|
+
/** Deduplicates concurrent live-sync repairs for the same tenant/protocol. */
|
|
364
|
+
private readonly _protocolMetadataRepairs: Map<string, Promise<boolean>> = new Map();
|
|
365
|
+
|
|
260
366
|
/** Maximum entries in the echo-loop suppression cache. */
|
|
261
367
|
private static readonly ECHO_SUPPRESS_MAX_ENTRIES = 10_000;
|
|
262
368
|
|
|
@@ -273,13 +379,67 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
273
379
|
}
|
|
274
380
|
}
|
|
275
381
|
|
|
382
|
+
private async buildSyncTargetsForEndpoint(did: string, dwnUrl: string, options: SyncIdentityOptions): Promise<SyncTarget[]> {
|
|
383
|
+
const requestedScope = syncScopeFromProtocols(options.protocols);
|
|
384
|
+
const resolutions = await this.buildSyncTargetResolutions(did, requestedScope, options);
|
|
385
|
+
|
|
386
|
+
return resolutions.map(resolution => ({
|
|
387
|
+
did,
|
|
388
|
+
dwnUrl,
|
|
389
|
+
...resolution,
|
|
390
|
+
}));
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
private async buildSyncTargetResolutions(did: string, requestedScope: SyncScope, options: SyncIdentityOptions): Promise<SyncTargetResolution[]> {
|
|
394
|
+
const { delegateDid } = options;
|
|
395
|
+
|
|
396
|
+
if (delegateDid === undefined) {
|
|
397
|
+
return [{
|
|
398
|
+
scope : requestedScope,
|
|
399
|
+
authorization : { kind: 'owner' },
|
|
400
|
+
authorizationEpoch : await computeAuthorizationEpoch({ kind: 'owner' }),
|
|
401
|
+
}];
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const resolvedScopes = await resolveMessagesSyncScopes({
|
|
405
|
+
did,
|
|
406
|
+
delegateDid,
|
|
407
|
+
requestedScope,
|
|
408
|
+
messageType : DwnInterface.MessagesSync,
|
|
409
|
+
permissionsApi : this._permissionsApi,
|
|
410
|
+
});
|
|
411
|
+
|
|
412
|
+
return Promise.all(resolvedScopes.map(async ({ scope, permissionGrants }) => {
|
|
413
|
+
const permissionGrantIds = permissionGrantIdsFromEntries(permissionGrants);
|
|
414
|
+
if (permissionGrantIds === undefined) {
|
|
415
|
+
throw new Error(`SyncEngineLevel: delegate ${delegateDid} has no active sync grants for ${did}.`);
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
return {
|
|
419
|
+
scope,
|
|
420
|
+
delegateDid,
|
|
421
|
+
authorization: {
|
|
422
|
+
kind: 'delegate' as const,
|
|
423
|
+
delegateDid,
|
|
424
|
+
permissionGrantIds,
|
|
425
|
+
},
|
|
426
|
+
authorizationEpoch: await computeAuthorizationEpoch({
|
|
427
|
+
kind : 'delegate' as const,
|
|
428
|
+
delegateDid,
|
|
429
|
+
grants : toSyncAuthorizationGrants(permissionGrants),
|
|
430
|
+
}),
|
|
431
|
+
permissionGrantIds,
|
|
432
|
+
};
|
|
433
|
+
}));
|
|
434
|
+
}
|
|
435
|
+
|
|
276
436
|
/**
|
|
277
437
|
* Cached sync targets result from the last {@link getSyncTargets} call.
|
|
278
438
|
* Invalidated on identity registration/unregistration/update.
|
|
279
439
|
* TTL-based: cleared after 30 seconds to pick up DID document changes.
|
|
280
440
|
*/
|
|
281
441
|
private _syncTargetsCache?: {
|
|
282
|
-
targets:
|
|
442
|
+
targets: SyncTarget[];
|
|
283
443
|
timestamp: number;
|
|
284
444
|
};
|
|
285
445
|
|
|
@@ -295,6 +455,9 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
295
455
|
/** TTL for the sync targets cache (30 seconds). */
|
|
296
456
|
private static readonly SYNC_TARGETS_CACHE_TTL_MS = 30_000;
|
|
297
457
|
|
|
458
|
+
/** Backoff schedule for recently published did:dht records. */
|
|
459
|
+
private static readonly DID_RESOLUTION_RETRY_BACKOFF_MS = [2000, 4000, 8000];
|
|
460
|
+
|
|
298
461
|
/** Count of consecutive SMT sync failures (for backoff in poll mode). */
|
|
299
462
|
private _consecutiveFailures = 0;
|
|
300
463
|
|
|
@@ -428,7 +591,12 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
428
591
|
|
|
429
592
|
// If live sync is active, hot-add subscriptions for this identity.
|
|
430
593
|
if (this._syncMode === 'live') {
|
|
431
|
-
await this.addIdentityToLiveSync(did, options);
|
|
594
|
+
const currentIdentityKeys = await this.addIdentityToLiveSync(did, options);
|
|
595
|
+
if (currentIdentityKeys.size > 0) {
|
|
596
|
+
await this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
597
|
+
}
|
|
598
|
+
} else {
|
|
599
|
+
await this.tryPruneSupersededDurableLinksForRegisteredIdentity(did, options);
|
|
432
600
|
}
|
|
433
601
|
}
|
|
434
602
|
|
|
@@ -447,6 +615,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
447
615
|
await registeredIdentities.del(did);
|
|
448
616
|
this._syncTargetsCache = undefined;
|
|
449
617
|
this._syncTargetsCacheGeneration++;
|
|
618
|
+
await this.pruneSupersededDurableLinksForIdentity(did, new Set());
|
|
450
619
|
}
|
|
451
620
|
|
|
452
621
|
public async getIdentityOptions(did: string): Promise<SyncIdentityOptions | undefined> {
|
|
@@ -480,19 +649,17 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
480
649
|
this._syncTargetsCache = undefined;
|
|
481
650
|
this._syncTargetsCacheGeneration++;
|
|
482
651
|
|
|
483
|
-
// Always persist the new delegate to durable links, regardless of
|
|
484
|
-
// sync mode. If sync is stopped or polling, existing persisted links
|
|
485
|
-
// would otherwise keep the old delegateDid. When live sync starts
|
|
486
|
-
// later, initializeLinkTarget() loads the link from LevelDB without
|
|
487
|
-
// normalizing delegateDid, so repair/reconcile paths could use stale
|
|
488
|
-
// delegate data.
|
|
489
|
-
await this.ledger.updateDelegateDid(did, options.delegateDid);
|
|
490
|
-
|
|
491
652
|
// If live sync is active, tear down and rebuild subscriptions with
|
|
492
|
-
// the new options.
|
|
653
|
+
// the new options. Delegate/scope changes derive a new authorization
|
|
654
|
+
// epoch, so existing durable links are not mutated in place.
|
|
493
655
|
if (this._syncMode === 'live' && this.hasActiveLinksForDid(did)) {
|
|
494
656
|
await this.removeIdentityFromLiveSync(did);
|
|
495
|
-
await this.addIdentityToLiveSync(did, options);
|
|
657
|
+
const currentIdentityKeys = await this.addIdentityToLiveSync(did, options);
|
|
658
|
+
if (currentIdentityKeys.size > 0) {
|
|
659
|
+
await this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
660
|
+
}
|
|
661
|
+
} else {
|
|
662
|
+
await this.tryPruneSupersededDurableLinksForRegisteredIdentity(did, options);
|
|
496
663
|
}
|
|
497
664
|
}
|
|
498
665
|
|
|
@@ -526,15 +693,12 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
526
693
|
|
|
527
694
|
const results = await Promise.allSettled([...byUrl.entries()].map(async ([dwnUrl, targets]) => {
|
|
528
695
|
for (const target of targets) {
|
|
529
|
-
const { did, delegateDid, protocol } = target;
|
|
530
696
|
try {
|
|
531
|
-
await this.
|
|
532
|
-
did, dwnUrl, delegateDid, protocol,
|
|
533
|
-
}, { direction });
|
|
697
|
+
await this.reconcileProjectionTarget(target, { direction });
|
|
534
698
|
} catch (error: any) {
|
|
535
699
|
// Skip remaining targets for this DWN endpoint.
|
|
536
700
|
groupsFailed++;
|
|
537
|
-
console.error(`SyncEngineLevel: Error syncing ${did} with ${dwnUrl}`, error);
|
|
701
|
+
console.error(`SyncEngineLevel: Error syncing ${target.did} with ${dwnUrl}`, error);
|
|
538
702
|
return;
|
|
539
703
|
}
|
|
540
704
|
}
|
|
@@ -636,7 +800,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
636
800
|
}
|
|
637
801
|
|
|
638
802
|
// ---------------------------------------------------------------------------
|
|
639
|
-
// Poll-mode sync
|
|
803
|
+
// Poll-mode sync
|
|
640
804
|
// ---------------------------------------------------------------------------
|
|
641
805
|
|
|
642
806
|
private async startPollSync(intervalMilliseconds: number): Promise<void> {
|
|
@@ -768,7 +932,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
768
932
|
}
|
|
769
933
|
|
|
770
934
|
// ---------------------------------------------------------------------------
|
|
771
|
-
// Per-link repair and degraded-poll orchestration
|
|
935
|
+
// Per-link repair and degraded-poll orchestration
|
|
772
936
|
// ---------------------------------------------------------------------------
|
|
773
937
|
|
|
774
938
|
/** Maximum consecutive repair attempts before falling back to degraded_poll. */
|
|
@@ -811,27 +975,19 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
811
975
|
link: ReplicationLinkState,
|
|
812
976
|
options?: { resumeToken?: ProgressToken },
|
|
813
977
|
): Promise<void> {
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
link.connectivity = 'offline';
|
|
817
|
-
await this.ledger.setStatus(link, 'repairing');
|
|
818
|
-
|
|
819
|
-
this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevStatus, to: 'repairing' });
|
|
820
|
-
if (prevConnectivity !== 'offline') {
|
|
821
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevConnectivity, to: 'offline' });
|
|
978
|
+
if (link.status === 'terminal_incomplete') {
|
|
979
|
+
return;
|
|
822
980
|
}
|
|
823
981
|
|
|
982
|
+
await this.setLinkOfflineStatus(link, 'repairing');
|
|
983
|
+
|
|
824
984
|
if (options?.resumeToken) {
|
|
825
985
|
this._repairContext.set(linkKey, { resumeToken: options.resumeToken });
|
|
826
986
|
}
|
|
827
987
|
|
|
828
988
|
// Clear runtime ordinals immediately — stale state must not linger
|
|
829
989
|
// across repair attempts.
|
|
830
|
-
|
|
831
|
-
if (rt) {
|
|
832
|
-
rt.inflight.clear();
|
|
833
|
-
rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
|
|
834
|
-
}
|
|
990
|
+
this.clearLinkRuntimeInflight(linkKey);
|
|
835
991
|
|
|
836
992
|
// Kick off repair with retry scheduling on failure.
|
|
837
993
|
void this.repairLink(linkKey).catch(() => {
|
|
@@ -839,6 +995,68 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
839
995
|
});
|
|
840
996
|
}
|
|
841
997
|
|
|
998
|
+
private async transitionToTerminalIncomplete(
|
|
999
|
+
linkKey: string,
|
|
1000
|
+
link: ReplicationLinkState,
|
|
1001
|
+
): Promise<void> {
|
|
1002
|
+
if (link.status === 'terminal_incomplete') {
|
|
1003
|
+
return;
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
await this.setLinkOfflineStatus(link, 'terminal_incomplete');
|
|
1007
|
+
|
|
1008
|
+
await this.closeLinkSubscriptions(link);
|
|
1009
|
+
|
|
1010
|
+
this.clearLinkRuntimeInflight(linkKey);
|
|
1011
|
+
|
|
1012
|
+
const retryTimer = this._repairRetryTimers.get(linkKey);
|
|
1013
|
+
if (retryTimer) {
|
|
1014
|
+
clearTimeout(retryTimer);
|
|
1015
|
+
this._repairRetryTimers.delete(linkKey);
|
|
1016
|
+
}
|
|
1017
|
+
const degradedTimer = this._degradedPollTimers.get(linkKey);
|
|
1018
|
+
if (degradedTimer) {
|
|
1019
|
+
clearInterval(degradedTimer);
|
|
1020
|
+
this._degradedPollTimers.delete(linkKey);
|
|
1021
|
+
}
|
|
1022
|
+
const reconcileTimer = this._reconcileTimers.get(linkKey);
|
|
1023
|
+
if (reconcileTimer) {
|
|
1024
|
+
clearTimeout(reconcileTimer);
|
|
1025
|
+
this._reconcileTimers.delete(linkKey);
|
|
1026
|
+
}
|
|
1027
|
+
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
1028
|
+
if (pushRuntime?.timer) {
|
|
1029
|
+
clearTimeout(pushRuntime.timer);
|
|
1030
|
+
}
|
|
1031
|
+
this._pushRuntimes.delete(linkKey);
|
|
1032
|
+
|
|
1033
|
+
this._repairAttempts.delete(linkKey);
|
|
1034
|
+
this._repairContext.delete(linkKey);
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
private async setLinkOfflineStatus(link: ReplicationLinkState, status: ReplicationLinkState['status']): Promise<void> {
|
|
1038
|
+
const prevStatus = link.status;
|
|
1039
|
+
const prevConnectivity = link.connectivity;
|
|
1040
|
+
link.connectivity = 'offline';
|
|
1041
|
+
await this.ledger.setStatus(link, status);
|
|
1042
|
+
|
|
1043
|
+
const eventScope = syncEventScope(link.scope);
|
|
1044
|
+
this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope, from: prevStatus, to: status });
|
|
1045
|
+
if (prevConnectivity !== 'offline') {
|
|
1046
|
+
this.emitEvent({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope, from: prevConnectivity, to: 'offline' });
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
private clearLinkRuntimeInflight(linkKey: string): void {
|
|
1051
|
+
const rt = this._linkRuntimes.get(linkKey);
|
|
1052
|
+
if (!rt) {
|
|
1053
|
+
return;
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
rt.inflight.clear();
|
|
1057
|
+
rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
|
|
1058
|
+
}
|
|
1059
|
+
|
|
842
1060
|
/**
|
|
843
1061
|
* Schedule a retry for a failed repair. Uses exponential backoff.
|
|
844
1062
|
* No-op if the link is already in `degraded_poll` (timer loop owns retries)
|
|
@@ -925,9 +1143,10 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
925
1143
|
// The old repair closure must not mutate the replacement link's state.
|
|
926
1144
|
const isStaleLink = (): boolean => this._activeLinks.get(linkKey) !== link;
|
|
927
1145
|
|
|
928
|
-
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid,
|
|
1146
|
+
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, scope, authorization } = link;
|
|
1147
|
+
const eventScope = syncEventScope(scope);
|
|
929
1148
|
|
|
930
|
-
this.emitEvent({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1149
|
+
this.emitEvent({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, attempt: (this._repairAttempts.get(linkKey) ?? 0) + 1 });
|
|
931
1150
|
const attempts = (this._repairAttempts.get(linkKey) ?? 0) + 1;
|
|
932
1151
|
this._repairAttempts.set(linkKey, attempts);
|
|
933
1152
|
|
|
@@ -945,9 +1164,13 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
945
1164
|
|
|
946
1165
|
try {
|
|
947
1166
|
// Step 3: Run SMT reconciliation for this link.
|
|
948
|
-
const reconcileOutcome = await this.
|
|
949
|
-
|
|
950
|
-
|
|
1167
|
+
const reconcileOutcome = await this.reconcileProjectionTarget({
|
|
1168
|
+
did,
|
|
1169
|
+
dwnUrl,
|
|
1170
|
+
delegateDid,
|
|
1171
|
+
scope,
|
|
1172
|
+
authorization,
|
|
1173
|
+
}, undefined, () => this._engineGeneration === generation && !isStaleLink());
|
|
951
1174
|
if (reconcileOutcome.aborted) { return; }
|
|
952
1175
|
|
|
953
1176
|
// Step 4: Determine the post-repair pull resume token.
|
|
@@ -972,7 +1195,16 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
972
1195
|
await this.ledger.saveLink(link);
|
|
973
1196
|
if (this._engineGeneration !== generation || isStaleLink()) { return; }
|
|
974
1197
|
|
|
975
|
-
const target = {
|
|
1198
|
+
const target = {
|
|
1199
|
+
did,
|
|
1200
|
+
dwnUrl,
|
|
1201
|
+
delegateDid,
|
|
1202
|
+
scope,
|
|
1203
|
+
authorization,
|
|
1204
|
+
authorizationEpoch : link.authorizationEpoch,
|
|
1205
|
+
permissionGrantIds : this.getAuthorizationGrantIds(authorization),
|
|
1206
|
+
linkKey,
|
|
1207
|
+
};
|
|
976
1208
|
try {
|
|
977
1209
|
await this.openLivePullSubscription(target);
|
|
978
1210
|
} catch (pullErr: any) {
|
|
@@ -1013,16 +1245,15 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1013
1245
|
link.connectivity = 'online';
|
|
1014
1246
|
await this.ledger.setStatus(link, 'live');
|
|
1015
1247
|
|
|
1016
|
-
//
|
|
1017
|
-
//
|
|
1018
|
-
//
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, protocol });
|
|
1248
|
+
// Root convergence proves primary CID membership matches, but it does
|
|
1249
|
+
// not prove dependencies are usable. Keep closure failures until a later
|
|
1250
|
+
// successful apply/closure pass clears the specific CID.
|
|
1251
|
+
await this.clearRootConvergenceDeadLettersForScope(did, dwnUrl, scope);
|
|
1252
|
+
this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope });
|
|
1022
1253
|
if (prevRepairConnectivity !== 'online') {
|
|
1023
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1254
|
+
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: prevRepairConnectivity, to: 'online' });
|
|
1024
1255
|
}
|
|
1025
|
-
this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1256
|
+
this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: 'repairing', to: 'live' });
|
|
1026
1257
|
|
|
1027
1258
|
} catch (error: any) {
|
|
1028
1259
|
// If teardown occurred during repair or the link was replaced by a
|
|
@@ -1030,7 +1261,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1030
1261
|
if (this._engineGeneration !== generation || isStaleLink()) { return; }
|
|
1031
1262
|
|
|
1032
1263
|
console.error(`SyncEngineLevel: Repair failed for ${did} -> ${dwnUrl} (attempt ${attempts})`, error);
|
|
1033
|
-
this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1264
|
+
this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, attempt: attempts, error: String(error.message ?? error) });
|
|
1034
1265
|
|
|
1035
1266
|
if (attempts >= SyncEngineLevel.MAX_REPAIR_ATTEMPTS) {
|
|
1036
1267
|
console.warn(`SyncEngineLevel: Max repair attempts reached for ${did} -> ${dwnUrl}, entering degraded_poll`);
|
|
@@ -1048,21 +1279,26 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1048
1279
|
*/
|
|
1049
1280
|
private async closeLinkSubscriptions(link: ReplicationLinkState): Promise<void> {
|
|
1050
1281
|
const { tenantDid: did, remoteEndpoint: dwnUrl } = link;
|
|
1051
|
-
const linkKey = this.buildLinkKey(did, dwnUrl, link.
|
|
1282
|
+
const linkKey = this.buildLinkKey(did, dwnUrl, link.projectionId, link.authorizationEpoch);
|
|
1052
1283
|
|
|
1053
|
-
|
|
1284
|
+
await this.closeLiveSubscription(linkKey);
|
|
1285
|
+
await this.closeLocalSubscription(linkKey);
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
private async closeLiveSubscription(linkKey: string): Promise<void> {
|
|
1054
1289
|
const pullSub = this._liveSubscriptions.find((s) => s.linkKey === linkKey);
|
|
1055
|
-
if (pullSub) {
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1290
|
+
if (!pullSub) { return; }
|
|
1291
|
+
|
|
1292
|
+
try { await pullSub.close(); } catch { /* best effort */ }
|
|
1293
|
+
this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
|
|
1294
|
+
}
|
|
1059
1295
|
|
|
1060
|
-
|
|
1296
|
+
private async closeLocalSubscription(linkKey: string): Promise<void> {
|
|
1061
1297
|
const pushSub = this._localSubscriptions.find((s) => s.linkKey === linkKey);
|
|
1062
|
-
if (pushSub) {
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1298
|
+
if (!pushSub) { return; }
|
|
1299
|
+
|
|
1300
|
+
try { await pushSub.close(); } catch { /* best effort */ }
|
|
1301
|
+
this._localSubscriptions = this._localSubscriptions.filter(s => s !== pushSub);
|
|
1066
1302
|
}
|
|
1067
1303
|
|
|
1068
1304
|
/**
|
|
@@ -1078,8 +1314,9 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1078
1314
|
const prevDegradedStatus = link.status;
|
|
1079
1315
|
await this.ledger.setStatus(link, 'degraded_poll');
|
|
1080
1316
|
this._repairAttempts.delete(linkKey);
|
|
1081
|
-
|
|
1082
|
-
this.emitEvent({ type: '
|
|
1317
|
+
const eventScope = syncEventScope(link.scope);
|
|
1318
|
+
this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope, from: prevDegradedStatus, to: 'degraded_poll' });
|
|
1319
|
+
this.emitEvent({ type: 'degraded-poll:entered', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope });
|
|
1083
1320
|
|
|
1084
1321
|
// Clear any existing timer for this link.
|
|
1085
1322
|
const existing = this._degradedPollTimers.get(linkKey);
|
|
@@ -1194,7 +1431,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1194
1431
|
type : 'link:connectivity-change',
|
|
1195
1432
|
tenantDid : link.tenantDid,
|
|
1196
1433
|
remoteEndpoint : link.remoteEndpoint,
|
|
1197
|
-
|
|
1434
|
+
...syncEventScope(link.scope),
|
|
1198
1435
|
from : prev,
|
|
1199
1436
|
to : 'offline',
|
|
1200
1437
|
});
|
|
@@ -1306,6 +1543,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1306
1543
|
|
|
1307
1544
|
// Clear closure evaluation contexts.
|
|
1308
1545
|
this._closureContexts.clear();
|
|
1546
|
+
this._protocolMetadataRepairs.clear();
|
|
1309
1547
|
this._recentlyPulledCids.clear();
|
|
1310
1548
|
|
|
1311
1549
|
// Clear the in-memory link and runtime state.
|
|
@@ -1319,80 +1557,144 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1319
1557
|
|
|
1320
1558
|
/**
|
|
1321
1559
|
* Initialize a single replication link target: create or resume the durable
|
|
1322
|
-
* link,
|
|
1323
|
-
* transition the link to `'live'`.
|
|
1560
|
+
* link, open pull + push subscriptions, and transition the link to `'live'`.
|
|
1324
1561
|
*/
|
|
1325
|
-
private async initializeLinkTarget(target: {
|
|
1326
|
-
did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
|
|
1327
|
-
}): Promise<void> {
|
|
1562
|
+
private async initializeLinkTarget(target: SyncTarget): Promise<LinkInitializationResult> {
|
|
1328
1563
|
let link: ReplicationLinkState | undefined;
|
|
1329
1564
|
try {
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
link
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
scope : linkScope,
|
|
1337
|
-
delegateDid : target.delegateDid,
|
|
1338
|
-
protocol : target.protocol,
|
|
1339
|
-
});
|
|
1340
|
-
|
|
1341
|
-
const linkKey = this.buildLinkKey(target.did, target.dwnUrl, link.scopeId);
|
|
1565
|
+
link = await this.getOrCreateReplicationLink(target);
|
|
1566
|
+
const linkKey = this.getReplicationLinkKey(target, link);
|
|
1567
|
+
this._activeLinks.set(linkKey, link);
|
|
1568
|
+
if (link.status === 'terminal_incomplete') {
|
|
1569
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1570
|
+
}
|
|
1342
1571
|
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
await this.ledger.saveLink(link);
|
|
1349
|
-
await this.deleteLegacyCursor(legacyKey);
|
|
1350
|
-
}
|
|
1572
|
+
const subscriptionResult = await this.openLinkSubscriptions({ ...target, linkKey });
|
|
1573
|
+
if (subscriptionResult === LinkSubscriptionOpenResult.ReadyForLive) {
|
|
1574
|
+
await this.markLinkLive(target, link, linkKey);
|
|
1575
|
+
} else if (subscriptionResult === LinkSubscriptionOpenResult.Polling) {
|
|
1576
|
+
await this.markLinkPolling(target, link);
|
|
1351
1577
|
}
|
|
1578
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1579
|
+
} catch (error: any) {
|
|
1580
|
+
return this.handleInitializeLinkTargetError(target, link, error);
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1352
1583
|
|
|
1353
|
-
|
|
1584
|
+
private async getOrCreateReplicationLink(target: SyncTarget): Promise<ReplicationLinkState> {
|
|
1585
|
+
return this.ledger.getOrCreateLink({
|
|
1586
|
+
tenantDid : target.did,
|
|
1587
|
+
remoteEndpoint : target.dwnUrl,
|
|
1588
|
+
scope : target.scope,
|
|
1589
|
+
authorization : target.authorization,
|
|
1590
|
+
authorizationEpoch : target.authorizationEpoch,
|
|
1591
|
+
delegateDid : target.delegateDid,
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1354
1594
|
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
await this.openLocalPushSubscription(targetWithKey);
|
|
1359
|
-
} catch (pushError) {
|
|
1360
|
-
const pullSub = this._liveSubscriptions.find((s) => s.linkKey === linkKey);
|
|
1361
|
-
if (pullSub) {
|
|
1362
|
-
try { await pullSub.close(); } catch { /* best effort */ }
|
|
1363
|
-
this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
|
|
1364
|
-
}
|
|
1365
|
-
throw pushError;
|
|
1366
|
-
}
|
|
1595
|
+
private getReplicationLinkKey(target: SyncTarget, link: ReplicationLinkState): string {
|
|
1596
|
+
return this.buildLinkKey(target.did, target.dwnUrl, link.projectionId, link.authorizationEpoch);
|
|
1597
|
+
}
|
|
1367
1598
|
|
|
1368
|
-
|
|
1369
|
-
|
|
1599
|
+
private async openLinkSubscriptions(target: LinkSyncTarget): Promise<LinkSubscriptionOpenResult> {
|
|
1600
|
+
if (!SyncEngineLevel.supportsLiveSubscriptions(target.scope)) {
|
|
1601
|
+
return LinkSubscriptionOpenResult.Polling;
|
|
1602
|
+
}
|
|
1370
1603
|
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
: buildLegacyCursorKey(target.did, target.dwnUrl, target.protocol);
|
|
1378
|
-
|
|
1379
|
-
if (error.isProgressGap && link) {
|
|
1380
|
-
console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
|
|
1381
|
-
this.emitEvent({ type: 'gap:detected', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, reason: 'ProgressGap' });
|
|
1382
|
-
await this.transitionToRepairing(linkKey, link, {
|
|
1383
|
-
resumeToken: error.gapInfo?.latestAvailable,
|
|
1384
|
-
});
|
|
1385
|
-
return;
|
|
1386
|
-
}
|
|
1604
|
+
await this.openLivePullSubscription(target);
|
|
1605
|
+
const link = this._activeLinks.get(target.linkKey);
|
|
1606
|
+
if (link?.status === 'repairing') {
|
|
1607
|
+
await this.closeLiveSubscription(target.linkKey);
|
|
1608
|
+
return LinkSubscriptionOpenResult.Repairing;
|
|
1609
|
+
}
|
|
1387
1610
|
|
|
1388
|
-
|
|
1611
|
+
try {
|
|
1612
|
+
await this.openLocalPushSubscription(target);
|
|
1613
|
+
} catch (error) {
|
|
1614
|
+
await this.closeLiveSubscription(target.linkKey);
|
|
1615
|
+
throw error;
|
|
1616
|
+
}
|
|
1617
|
+
return LinkSubscriptionOpenResult.ReadyForLive;
|
|
1618
|
+
}
|
|
1389
1619
|
|
|
1390
|
-
|
|
1391
|
-
|
|
1620
|
+
private static supportsLiveSubscriptions(scope: SyncScope): boolean {
|
|
1621
|
+
// Records-primary projected links reconcile by root/diff polling until the
|
|
1622
|
+
// DWN has explicit path/context live subscription semantics.
|
|
1623
|
+
return scope.kind !== 'recordsProjection';
|
|
1624
|
+
}
|
|
1392
1625
|
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1626
|
+
private async markLinkLive(target: SyncTarget, link: ReplicationLinkState, linkKey: string): Promise<void> {
|
|
1627
|
+
this.emitEvent({
|
|
1628
|
+
type : 'link:status-change',
|
|
1629
|
+
tenantDid : target.did,
|
|
1630
|
+
remoteEndpoint : target.dwnUrl,
|
|
1631
|
+
...syncEventScope(target.scope),
|
|
1632
|
+
from : 'initializing',
|
|
1633
|
+
to : 'live'
|
|
1634
|
+
});
|
|
1635
|
+
await this.ledger.setStatus(link, 'live');
|
|
1636
|
+
|
|
1637
|
+
if (link.needsReconcile) {
|
|
1638
|
+
this.scheduleReconcile(linkKey, 1000);
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
private async markLinkPolling(target: SyncTarget, link: ReplicationLinkState): Promise<void> {
|
|
1643
|
+
this.emitEvent({
|
|
1644
|
+
type : 'link:status-change',
|
|
1645
|
+
tenantDid : target.did,
|
|
1646
|
+
remoteEndpoint : target.dwnUrl,
|
|
1647
|
+
...syncEventScope(target.scope),
|
|
1648
|
+
from : 'initializing',
|
|
1649
|
+
to : 'polling'
|
|
1650
|
+
});
|
|
1651
|
+
await this.ledger.setStatus(link, 'polling');
|
|
1652
|
+
}
|
|
1653
|
+
|
|
1654
|
+
private async handleInitializeLinkTargetError(
|
|
1655
|
+
target: SyncTarget,
|
|
1656
|
+
link: ReplicationLinkState | undefined,
|
|
1657
|
+
error: any,
|
|
1658
|
+
): Promise<LinkInitializationResult> {
|
|
1659
|
+
if (error.isProgressGap && link) {
|
|
1660
|
+
const linkKey = this.getReplicationLinkKey(target, link);
|
|
1661
|
+
console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
|
|
1662
|
+
this.emitEvent({
|
|
1663
|
+
type : 'gap:detected',
|
|
1664
|
+
tenantDid : target.did,
|
|
1665
|
+
remoteEndpoint : target.dwnUrl,
|
|
1666
|
+
...syncEventScope(target.scope),
|
|
1667
|
+
reason : 'ProgressGap'
|
|
1668
|
+
});
|
|
1669
|
+
await this.transitionToRepairing(linkKey, link, {
|
|
1670
|
+
resumeToken: error.gapInfo?.latestAvailable,
|
|
1671
|
+
});
|
|
1672
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
|
|
1676
|
+
if (link) {
|
|
1677
|
+
this.cleanupFailedLinkInitialization(this.getReplicationLinkKey(target, link));
|
|
1678
|
+
}
|
|
1679
|
+
if (this.isDidResolutionFailure(error)) {
|
|
1680
|
+
throw error;
|
|
1681
|
+
}
|
|
1682
|
+
return { status: LinkInitializationStatus.Failed };
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
private createActiveLinkInitializationResult(link: ReplicationLinkState): LinkInitializationResult {
|
|
1686
|
+
return {
|
|
1687
|
+
status : LinkInitializationStatus.Active,
|
|
1688
|
+
durableLinkIdentityKey : this.getDurableLinkIdentityKey(link),
|
|
1689
|
+
};
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
private cleanupFailedLinkInitialization(linkKey: string): void {
|
|
1693
|
+
this._activeLinks.delete(linkKey);
|
|
1694
|
+
this._linkRuntimes.delete(linkKey);
|
|
1695
|
+
|
|
1696
|
+
if (this._liveSubscriptions.length === 0) {
|
|
1697
|
+
this._connectivityState = 'unknown';
|
|
1396
1698
|
}
|
|
1397
1699
|
}
|
|
1398
1700
|
|
|
@@ -1404,31 +1706,31 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1404
1706
|
* causing a 401. Retrying with exponential backoff lets the
|
|
1405
1707
|
* propagation settle before giving up.
|
|
1406
1708
|
*/
|
|
1407
|
-
private async initializeLinkTargetWithRetry(target: {
|
|
1408
|
-
did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
|
|
1409
|
-
}): Promise<void> {
|
|
1709
|
+
private async initializeLinkTargetWithRetry(target: SyncTarget): Promise<LinkInitializationResult> {
|
|
1410
1710
|
try {
|
|
1411
|
-
await this.initializeLinkTarget(target);
|
|
1711
|
+
return await this.initializeLinkTarget(target);
|
|
1412
1712
|
} catch (error: any) {
|
|
1413
|
-
|
|
1414
|
-
const isDidResolutionFailure = msg.includes('GetPublicKeyNotFound') || msg.includes('notFound');
|
|
1415
|
-
if (!isDidResolutionFailure) { throw error; }
|
|
1713
|
+
if (!this.isDidResolutionFailure(error)) { throw error; }
|
|
1416
1714
|
|
|
1417
|
-
const
|
|
1418
|
-
for (const delay of delays) {
|
|
1715
|
+
for (const delay of SyncEngineLevel.DID_RESOLUTION_RETRY_BACKOFF_MS) {
|
|
1419
1716
|
await sleep(delay);
|
|
1420
1717
|
try {
|
|
1421
|
-
await this.initializeLinkTarget(target);
|
|
1422
|
-
return;
|
|
1718
|
+
return await this.initializeLinkTarget(target);
|
|
1423
1719
|
} catch {
|
|
1424
1720
|
// Continue to next attempt.
|
|
1425
1721
|
}
|
|
1426
1722
|
}
|
|
1427
1723
|
// All retries exhausted — the original error was already logged
|
|
1428
1724
|
// by initializeLinkTarget's catch block.
|
|
1725
|
+
return { status: LinkInitializationStatus.Failed };
|
|
1429
1726
|
}
|
|
1430
1727
|
}
|
|
1431
1728
|
|
|
1729
|
+
private isDidResolutionFailure(error: any): boolean {
|
|
1730
|
+
const message = error.message ?? '';
|
|
1731
|
+
return message.includes('GetPublicKeyNotFound');
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1432
1734
|
// ---------------------------------------------------------------------------
|
|
1433
1735
|
// Hot-add / hot-remove: per-identity live sync management
|
|
1434
1736
|
// ---------------------------------------------------------------------------
|
|
@@ -1447,23 +1749,23 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1447
1749
|
}
|
|
1448
1750
|
|
|
1449
1751
|
/** Hot-add a single identity to the active live sync session. */
|
|
1450
|
-
private async addIdentityToLiveSync(did: string, options: SyncIdentityOptions): Promise<
|
|
1451
|
-
const { protocols, delegateDid } = options;
|
|
1752
|
+
private async addIdentityToLiveSync(did: string, options: SyncIdentityOptions): Promise<Set<string>> {
|
|
1452
1753
|
const dwnEndpointUrls = await this.agent.dwn.getDwnEndpointUrlsForTarget(did);
|
|
1453
|
-
if (dwnEndpointUrls.length === 0) { return; }
|
|
1754
|
+
if (dwnEndpointUrls.length === 0) { return new Set(); }
|
|
1454
1755
|
|
|
1455
|
-
const targets:
|
|
1756
|
+
const targets: SyncTarget[] = [];
|
|
1456
1757
|
for (const dwnUrl of dwnEndpointUrls) {
|
|
1457
|
-
|
|
1458
|
-
targets.push({ did, delegateDid, dwnUrl });
|
|
1459
|
-
} else {
|
|
1460
|
-
for (const protocol of protocols) {
|
|
1461
|
-
targets.push({ did, delegateDid, dwnUrl, protocol });
|
|
1462
|
-
}
|
|
1463
|
-
}
|
|
1758
|
+
targets.push(...await this.buildSyncTargetsForEndpoint(did, dwnUrl, options));
|
|
1464
1759
|
}
|
|
1465
1760
|
|
|
1466
|
-
await Promise.allSettled(targets.map(t => this.initializeLinkTargetWithRetry(t)));
|
|
1761
|
+
const results = await Promise.allSettled(targets.map(t => this.initializeLinkTargetWithRetry(t)));
|
|
1762
|
+
const currentIdentityKeys = new Set<string>();
|
|
1763
|
+
for (const result of results) {
|
|
1764
|
+
if (result.status === 'fulfilled' && result.value.status === LinkInitializationStatus.Active) {
|
|
1765
|
+
currentIdentityKeys.add(result.value.durableLinkIdentityKey);
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
return currentIdentityKeys;
|
|
1467
1769
|
}
|
|
1468
1770
|
|
|
1469
1771
|
/** Hot-remove a single identity from the active live sync session. */
|
|
@@ -1512,6 +1814,36 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1512
1814
|
this._closureContexts.delete(did);
|
|
1513
1815
|
}
|
|
1514
1816
|
|
|
1817
|
+
private async tryPruneSupersededDurableLinksForRegisteredIdentity(did: string, options: SyncIdentityOptions): Promise<void> {
|
|
1818
|
+
try {
|
|
1819
|
+
const currentIdentityKeys = await this.getDurableLinkIdentityKeysForRegisteredIdentity(did, options);
|
|
1820
|
+
await this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
1821
|
+
} catch (error: unknown) {
|
|
1822
|
+
console.warn(`SyncEngineLevel: Failed to prune superseded durable links for ${did}`, error);
|
|
1823
|
+
}
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
private async getDurableLinkIdentityKeysForRegisteredIdentity(did: string, options: SyncIdentityOptions): Promise<Set<string>> {
|
|
1827
|
+
const scope = syncScopeFromProtocols(options.protocols);
|
|
1828
|
+
const resolutions = await this.buildSyncTargetResolutions(did, scope, options);
|
|
1829
|
+
const keys = new Set<string>();
|
|
1830
|
+
for (const resolution of resolutions) {
|
|
1831
|
+
const projectionId = await computeProjectionId(did, resolution.scope);
|
|
1832
|
+
keys.add(SyncEngineLevel.durableLinkIdentityKey(did, projectionId, resolution.authorizationEpoch));
|
|
1833
|
+
}
|
|
1834
|
+
return keys;
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
private async pruneSupersededDurableLinksForIdentity(did: string, currentIdentityKeys: Set<string>): Promise<void> {
|
|
1838
|
+
const links = await this.ledger.getLinksForTenant(did);
|
|
1839
|
+
await Promise.all(links.map(async link => {
|
|
1840
|
+
if (currentIdentityKeys.has(this.getDurableLinkIdentityKey(link))) {
|
|
1841
|
+
return;
|
|
1842
|
+
}
|
|
1843
|
+
await this.ledger.deleteLink(link.tenantDid, link.remoteEndpoint, link.projectionId, link.authorizationEpoch);
|
|
1844
|
+
}));
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1515
1847
|
// ---------------------------------------------------------------------------
|
|
1516
1848
|
// Live pull: MessagesSubscribe to remote DWN
|
|
1517
1849
|
// ---------------------------------------------------------------------------
|
|
@@ -1520,60 +1852,18 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1520
1852
|
* Opens a MessagesSubscribe WebSocket subscription to a remote DWN.
|
|
1521
1853
|
* Incoming events are processed locally as they arrive.
|
|
1522
1854
|
*/
|
|
1523
|
-
private async openLivePullSubscription(target: {
|
|
1524
|
-
did
|
|
1525
|
-
|
|
1526
|
-
}): Promise<void> {
|
|
1527
|
-
const { did, delegateDid, dwnUrl, protocol } = target;
|
|
1855
|
+
private async openLivePullSubscription(target: LinkSyncTarget): Promise<void> {
|
|
1856
|
+
const { did, delegateDid, dwnUrl } = target;
|
|
1857
|
+
const eventScope = syncEventScope(target.scope);
|
|
1528
1858
|
|
|
1529
|
-
// Resolve the cursor from the link's durable pull checkpoint.
|
|
1530
|
-
// Legacy syncCursors migration happens at link load time in startLiveSync().
|
|
1531
1859
|
const cursorKey = target.linkKey;
|
|
1532
1860
|
const link = this._activeLinks.get(cursorKey);
|
|
1533
|
-
|
|
1861
|
+
const cursor = await this.getInitialPullCursor({ did, dwnUrl, link });
|
|
1534
1862
|
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
// start from the beginning rather than crash the subscription.
|
|
1538
|
-
if (cursor && (!cursor.streamId || !cursor.messageCid || !cursor.epoch || !cursor.position)) {
|
|
1539
|
-
console.warn(`SyncEngineLevel: Discarding stored cursor with empty field(s) for ${did} -> ${dwnUrl}`);
|
|
1540
|
-
cursor = undefined;
|
|
1541
|
-
if (link) {
|
|
1542
|
-
ReplicationLedger.resetCheckpoint(link.pull);
|
|
1543
|
-
await this.ledger.saveLink(link);
|
|
1544
|
-
}
|
|
1545
|
-
}
|
|
1546
|
-
|
|
1547
|
-
// Build the MessagesSubscribe filters.
|
|
1548
|
-
// When the link has protocolPathPrefixes, include them in the filter so the
|
|
1549
|
-
// EventLog delivers only matching events (server-side filtering). This replaces
|
|
1550
|
-
// the less efficient agent-side isEventInScope filtering for the pull path.
|
|
1551
|
-
// Note: only the first prefix is used as the MessagesFilter field because
|
|
1552
|
-
// MessagesFilter.protocolPathPrefix is a single string. Multiple prefixes
|
|
1553
|
-
// would need multiple filters (OR semantics) — for now we use the first one.
|
|
1554
|
-
const protocolPathPrefix = link?.scope.kind === 'protocol'
|
|
1555
|
-
? link.scope.protocolPathPrefixes?.[0]
|
|
1556
|
-
: undefined;
|
|
1557
|
-
const filters = protocol
|
|
1558
|
-
? [{ protocol, ...(protocolPathPrefix ? { protocolPathPrefix } : {}) }]
|
|
1863
|
+
const filters = target.scope.kind === 'protocolSet'
|
|
1864
|
+
? target.scope.protocols.map(protocol => ({ protocol }))
|
|
1559
1865
|
: [];
|
|
1560
1866
|
|
|
1561
|
-
// Look up permission grant for MessagesSubscribe if using a delegate.
|
|
1562
|
-
// The unified scope matching in AgentPermissionsApi accepts a
|
|
1563
|
-
// Messages.Read grant for MessagesSubscribe requests, so a single
|
|
1564
|
-
// lookup is sufficient.
|
|
1565
|
-
let permissionGrantId: string | undefined;
|
|
1566
|
-
if (delegateDid) {
|
|
1567
|
-
const grant = await this._permissionsApi.getPermissionForRequest({
|
|
1568
|
-
connectedDid : did,
|
|
1569
|
-
messageType : DwnInterface.MessagesSubscribe,
|
|
1570
|
-
delegateDid,
|
|
1571
|
-
protocol,
|
|
1572
|
-
cached : true
|
|
1573
|
-
});
|
|
1574
|
-
permissionGrantId = grant.grant.id;
|
|
1575
|
-
}
|
|
1576
|
-
|
|
1577
1867
|
const handlerGeneration = this._engineGeneration;
|
|
1578
1868
|
|
|
1579
1869
|
// Define the subscription handler that processes incoming events.
|
|
@@ -1582,249 +1872,20 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1582
1872
|
// ensures the checkpoint advances only when all earlier deliveries are committed.
|
|
1583
1873
|
// Capture the link reference at subscription-open time so we can
|
|
1584
1874
|
// detect remove+re-add via object identity, not just key existence.
|
|
1585
|
-
const
|
|
1586
|
-
const
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1875
|
+
const isStale = this.createLinkStalePredicate(cursorKey, link, handlerGeneration);
|
|
1876
|
+
const pullContext: LivePullContext = {
|
|
1877
|
+
did,
|
|
1878
|
+
dwnUrl,
|
|
1879
|
+
delegateDid,
|
|
1880
|
+
eventScope,
|
|
1881
|
+
linkKey : cursorKey,
|
|
1882
|
+
link,
|
|
1883
|
+
permissionGrantIds : target.permissionGrantIds,
|
|
1884
|
+
isStale,
|
|
1885
|
+
};
|
|
1590
1886
|
|
|
1591
1887
|
const subscriptionHandler = async (subMessage: SubscriptionMessage): Promise<void> => {
|
|
1592
|
-
|
|
1593
|
-
return;
|
|
1594
|
-
}
|
|
1595
|
-
|
|
1596
|
-
if (subMessage.type === 'eose') {
|
|
1597
|
-
// End-of-stored-events — catch-up complete.
|
|
1598
|
-
if (link) {
|
|
1599
|
-
// Guard: if the link transitioned to repairing while catch-up events
|
|
1600
|
-
// were being processed, skip all mutations — repair owns the state now.
|
|
1601
|
-
if (link.status !== 'live' && link.status !== 'initializing') {
|
|
1602
|
-
return;
|
|
1603
|
-
}
|
|
1604
|
-
|
|
1605
|
-
if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1606
|
-
console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1607
|
-
if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
|
|
1608
|
-
return;
|
|
1609
|
-
}
|
|
1610
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1611
|
-
this.drainCommittedPull(cursorKey);
|
|
1612
|
-
if (isStale()) { return; }
|
|
1613
|
-
await this.ledger.saveLink(link);
|
|
1614
|
-
}
|
|
1615
|
-
// Transport is reachable — set connectivity to online.
|
|
1616
|
-
if (link) {
|
|
1617
|
-
const prevEoseConnectivity = link.connectivity;
|
|
1618
|
-
link.connectivity = 'online';
|
|
1619
|
-
if (prevEoseConnectivity !== 'online') {
|
|
1620
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevEoseConnectivity, to: 'online' });
|
|
1621
|
-
}
|
|
1622
|
-
// If the link was marked dirty, schedule reconciliation now that it's healthy.
|
|
1623
|
-
if (link.needsReconcile) {
|
|
1624
|
-
this.scheduleReconcile(cursorKey, 500);
|
|
1625
|
-
}
|
|
1626
|
-
} else {
|
|
1627
|
-
this._connectivityState = 'online';
|
|
1628
|
-
}
|
|
1629
|
-
return;
|
|
1630
|
-
}
|
|
1631
|
-
|
|
1632
|
-
if (subMessage.type === 'event') {
|
|
1633
|
-
const event: MessageEvent = subMessage.event;
|
|
1634
|
-
|
|
1635
|
-
// Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
|
|
1636
|
-
// skip all processing. Old subscription handlers may still fire after the
|
|
1637
|
-
// link transitions — these events should be ignored entirely, not just
|
|
1638
|
-
// skipped at the checkpoint level.
|
|
1639
|
-
if (link && link.status !== 'live' && link.status !== 'initializing') {
|
|
1640
|
-
return;
|
|
1641
|
-
}
|
|
1642
|
-
|
|
1643
|
-
// Domain validation: reject tokens from a different stream/epoch.
|
|
1644
|
-
if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1645
|
-
console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1646
|
-
if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
|
|
1647
|
-
return;
|
|
1648
|
-
}
|
|
1649
|
-
|
|
1650
|
-
// Subset scope filtering: if the link has protocolPath/contextId prefixes,
|
|
1651
|
-
// skip events that don't match. This is agent-side filtering because
|
|
1652
|
-
// MessagesSubscribe only supports protocol-level filtering today.
|
|
1653
|
-
//
|
|
1654
|
-
// Skipped events MUST advance contiguousAppliedToken — otherwise the
|
|
1655
|
-
// link would replay the same filtered-out events indefinitely after
|
|
1656
|
-
// reconnect/repair. This is safe because the event is intentionally
|
|
1657
|
-
// excluded from this scope and doesn't need processing.
|
|
1658
|
-
if (link && !isEventInScope(event.message, link.scope)) {
|
|
1659
|
-
if (!isStale()) {
|
|
1660
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1661
|
-
ReplicationLedger.commitContiguousToken(link.pull, subMessage.cursor);
|
|
1662
|
-
await this.ledger.saveLink(link);
|
|
1663
|
-
}
|
|
1664
|
-
return;
|
|
1665
|
-
}
|
|
1666
|
-
|
|
1667
|
-
// Assign a delivery ordinal BEFORE async processing begins.
|
|
1668
|
-
// This captures the delivery order even if processing completes out of order.
|
|
1669
|
-
const rt = link ? this.getOrCreateRuntime(cursorKey) : undefined;
|
|
1670
|
-
const ordinal = rt ? rt.nextDeliveryOrdinal++ : -1;
|
|
1671
|
-
if (rt) {
|
|
1672
|
-
rt.inflight.set(ordinal, { ordinal, token: subMessage.cursor, committed: false });
|
|
1673
|
-
}
|
|
1674
|
-
|
|
1675
|
-
try {
|
|
1676
|
-
// Extract inline data from the event (available for records <= 30 KB).
|
|
1677
|
-
let dataStream = this.extractDataStream(event);
|
|
1678
|
-
|
|
1679
|
-
// For large RecordsWrite messages (no inline data), fetch the data
|
|
1680
|
-
// from the remote DWN via MessagesRead before storing locally.
|
|
1681
|
-
if (!dataStream && isRecordsWrite(event) && (event.message.descriptor as any).dataCid) {
|
|
1682
|
-
const messageCid = await Message.getCid(event.message);
|
|
1683
|
-
const fetched = await fetchRemoteMessages({
|
|
1684
|
-
did, dwnUrl, delegateDid, protocol,
|
|
1685
|
-
messageCids : [messageCid],
|
|
1686
|
-
agent : this.agent,
|
|
1687
|
-
permissionsApi : this._permissionsApi,
|
|
1688
|
-
});
|
|
1689
|
-
if (fetched.length > 0 && fetched[0].dataStream) {
|
|
1690
|
-
dataStream = fetched[0].dataStream;
|
|
1691
|
-
}
|
|
1692
|
-
}
|
|
1693
|
-
|
|
1694
|
-
await this.agent.dwn.processRawMessage(did, event.message, { dataStream });
|
|
1695
|
-
if (isStale()) { return; }
|
|
1696
|
-
|
|
1697
|
-
// Invalidate closure cache entries that may be affected by this message.
|
|
1698
|
-
// Must run before closure validation so subsequent evaluations in the
|
|
1699
|
-
// same session see the updated local state.
|
|
1700
|
-
const closureCtxForInvalidation = this._closureContexts.get(did);
|
|
1701
|
-
if (closureCtxForInvalidation) {
|
|
1702
|
-
invalidateClosureCache(closureCtxForInvalidation, event.message);
|
|
1703
|
-
}
|
|
1704
|
-
|
|
1705
|
-
// Closure validation for scoped subset sync (Phase 3).
|
|
1706
|
-
// For protocol-scoped links, verify that all hard dependencies for
|
|
1707
|
-
// this operation are locally present before considering it committed.
|
|
1708
|
-
// Full-tenant scope bypasses this entirely (returns complete with 0 queries).
|
|
1709
|
-
if (link?.scope.kind === 'protocol') {
|
|
1710
|
-
const messageStore = this.agent.dwn.node.storage.messageStore;
|
|
1711
|
-
let closureCtx = this._closureContexts.get(did);
|
|
1712
|
-
if (!closureCtx) {
|
|
1713
|
-
closureCtx = createClosureContext(did, undefined, {
|
|
1714
|
-
isDelegateSession: !!delegateDid,
|
|
1715
|
-
});
|
|
1716
|
-
this._closureContexts.set(did, closureCtx);
|
|
1717
|
-
}
|
|
1718
|
-
|
|
1719
|
-
const closureResult = await evaluateClosure(
|
|
1720
|
-
event.message, messageStore, link.scope, closureCtx
|
|
1721
|
-
);
|
|
1722
|
-
|
|
1723
|
-
if (isStale()) { return; }
|
|
1724
|
-
|
|
1725
|
-
if (!closureResult.complete) {
|
|
1726
|
-
const failureCode = closureResult.failure!.code;
|
|
1727
|
-
const failureDetail = closureResult.failure!.detail;
|
|
1728
|
-
console.warn(
|
|
1729
|
-
`SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
|
|
1730
|
-
`${failureCode} — ${failureDetail}`
|
|
1731
|
-
);
|
|
1732
|
-
|
|
1733
|
-
// Record the message that triggered the closure failure.
|
|
1734
|
-
const closureCid = await Message.getCid(event.message);
|
|
1735
|
-
void this.recordDeadLetter({
|
|
1736
|
-
messageCid : closureCid,
|
|
1737
|
-
tenantDid : did,
|
|
1738
|
-
remoteEndpoint : dwnUrl,
|
|
1739
|
-
protocol,
|
|
1740
|
-
category : 'closure',
|
|
1741
|
-
errorCode : failureCode,
|
|
1742
|
-
errorDetail : failureDetail,
|
|
1743
|
-
});
|
|
1744
|
-
|
|
1745
|
-
if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
|
|
1746
|
-
return;
|
|
1747
|
-
}
|
|
1748
|
-
}
|
|
1749
|
-
|
|
1750
|
-
// Squash convergence: processRawMessage triggers the DWN's built-in
|
|
1751
|
-
// squash resumable task (performRecordsSquash) which runs inline and
|
|
1752
|
-
// handles subset consumers correctly:
|
|
1753
|
-
// - If older siblings are locally present → purges them
|
|
1754
|
-
// - If squash arrives before older siblings → backstop rejects them (409)
|
|
1755
|
-
// - If no older siblings are local → no-op (correct)
|
|
1756
|
-
// Both sync orderings (squash-first or siblings-first) converge to
|
|
1757
|
-
// the same final state. No additional sync-engine side-effect is needed.
|
|
1758
|
-
|
|
1759
|
-
// Track this CID for echo-loop suppression, scoped to the source endpoint.
|
|
1760
|
-
const pulledCid = await Message.getCid(event.message);
|
|
1761
|
-
this._recentlyPulledCids.set(`${pulledCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
|
|
1762
|
-
this.evictExpiredEchoEntries();
|
|
1763
|
-
|
|
1764
|
-
// Auto-clear any dead letter for this CID — it was processed
|
|
1765
|
-
// successfully, so a previous failure has been self-healed.
|
|
1766
|
-
this.clearFailedMessage(pulledCid, dwnUrl).catch(() => { /* teardown race */ });
|
|
1767
|
-
|
|
1768
|
-
// Mark this ordinal as committed and drain the checkpoint.
|
|
1769
|
-
// Guard: if the link transitioned to repairing while this handler was
|
|
1770
|
-
// in-flight (e.g., an earlier ordinal's handler failed concurrently),
|
|
1771
|
-
// skip all state mutations — the repair process owns progression now.
|
|
1772
|
-
if (link && rt && link.status === 'live' && !isStale()) {
|
|
1773
|
-
const entry = rt.inflight.get(ordinal);
|
|
1774
|
-
if (entry) { entry.committed = true; }
|
|
1775
|
-
|
|
1776
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1777
|
-
const drained = this.drainCommittedPull(cursorKey);
|
|
1778
|
-
if (drained > 0) {
|
|
1779
|
-
await this.ledger.saveLink(link);
|
|
1780
|
-
// Emit after durable save — "advanced" means persisted.
|
|
1781
|
-
if (link.pull.contiguousAppliedToken) {
|
|
1782
|
-
this.emitEvent({
|
|
1783
|
-
type : 'checkpoint:pull-advance',
|
|
1784
|
-
tenantDid : link.tenantDid,
|
|
1785
|
-
remoteEndpoint : link.remoteEndpoint,
|
|
1786
|
-
protocol : link.protocol,
|
|
1787
|
-
position : link.pull.contiguousAppliedToken.position,
|
|
1788
|
-
messageCid : link.pull.contiguousAppliedToken.messageCid,
|
|
1789
|
-
});
|
|
1790
|
-
}
|
|
1791
|
-
}
|
|
1792
|
-
|
|
1793
|
-
// Overflow: too many in-flight ordinals without draining.
|
|
1794
|
-
if (rt.inflight.size > MAX_PENDING_TOKENS) {
|
|
1795
|
-
console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1796
|
-
await this.transitionToRepairing(cursorKey, link);
|
|
1797
|
-
}
|
|
1798
|
-
}
|
|
1799
|
-
} catch (error: any) {
|
|
1800
|
-
console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
|
|
1801
|
-
|
|
1802
|
-
// Record the failing message in the dead letter store before
|
|
1803
|
-
// transitioning to repair. The CID identifies which specific
|
|
1804
|
-
// message caused the transition.
|
|
1805
|
-
try {
|
|
1806
|
-
const failedCid = await Message.getCid(event.message);
|
|
1807
|
-
void this.recordDeadLetter({
|
|
1808
|
-
messageCid : failedCid,
|
|
1809
|
-
tenantDid : did,
|
|
1810
|
-
remoteEndpoint : dwnUrl,
|
|
1811
|
-
protocol,
|
|
1812
|
-
category : 'pull-processing',
|
|
1813
|
-
errorDetail : error.message ?? String(error),
|
|
1814
|
-
});
|
|
1815
|
-
} catch {
|
|
1816
|
-
// Best effort — don't let dead letter recording block repair.
|
|
1817
|
-
}
|
|
1818
|
-
|
|
1819
|
-
// A failed processRawMessage means local state is incomplete.
|
|
1820
|
-
// Transition to repairing immediately — do NOT advance the checkpoint
|
|
1821
|
-
// past this failure or let later ordinals commit past it. SMT
|
|
1822
|
-
// reconciliation will discover and fill the gap.
|
|
1823
|
-
if (link && !isStale()) {
|
|
1824
|
-
await this.transitionToRepairing(cursorKey, link);
|
|
1825
|
-
}
|
|
1826
|
-
}
|
|
1827
|
-
}
|
|
1888
|
+
await this.handleLivePullMessage(pullContext, subMessage);
|
|
1828
1889
|
};
|
|
1829
1890
|
|
|
1830
1891
|
// Construct the subscribe message and send it directly to the specific
|
|
@@ -1837,7 +1898,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1837
1898
|
target : did,
|
|
1838
1899
|
messageType : DwnInterface.MessagesSubscribe as const,
|
|
1839
1900
|
granteeDid : delegateDid,
|
|
1840
|
-
messageParams : { filters, cursor,
|
|
1901
|
+
messageParams : { filters, cursor, permissionGrantIds: toMessagesPermissionGrantIds(target.permissionGrantIds) },
|
|
1841
1902
|
};
|
|
1842
1903
|
|
|
1843
1904
|
const { message } = await this.agent.dwn.processRequest(subscribeRequest);
|
|
@@ -1890,13 +1951,14 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1890
1951
|
throw new Error(`SyncEngineLevel: MessagesSubscribe failed for ${did} -> ${dwnUrl}: ${reply.status.code} ${reply.status.detail}`);
|
|
1891
1952
|
}
|
|
1892
1953
|
|
|
1954
|
+
const linkKey = cursorKey;
|
|
1955
|
+
const close = async (): Promise<void> => { await reply.subscription!.close(); };
|
|
1893
1956
|
this._liveSubscriptions.push({
|
|
1894
|
-
linkKey
|
|
1957
|
+
linkKey,
|
|
1895
1958
|
did,
|
|
1896
1959
|
dwnUrl,
|
|
1897
1960
|
delegateDid,
|
|
1898
|
-
|
|
1899
|
-
close : async (): Promise<void> => { await reply.subscription!.close(); },
|
|
1961
|
+
close,
|
|
1900
1962
|
});
|
|
1901
1963
|
|
|
1902
1964
|
// Set per-link connectivity to online after successful subscription setup.
|
|
@@ -1905,301 +1967,2020 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1905
1967
|
const prevPullConnectivity = pullLink.connectivity;
|
|
1906
1968
|
pullLink.connectivity = 'online';
|
|
1907
1969
|
if (prevPullConnectivity !== 'online') {
|
|
1908
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1970
|
+
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: prevPullConnectivity, to: 'online' });
|
|
1909
1971
|
}
|
|
1910
1972
|
}
|
|
1911
1973
|
}
|
|
1912
1974
|
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1975
|
+
private async getInitialPullCursor({ did, dwnUrl, link }: {
|
|
1976
|
+
did: string;
|
|
1977
|
+
dwnUrl: string;
|
|
1978
|
+
link?: ReplicationLinkState;
|
|
1979
|
+
}): Promise<ProgressToken | undefined> {
|
|
1980
|
+
// Resolve the cursor from the link's durable pull checkpoint.
|
|
1981
|
+
if (!link) {
|
|
1982
|
+
return undefined;
|
|
1983
|
+
}
|
|
1916
1984
|
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1985
|
+
const cursor = link.pull.contiguousAppliedToken;
|
|
1986
|
+
if (!cursor || this.isValidProgressToken(cursor)) {
|
|
1987
|
+
return cursor;
|
|
1988
|
+
}
|
|
1989
|
+
|
|
1990
|
+
// Guard against corrupted tokens with empty fields — these would fail
|
|
1991
|
+
// MessagesSubscribe JSON schema validation (minLength: 1). Discard and
|
|
1992
|
+
// start from the beginning rather than crash the subscription.
|
|
1993
|
+
console.warn(`SyncEngineLevel: Discarding stored cursor with empty field(s) for ${did} -> ${dwnUrl}`);
|
|
1994
|
+
ReplicationLedger.resetCheckpoint(link.pull);
|
|
1995
|
+
await this.ledger.saveLink(link);
|
|
1996
|
+
return undefined;
|
|
1997
|
+
}
|
|
1998
|
+
|
|
1999
|
+
private isValidProgressToken(token: ProgressToken): boolean {
|
|
2000
|
+
return !!(token.streamId && token.messageCid && token.epoch && token.position);
|
|
2001
|
+
}
|
|
2002
|
+
|
|
2003
|
+
private createLinkStalePredicate(
|
|
2004
|
+
linkKey: string,
|
|
2005
|
+
capturedLink: ReplicationLinkState | undefined,
|
|
2006
|
+
generation: number,
|
|
2007
|
+
): () => boolean {
|
|
2008
|
+
return (): boolean =>
|
|
2009
|
+
this._engineGeneration !== generation ||
|
|
2010
|
+
!this._activeLinks.has(linkKey) ||
|
|
2011
|
+
(capturedLink !== undefined && this._activeLinks.get(linkKey) !== capturedLink);
|
|
2012
|
+
}
|
|
2013
|
+
|
|
2014
|
+
private async handleLivePullMessage(context: LivePullContext, subMessage: SubscriptionMessage): Promise<void> {
|
|
2015
|
+
if (context.isStale()) {
|
|
2016
|
+
return;
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
if (subMessage.type === 'eose') {
|
|
2020
|
+
await this.handleLivePullEose(context, subMessage);
|
|
2021
|
+
return;
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
if (subMessage.type === 'error') {
|
|
2025
|
+
await this.handleLivePullSubscriptionError(context, subMessage);
|
|
2026
|
+
return;
|
|
2027
|
+
}
|
|
2028
|
+
|
|
2029
|
+
if (subMessage.type === 'event') {
|
|
2030
|
+
await this.handleLivePullEvent(context, subMessage);
|
|
2031
|
+
}
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
private async handleLivePullEose(
|
|
2035
|
+
{ did, dwnUrl, eventScope, linkKey, link, isStale }: LivePullContext,
|
|
2036
|
+
subMessage: Extract<SubscriptionMessage, { type: 'eose' }>,
|
|
2037
|
+
): Promise<void> {
|
|
2038
|
+
if (link) {
|
|
2039
|
+
// Guard: if the link transitioned to repairing while catch-up events
|
|
2040
|
+
// were being processed, skip all mutations — repair owns the state now.
|
|
2041
|
+
if (link.status !== 'live' && link.status !== 'initializing') {
|
|
2042
|
+
return;
|
|
2043
|
+
}
|
|
2044
|
+
|
|
2045
|
+
if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
2046
|
+
console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
2047
|
+
if (!isStale()) { await this.transitionToRepairing(linkKey, link); }
|
|
2048
|
+
return;
|
|
2049
|
+
}
|
|
2050
|
+
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
2051
|
+
this.drainCommittedPull(linkKey);
|
|
2052
|
+
if (isStale()) { return; }
|
|
2053
|
+
await this.ledger.saveLink(link);
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
this.markPullLinkOnline({ did, dwnUrl, eventScope, linkKey, link });
|
|
2057
|
+
}
|
|
2058
|
+
|
|
2059
|
+
private markPullLinkOnline({ did, dwnUrl, eventScope, linkKey, link }: {
|
|
2060
|
+
did: string;
|
|
2061
|
+
dwnUrl: string;
|
|
2062
|
+
eventScope: SyncEventScope;
|
|
1923
2063
|
linkKey: string;
|
|
2064
|
+
link?: ReplicationLinkState;
|
|
2065
|
+
}): void {
|
|
2066
|
+
if (!link) {
|
|
2067
|
+
this._connectivityState = 'online';
|
|
2068
|
+
return;
|
|
2069
|
+
}
|
|
2070
|
+
|
|
2071
|
+
const previous = link.connectivity;
|
|
2072
|
+
link.connectivity = 'online';
|
|
2073
|
+
if (previous !== 'online') {
|
|
2074
|
+
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: previous, to: 'online' });
|
|
2075
|
+
}
|
|
2076
|
+
if (link.needsReconcile) {
|
|
2077
|
+
this.scheduleReconcile(linkKey, 500);
|
|
2078
|
+
}
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
private async handleLivePullSubscriptionError(
|
|
2082
|
+
{ did, dwnUrl, linkKey, link, isStale }: LivePullContext,
|
|
2083
|
+
subMessage: Extract<SubscriptionMessage, { type: 'error' }>,
|
|
2084
|
+
): Promise<void> {
|
|
2085
|
+
console.warn(`SyncEngineLevel: subscription error for ${did} -> ${dwnUrl}: ${subMessage.error.code}`);
|
|
2086
|
+
|
|
2087
|
+
if (link && !isStale()) {
|
|
2088
|
+
await this.transitionToRepairing(linkKey, link);
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
private async handleLivePullEvent(
|
|
2093
|
+
context: LivePullContext,
|
|
2094
|
+
subMessage: Extract<SubscriptionMessage, { type: 'event' }>,
|
|
2095
|
+
): Promise<void> {
|
|
2096
|
+
const event = subMessage.event;
|
|
2097
|
+
if (await this.shouldSkipLivePullEvent(context, subMessage)) {
|
|
2098
|
+
return;
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
const delivery = this.startPullDelivery(context, subMessage.cursor);
|
|
2102
|
+
try {
|
|
2103
|
+
const pulledCid = await this.processLivePullEvent(context, event);
|
|
2104
|
+
if (!pulledCid) { return; }
|
|
2105
|
+
|
|
2106
|
+
this.trackRecentlyPulledMessage(pulledCid, context.dwnUrl);
|
|
2107
|
+
this.clearFailedMessage(pulledCid, context.dwnUrl).catch(() => { /* teardown race */ });
|
|
2108
|
+
await this.commitPullDelivery(context, subMessage.cursor, delivery);
|
|
2109
|
+
} catch (error: any) {
|
|
2110
|
+
await this.handleLivePullProcessingError(context, event, error);
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
|
|
2114
|
+
private async shouldSkipLivePullEvent(
|
|
2115
|
+
{ did, dwnUrl, linkKey, link, isStale }: LivePullContext,
|
|
2116
|
+
subMessage: Extract<SubscriptionMessage, { type: 'event' }>,
|
|
2117
|
+
): Promise<boolean> {
|
|
2118
|
+
// Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
|
|
2119
|
+
// skip all processing. Old subscription handlers may still fire after the
|
|
2120
|
+
// link transitions — these events should be ignored entirely, not just
|
|
2121
|
+
// skipped at the checkpoint level.
|
|
2122
|
+
if (link && link.status !== 'live' && link.status !== 'initializing') {
|
|
2123
|
+
return true;
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
// Domain validation: reject tokens from a different stream/epoch.
|
|
2127
|
+
if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
2128
|
+
console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
2129
|
+
if (!isStale()) { await this.transitionToRepairing(linkKey, link); }
|
|
2130
|
+
return true;
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2133
|
+
if (link) {
|
|
2134
|
+
const scopeClassification = classifySyncEventScope(subMessage.event, link.scope);
|
|
2135
|
+
if (scopeClassification === 'out-of-scope') {
|
|
2136
|
+
await this.skipOutOfScopePullEvent({ link, cursor: subMessage.cursor, isStale });
|
|
2137
|
+
return true;
|
|
2138
|
+
}
|
|
2139
|
+
if (scopeClassification === 'unknown') {
|
|
2140
|
+
console.warn(`SyncEngineLevel: Unable to classify scoped pull event for ${did} -> ${dwnUrl}, transitioning to repair`);
|
|
2141
|
+
if (!isStale()) { await this.transitionToRepairing(linkKey, link); }
|
|
2142
|
+
return true;
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
return false;
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
private async skipOutOfScopePullEvent({ link, cursor, isStale }: {
|
|
2150
|
+
link: ReplicationLinkState;
|
|
2151
|
+
cursor: ProgressToken;
|
|
2152
|
+
isStale: () => boolean;
|
|
1924
2153
|
}): Promise<void> {
|
|
1925
|
-
|
|
2154
|
+
// Skipped events MUST advance contiguousAppliedToken — otherwise the link
|
|
2155
|
+
// would replay the same filtered-out events indefinitely after reconnect or
|
|
2156
|
+
// repair. This is safe because the event is intentionally excluded from
|
|
2157
|
+
// this scope and doesn't need processing.
|
|
2158
|
+
if (isStale()) { return; }
|
|
2159
|
+
|
|
2160
|
+
ReplicationLedger.setReceivedToken(link.pull, cursor);
|
|
2161
|
+
ReplicationLedger.commitContiguousToken(link.pull, cursor);
|
|
2162
|
+
await this.ledger.saveLink(link);
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
private startPullDelivery({ linkKey, link }: LivePullContext, cursor: ProgressToken): PullDelivery {
|
|
2166
|
+
// Assign a delivery ordinal BEFORE async processing begins. This captures
|
|
2167
|
+
// delivery order even if processing completes out of order.
|
|
2168
|
+
const runtime = link ? this.getOrCreateRuntime(linkKey) : undefined;
|
|
2169
|
+
const ordinal = runtime ? runtime.nextDeliveryOrdinal++ : -1;
|
|
2170
|
+
if (runtime) {
|
|
2171
|
+
runtime.inflight.set(ordinal, { ordinal, token: cursor, committed: false });
|
|
2172
|
+
}
|
|
2173
|
+
return { runtime, ordinal };
|
|
2174
|
+
}
|
|
2175
|
+
|
|
2176
|
+
private async processLivePullEvent(context: LivePullContext, event: MessageEvent): Promise<string | undefined> {
|
|
2177
|
+
const dataStreamFactory = await this.createLivePullDataStreamFactory(context, event);
|
|
2178
|
+
let applyStatus = await this.applyLivePullEvent(context, event, dataStreamFactory);
|
|
2179
|
+
if (context.isStale()) { return undefined; }
|
|
2180
|
+
|
|
2181
|
+
let applied = SyncEngineLevel.isApplySuccess(applyStatus.code);
|
|
2182
|
+
if (applied) {
|
|
2183
|
+
this.invalidateClosureCacheForMessage(context.did, event.message);
|
|
2184
|
+
}
|
|
2185
|
+
|
|
2186
|
+
if (!await this.ensureClosureComplete(context, event)) {
|
|
2187
|
+
return undefined;
|
|
2188
|
+
}
|
|
2189
|
+
|
|
2190
|
+
if (!applied) {
|
|
2191
|
+
applyStatus = await this.applyLivePullEvent(context, event, dataStreamFactory);
|
|
2192
|
+
if (context.isStale()) { return undefined; }
|
|
2193
|
+
|
|
2194
|
+
applied = SyncEngineLevel.isApplySuccess(applyStatus.code);
|
|
2195
|
+
if (!applied) {
|
|
2196
|
+
throw await this.createLivePullApplyError(event, applyStatus);
|
|
2197
|
+
}
|
|
2198
|
+
this.invalidateClosureCacheForMessage(context.did, event.message);
|
|
2199
|
+
}
|
|
2200
|
+
|
|
2201
|
+
// Squash convergence: processRawMessage triggers the DWN's built-in
|
|
2202
|
+
// squash resumable task (performRecordsSquash), so no additional
|
|
2203
|
+
// sync-engine side effect is needed here.
|
|
2204
|
+
return Message.getCid(event.message);
|
|
2205
|
+
}
|
|
2206
|
+
|
|
2207
|
+
private async applyLivePullEvent(
|
|
2208
|
+
context: LivePullContext,
|
|
2209
|
+
event: MessageEvent,
|
|
2210
|
+
dataStreamFactory: LivePullDataStreamFactory,
|
|
2211
|
+
): Promise<ApplyStatus> {
|
|
2212
|
+
const dataStream = await dataStreamFactory();
|
|
2213
|
+
const reply = await this.agent.dwn.processRawMessage(context.did, event.message, { dataStream });
|
|
2214
|
+
return reply.status;
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2217
|
+
private async createLivePullDataStreamFactory(
|
|
2218
|
+
context: LivePullContext,
|
|
2219
|
+
event: MessageEvent,
|
|
2220
|
+
): Promise<LivePullDataStreamFactory> {
|
|
2221
|
+
if (!isRecordsWrite(event)) {
|
|
2222
|
+
return async () => undefined;
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
const encodedData = (event.message as any).encodedData as string | undefined;
|
|
2226
|
+
if (encodedData) {
|
|
2227
|
+
delete (event.message as any).encodedData;
|
|
2228
|
+
const bytes = Encoder.base64UrlToBytes(encodedData);
|
|
2229
|
+
return async () => SyncEngineLevel.dataStreamFromBytes(bytes);
|
|
2230
|
+
}
|
|
2231
|
+
|
|
2232
|
+
const eventData = (event as any).data as ReadableStream<Uint8Array> | undefined;
|
|
2233
|
+
if (eventData) {
|
|
2234
|
+
const bytes = await SyncEngineLevel.readStreamBytes(eventData);
|
|
2235
|
+
return async () => SyncEngineLevel.dataStreamFromBytes(bytes);
|
|
2236
|
+
}
|
|
2237
|
+
|
|
2238
|
+
if (!(event.message.descriptor as any).dataCid) {
|
|
2239
|
+
return async () => undefined;
|
|
2240
|
+
}
|
|
2241
|
+
|
|
2242
|
+
// For large RecordsWrite messages (no inline data), fetch the data from
|
|
2243
|
+
// the remote DWN via MessagesRead before each store attempt. ReadableStream
|
|
2244
|
+
// instances are single-use, so a repair-triggered retry needs a fresh fetch.
|
|
2245
|
+
const { did, dwnUrl, delegateDid, permissionGrantIds } = context;
|
|
2246
|
+
const messageCid = await Message.getCid(event.message);
|
|
2247
|
+
return async () => {
|
|
2248
|
+
const fetched = await fetchRemoteMessages({
|
|
2249
|
+
did,
|
|
2250
|
+
dwnUrl,
|
|
2251
|
+
delegateDid,
|
|
2252
|
+
permissionGrantIds,
|
|
2253
|
+
messageCids : [messageCid],
|
|
2254
|
+
agent : this.agent,
|
|
2255
|
+
});
|
|
2256
|
+
return fetched[0]?.dataStream;
|
|
2257
|
+
};
|
|
2258
|
+
}
|
|
2259
|
+
|
|
2260
|
+
private async createLivePullApplyError(event: MessageEvent, status: ApplyStatus): Promise<Error> {
|
|
2261
|
+
const cid = await Message.getCid(event.message);
|
|
2262
|
+
return new Error(
|
|
2263
|
+
`SyncEngineLevel: live pull apply failed for ${cid}: ${status.code} ${status.detail ?? ''}`.trim()
|
|
2264
|
+
);
|
|
2265
|
+
}
|
|
2266
|
+
|
|
2267
|
+
private static dataStreamFromBytes(bytes: Uint8Array): ReadableStream<Uint8Array> {
|
|
2268
|
+
return new ReadableStream<Uint8Array>({
|
|
2269
|
+
start(controller): void {
|
|
2270
|
+
controller.enqueue(bytes);
|
|
2271
|
+
controller.close();
|
|
2272
|
+
}
|
|
2273
|
+
});
|
|
2274
|
+
}
|
|
2275
|
+
|
|
2276
|
+
private static async readStreamBytes(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
|
|
2277
|
+
const reader = stream.getReader();
|
|
2278
|
+
const chunks: Uint8Array[] = [];
|
|
2279
|
+
let totalSize = 0;
|
|
2280
|
+
|
|
2281
|
+
try {
|
|
2282
|
+
for (;;) {
|
|
2283
|
+
const { done, value } = await reader.read();
|
|
2284
|
+
if (done) { break; }
|
|
2285
|
+
chunks.push(value);
|
|
2286
|
+
totalSize += value.byteLength;
|
|
2287
|
+
}
|
|
2288
|
+
} finally {
|
|
2289
|
+
reader.releaseLock();
|
|
2290
|
+
}
|
|
2291
|
+
|
|
2292
|
+
const bytes = new Uint8Array(totalSize);
|
|
2293
|
+
let offset = 0;
|
|
2294
|
+
for (const chunk of chunks) {
|
|
2295
|
+
bytes.set(chunk, offset);
|
|
2296
|
+
offset += chunk.byteLength;
|
|
2297
|
+
}
|
|
2298
|
+
return bytes;
|
|
2299
|
+
}
|
|
2300
|
+
|
|
2301
|
+
private static isApplySuccess(code: number): boolean {
|
|
2302
|
+
return (code >= 200 && code < 300) || code === 409;
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
private invalidateClosureCacheForMessage(did: string, message: GenericMessage): void {
|
|
2306
|
+
// Must run before closure validation so subsequent evaluations in the same
|
|
2307
|
+
// session see the updated local state.
|
|
2308
|
+
const closureCtx = this._closureContexts.get(did);
|
|
2309
|
+
if (closureCtx) {
|
|
2310
|
+
invalidateClosureCache(closureCtx, message);
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
|
|
2314
|
+
private async ensureClosureComplete(context: LivePullContext, event: MessageEvent): Promise<boolean> {
|
|
2315
|
+
const { did, delegateDid, link, isStale } = context;
|
|
2316
|
+
if (!link || link.scope.kind === 'full') {
|
|
2317
|
+
return true;
|
|
2318
|
+
}
|
|
2319
|
+
|
|
2320
|
+
let closureCtx = this._closureContexts.get(did);
|
|
2321
|
+
if (!closureCtx) {
|
|
2322
|
+
closureCtx = createClosureContext(did, undefined, {
|
|
2323
|
+
isDelegateSession: !!delegateDid,
|
|
2324
|
+
});
|
|
2325
|
+
this._closureContexts.set(did, closureCtx);
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
const messageStore = this.agent.dwn.node.storage.messageStore;
|
|
2329
|
+
let closureResult = await evaluateClosure(event.message, messageStore, link.scope, closureCtx);
|
|
2330
|
+
if (isStale()) { return false; }
|
|
2331
|
+
if (closureResult.complete) { return true; }
|
|
2332
|
+
|
|
2333
|
+
if (await this.tryRepairMissingProtocolMetadata(context, closureCtx, closureResult)) {
|
|
2334
|
+
if (isStale()) { return false; }
|
|
2335
|
+
|
|
2336
|
+
closureResult = await evaluateClosure(event.message, messageStore, link.scope, closureCtx);
|
|
2337
|
+
if (isStale()) { return false; }
|
|
2338
|
+
if (closureResult.complete) { return true; }
|
|
2339
|
+
}
|
|
2340
|
+
|
|
2341
|
+
await this.recordClosureFailure(context, event, closureResult.failure!.code, closureResult.failure!.detail);
|
|
2342
|
+
return false;
|
|
2343
|
+
}
|
|
2344
|
+
|
|
2345
|
+
private async tryRepairMissingProtocolMetadata(
|
|
2346
|
+
context: LivePullContext,
|
|
2347
|
+
closureCtx: ClosureEvaluationContext,
|
|
2348
|
+
closureResult: ClosureResult,
|
|
2349
|
+
): Promise<boolean> {
|
|
2350
|
+
const failure = closureResult.failure;
|
|
2351
|
+
if (!SyncEngineLevel.isRepairableProtocolMetadataFailure(failure)) {
|
|
2352
|
+
return false;
|
|
2353
|
+
}
|
|
2354
|
+
|
|
2355
|
+
const { did } = context;
|
|
2356
|
+
const repairKey = `${did}|${failure.edge.identifier}`;
|
|
2357
|
+
const activeRepair = this._protocolMetadataRepairs.get(repairKey);
|
|
2358
|
+
if (activeRepair) {
|
|
2359
|
+
return activeRepair;
|
|
2360
|
+
}
|
|
2361
|
+
|
|
2362
|
+
const repair = this.repairMissingProtocolMetadata(context, closureCtx, failure.edge.identifier);
|
|
2363
|
+
this._protocolMetadataRepairs.set(repairKey, repair);
|
|
2364
|
+
repair.finally(() => {
|
|
2365
|
+
if (this._protocolMetadataRepairs.get(repairKey) === repair) {
|
|
2366
|
+
this._protocolMetadataRepairs.delete(repairKey);
|
|
2367
|
+
}
|
|
2368
|
+
}).catch(() => { /* caller handles the repair result */ });
|
|
2369
|
+
return repair;
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
private async repairMissingProtocolMetadata(
|
|
2373
|
+
{ did, dwnUrl, delegateDid, isStale }: LivePullContext,
|
|
2374
|
+
closureCtx: ClosureEvaluationContext,
|
|
2375
|
+
protocol: string,
|
|
2376
|
+
): Promise<boolean> {
|
|
2377
|
+
if (isStale()) {
|
|
2378
|
+
return false;
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
const configs = await this.fetchRemoteProtocolConfigClosure({
|
|
2382
|
+
authorDid : delegateDid ?? did,
|
|
2383
|
+
delegateDid,
|
|
2384
|
+
dwnUrl,
|
|
2385
|
+
protocol,
|
|
2386
|
+
tenantDid : did,
|
|
2387
|
+
});
|
|
2388
|
+
if (isStale() || configs.length === 0) {
|
|
2389
|
+
return false;
|
|
2390
|
+
}
|
|
2391
|
+
|
|
2392
|
+
// Live subscriptions can deliver scoped records before the local replica
|
|
2393
|
+
// has the tenant's protocol metadata. Reuse the DWN ProtocolsQuery path and
|
|
2394
|
+
// only install configs that are signed by the tenant, including composed
|
|
2395
|
+
// protocol dependencies needed to authorize the record.
|
|
2396
|
+
let repaired = false;
|
|
2397
|
+
for (const config of configs) {
|
|
2398
|
+
if (isStale()) {
|
|
2399
|
+
return repaired;
|
|
2400
|
+
}
|
|
2401
|
+
const reply = await this.agent.dwn.processRawMessage(did, config);
|
|
2402
|
+
if (isStale()) {
|
|
2403
|
+
return repaired;
|
|
2404
|
+
}
|
|
2405
|
+
if (!SyncEngineLevel.protocolConfigApplySucceeded(reply.status.code)) {
|
|
2406
|
+
return repaired;
|
|
2407
|
+
}
|
|
2408
|
+
|
|
2409
|
+
invalidateClosureCache(closureCtx, config);
|
|
2410
|
+
repaired = true;
|
|
2411
|
+
}
|
|
2412
|
+
|
|
2413
|
+
return repaired;
|
|
2414
|
+
}
|
|
2415
|
+
|
|
2416
|
+
private async fetchRemoteProtocolConfigClosure({
|
|
2417
|
+
authorDid,
|
|
2418
|
+
delegateDid,
|
|
2419
|
+
dwnUrl,
|
|
2420
|
+
protocol,
|
|
2421
|
+
tenantDid,
|
|
2422
|
+
}: {
|
|
2423
|
+
authorDid: string;
|
|
2424
|
+
delegateDid?: string;
|
|
2425
|
+
dwnUrl: string;
|
|
2426
|
+
protocol: string;
|
|
2427
|
+
tenantDid: string;
|
|
2428
|
+
}): Promise<ProtocolsConfigureMessage[]> {
|
|
2429
|
+
const configsByProtocol = new Map<string, ProtocolsConfigureMessage>();
|
|
2430
|
+
const visiting = new Set<string>();
|
|
2431
|
+
|
|
2432
|
+
const visit = async (protocolUri: string): Promise<boolean> => {
|
|
2433
|
+
if (configsByProtocol.has(protocolUri)) {
|
|
2434
|
+
return true;
|
|
2435
|
+
}
|
|
2436
|
+
if (visiting.has(protocolUri)) {
|
|
2437
|
+
return true;
|
|
2438
|
+
}
|
|
2439
|
+
visiting.add(protocolUri);
|
|
2440
|
+
|
|
2441
|
+
const config = await this.fetchRemoteProtocolConfig({
|
|
2442
|
+
authorDid,
|
|
2443
|
+
delegateDid,
|
|
2444
|
+
dwnUrl,
|
|
2445
|
+
protocol: protocolUri,
|
|
2446
|
+
tenantDid,
|
|
2447
|
+
});
|
|
2448
|
+
if (config === undefined) {
|
|
2449
|
+
visiting.delete(protocolUri);
|
|
2450
|
+
return false;
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
for (const usedProtocol of SyncEngineLevel.protocolsConfigureUses(config)) {
|
|
2454
|
+
if (!await visit(usedProtocol)) {
|
|
2455
|
+
visiting.delete(protocolUri);
|
|
2456
|
+
return false;
|
|
2457
|
+
}
|
|
2458
|
+
}
|
|
2459
|
+
|
|
2460
|
+
configsByProtocol.set(protocolUri, config);
|
|
2461
|
+
visiting.delete(protocolUri);
|
|
2462
|
+
return true;
|
|
2463
|
+
};
|
|
2464
|
+
|
|
2465
|
+
return await visit(protocol) ? [...configsByProtocol.values()] : [];
|
|
2466
|
+
}
|
|
2467
|
+
|
|
2468
|
+
private async fetchRemoteProtocolConfig({
|
|
2469
|
+
authorDid,
|
|
2470
|
+
delegateDid,
|
|
2471
|
+
dwnUrl,
|
|
2472
|
+
protocol,
|
|
2473
|
+
tenantDid,
|
|
2474
|
+
}: {
|
|
2475
|
+
authorDid: string;
|
|
2476
|
+
delegateDid?: string;
|
|
2477
|
+
dwnUrl: string;
|
|
2478
|
+
protocol: string;
|
|
2479
|
+
tenantDid: string;
|
|
2480
|
+
}): Promise<ProtocolsConfigureMessage | undefined> {
|
|
2481
|
+
try {
|
|
2482
|
+
const permissionGrantId = await this.getProtocolsQueryPermissionGrantId({
|
|
2483
|
+
delegateDid,
|
|
2484
|
+
protocol,
|
|
2485
|
+
tenantDid,
|
|
2486
|
+
});
|
|
2487
|
+
const { message } = await this.agent.processDwnRequest({
|
|
2488
|
+
author : authorDid,
|
|
2489
|
+
messageParams : {
|
|
2490
|
+
filter: { protocol },
|
|
2491
|
+
...(permissionGrantId === undefined ? {} : { permissionGrantId }),
|
|
2492
|
+
},
|
|
2493
|
+
messageType : DwnInterface.ProtocolsQuery,
|
|
2494
|
+
store : false,
|
|
2495
|
+
target : tenantDid,
|
|
2496
|
+
});
|
|
2497
|
+
|
|
2498
|
+
const reply = await this.agent.rpc.sendDwnRequest({
|
|
2499
|
+
dwnUrl,
|
|
2500
|
+
message,
|
|
2501
|
+
targetDid: tenantDid,
|
|
2502
|
+
}) as ProtocolsQueryReply;
|
|
2503
|
+
if (reply.status.code !== 200 || reply.entries === undefined) {
|
|
2504
|
+
return undefined;
|
|
2505
|
+
}
|
|
2506
|
+
|
|
2507
|
+
const candidates: ProtocolsConfigureMessage[] = [];
|
|
2508
|
+
for (const entry of reply.entries) {
|
|
2509
|
+
const config = await this.toAuthenticatedTenantProtocolConfig(tenantDid, entry);
|
|
2510
|
+
if (config?.descriptor.definition.protocol === protocol) {
|
|
2511
|
+
candidates.push(config);
|
|
2512
|
+
}
|
|
2513
|
+
}
|
|
2514
|
+
|
|
2515
|
+
return SyncEngineLevel.newestProtocolConfig(candidates);
|
|
2516
|
+
} catch {
|
|
2517
|
+
return undefined;
|
|
2518
|
+
}
|
|
2519
|
+
}
|
|
2520
|
+
|
|
2521
|
+
private async getProtocolsQueryPermissionGrantId({
|
|
2522
|
+
delegateDid,
|
|
2523
|
+
protocol,
|
|
2524
|
+
tenantDid,
|
|
2525
|
+
}: {
|
|
2526
|
+
delegateDid?: string;
|
|
2527
|
+
protocol: string;
|
|
2528
|
+
tenantDid: string;
|
|
2529
|
+
}): Promise<string | undefined> {
|
|
2530
|
+
if (delegateDid === undefined) {
|
|
2531
|
+
return undefined;
|
|
2532
|
+
}
|
|
2533
|
+
|
|
2534
|
+
try {
|
|
2535
|
+
const { grant } = await this._permissionsApi.getPermissionForRequest({
|
|
2536
|
+
connectedDid : tenantDid,
|
|
2537
|
+
delegateDid,
|
|
2538
|
+
protocol,
|
|
2539
|
+
cached : true,
|
|
2540
|
+
messageType : DwnInterface.ProtocolsQuery,
|
|
2541
|
+
});
|
|
2542
|
+
return grant.id;
|
|
2543
|
+
} catch {
|
|
2544
|
+
return undefined;
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
|
|
2548
|
+
private async toAuthenticatedTenantProtocolConfig(
|
|
2549
|
+
tenantDid: string,
|
|
2550
|
+
message: GenericMessage,
|
|
2551
|
+
): Promise<ProtocolsConfigureMessage | undefined> {
|
|
2552
|
+
if (!SyncEngineLevel.isProtocolsConfigureDefinitionMessage(message)) {
|
|
2553
|
+
return undefined;
|
|
2554
|
+
}
|
|
2555
|
+
|
|
2556
|
+
try {
|
|
2557
|
+
await ProtocolsConfigure.parse(message);
|
|
2558
|
+
if (Message.getAuthor(message) !== tenantDid) {
|
|
2559
|
+
return undefined;
|
|
2560
|
+
}
|
|
2561
|
+
await authenticate(message.authorization, this.agent.did);
|
|
2562
|
+
return message;
|
|
2563
|
+
} catch {
|
|
2564
|
+
return undefined;
|
|
2565
|
+
}
|
|
2566
|
+
}
|
|
2567
|
+
|
|
2568
|
+
private static newestProtocolConfig(
|
|
2569
|
+
configs: ProtocolsConfigureMessage[],
|
|
2570
|
+
): ProtocolsConfigureMessage | undefined {
|
|
2571
|
+
let newest: ProtocolsConfigureMessage | undefined;
|
|
2572
|
+
for (const config of configs) {
|
|
2573
|
+
if (newest === undefined || SyncEngineLevel.isProtocolConfigNewer(config, newest)) {
|
|
2574
|
+
newest = config;
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
return newest;
|
|
2578
|
+
}
|
|
2579
|
+
|
|
2580
|
+
private static isProtocolConfigNewer(
|
|
2581
|
+
candidate: ProtocolsConfigureMessage,
|
|
2582
|
+
current: ProtocolsConfigureMessage,
|
|
2583
|
+
): boolean {
|
|
2584
|
+
return candidate.descriptor.messageTimestamp > current.descriptor.messageTimestamp;
|
|
2585
|
+
}
|
|
2586
|
+
|
|
2587
|
+
private static protocolConfigApplySucceeded(code: number): boolean {
|
|
2588
|
+
return (code >= 200 && code < 300) || code === 409;
|
|
2589
|
+
}
|
|
2590
|
+
|
|
2591
|
+
private static isRepairableProtocolMetadataFailure(
|
|
2592
|
+
failure: ClosureResult['failure'] | undefined,
|
|
2593
|
+
): failure is NonNullable<ClosureResult['failure']> {
|
|
2594
|
+
return failure?.edge.identifierType === 'protocol' &&
|
|
2595
|
+
(
|
|
2596
|
+
failure.code === ClosureFailureCode.ProtocolMetadataMissing ||
|
|
2597
|
+
failure.code === ClosureFailureCode.CrossProtocolReferenceMissing ||
|
|
2598
|
+
failure.code === ClosureFailureCode.EncryptionDependencyMissing
|
|
2599
|
+
);
|
|
2600
|
+
}
|
|
2601
|
+
|
|
2602
|
+
private async recordClosureFailure(
|
|
2603
|
+
{ did, dwnUrl, linkKey, link, isStale }: LivePullContext,
|
|
2604
|
+
event: MessageEvent,
|
|
2605
|
+
failureCode: string,
|
|
2606
|
+
failureDetail: string,
|
|
2607
|
+
): Promise<void> {
|
|
2608
|
+
console.warn(
|
|
2609
|
+
`SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
|
|
2610
|
+
`${failureCode} — ${failureDetail}`
|
|
2611
|
+
);
|
|
2612
|
+
|
|
2613
|
+
const closureCid = await Message.getCid(event.message);
|
|
2614
|
+
void this.recordDeadLetter({
|
|
2615
|
+
messageCid : closureCid,
|
|
2616
|
+
tenantDid : did,
|
|
2617
|
+
remoteEndpoint : dwnUrl,
|
|
2618
|
+
protocol : (event.message.descriptor as Record<string, unknown>).protocol as string | undefined,
|
|
2619
|
+
category : 'closure',
|
|
2620
|
+
errorCode : failureCode,
|
|
2621
|
+
errorDetail : failureDetail,
|
|
2622
|
+
});
|
|
2623
|
+
|
|
2624
|
+
if (link && !isStale() && isTerminalClosureFailureCode(failureCode)) {
|
|
2625
|
+
await this.transitionToTerminalIncomplete(linkKey, link);
|
|
2626
|
+
return;
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2629
|
+
if (link && !isStale()) {
|
|
2630
|
+
await this.transitionToRepairing(linkKey, link);
|
|
2631
|
+
}
|
|
2632
|
+
}
|
|
2633
|
+
|
|
2634
|
+
private trackRecentlyPulledMessage(messageCid: string, dwnUrl: string): void {
|
|
2635
|
+
this._recentlyPulledCids.set(`${messageCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
|
|
2636
|
+
this.evictExpiredEchoEntries();
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2639
|
+
private async commitPullDelivery(
|
|
2640
|
+
{ did, dwnUrl, linkKey, link, isStale }: LivePullContext,
|
|
2641
|
+
cursor: ProgressToken,
|
|
2642
|
+
delivery: PullDelivery,
|
|
2643
|
+
): Promise<void> {
|
|
2644
|
+
// Guard: if the link transitioned to repairing while this handler was
|
|
2645
|
+
// in-flight, skip all state mutations — the repair process owns progression.
|
|
2646
|
+
if (!link || !delivery.runtime || link.status !== 'live' || isStale()) {
|
|
2647
|
+
return;
|
|
2648
|
+
}
|
|
2649
|
+
|
|
2650
|
+
const entry = delivery.runtime.inflight.get(delivery.ordinal);
|
|
2651
|
+
if (entry) { entry.committed = true; }
|
|
2652
|
+
|
|
2653
|
+
ReplicationLedger.setReceivedToken(link.pull, cursor);
|
|
2654
|
+
const drained = this.drainCommittedPull(linkKey);
|
|
2655
|
+
if (drained > 0) {
|
|
2656
|
+
await this.ledger.saveLink(link);
|
|
2657
|
+
this.emitPullCheckpointAdvance(link);
|
|
2658
|
+
}
|
|
2659
|
+
|
|
2660
|
+
if (delivery.runtime.inflight.size > MAX_PENDING_TOKENS) {
|
|
2661
|
+
console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
2662
|
+
await this.transitionToRepairing(linkKey, link);
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
private emitPullCheckpointAdvance(link: ReplicationLinkState): void {
|
|
2667
|
+
if (!link.pull.contiguousAppliedToken) {
|
|
2668
|
+
return;
|
|
2669
|
+
}
|
|
2670
|
+
|
|
2671
|
+
// Emit after durable save — "advanced" means persisted.
|
|
2672
|
+
this.emitEvent({
|
|
2673
|
+
type : 'checkpoint:pull-advance',
|
|
2674
|
+
tenantDid : link.tenantDid,
|
|
2675
|
+
remoteEndpoint : link.remoteEndpoint,
|
|
2676
|
+
...syncEventScope(link.scope),
|
|
2677
|
+
position : link.pull.contiguousAppliedToken.position,
|
|
2678
|
+
messageCid : link.pull.contiguousAppliedToken.messageCid,
|
|
2679
|
+
});
|
|
2680
|
+
}
|
|
2681
|
+
|
|
2682
|
+
private async handleLivePullProcessingError(
|
|
2683
|
+
{ did, dwnUrl, linkKey, link, isStale }: LivePullContext,
|
|
2684
|
+
event: MessageEvent,
|
|
2685
|
+
error: any,
|
|
2686
|
+
): Promise<void> {
|
|
2687
|
+
console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
|
|
2688
|
+
await this.recordPullProcessingFailure({ did, dwnUrl, event, error });
|
|
2689
|
+
|
|
2690
|
+
// A failed processRawMessage means local state is incomplete. Transition
|
|
2691
|
+
// to repairing immediately — do NOT advance the checkpoint past this
|
|
2692
|
+
// failure or let later ordinals commit past it. SMT reconciliation will
|
|
2693
|
+
// discover and fill the gap.
|
|
2694
|
+
if (link && !isStale()) {
|
|
2695
|
+
await this.transitionToRepairing(linkKey, link);
|
|
2696
|
+
}
|
|
2697
|
+
}
|
|
2698
|
+
|
|
2699
|
+
private async recordPullProcessingFailure({ did, dwnUrl, event, error }: {
|
|
2700
|
+
did: string;
|
|
2701
|
+
dwnUrl: string;
|
|
2702
|
+
event: MessageEvent;
|
|
2703
|
+
error: any;
|
|
2704
|
+
}): Promise<void> {
|
|
2705
|
+
try {
|
|
2706
|
+
const failedCid = await Message.getCid(event.message);
|
|
2707
|
+
void this.recordDeadLetter({
|
|
2708
|
+
messageCid : failedCid,
|
|
2709
|
+
tenantDid : did,
|
|
2710
|
+
remoteEndpoint : dwnUrl,
|
|
2711
|
+
protocol : (event.message.descriptor as Record<string, unknown>).protocol as string | undefined,
|
|
2712
|
+
category : 'pull-processing',
|
|
2713
|
+
errorDetail : error.message ?? String(error),
|
|
2714
|
+
});
|
|
2715
|
+
} catch {
|
|
2716
|
+
// Best effort — don't let dead letter recording block repair.
|
|
2717
|
+
}
|
|
2718
|
+
}
|
|
2719
|
+
|
|
2720
|
+
// ---------------------------------------------------------------------------
|
|
2721
|
+
// Live push: local EventLog subscription for immediate push
|
|
2722
|
+
// ---------------------------------------------------------------------------
|
|
2723
|
+
|
|
2724
|
+
/**
|
|
2725
|
+
* Subscribes to the local DWN's EventLog so that writes by the user are
|
|
2726
|
+
* immediately pushed to the remote DWN instead of waiting for the next poll.
|
|
2727
|
+
*/
|
|
2728
|
+
private async openLocalPushSubscription(target: LinkSyncTarget): Promise<void> {
|
|
2729
|
+
const { did, delegateDid, dwnUrl } = target;
|
|
2730
|
+
const protocol = singleProtocolForSyncScope(target.scope);
|
|
2731
|
+
|
|
2732
|
+
const filters = target.scope.kind === 'protocolSet'
|
|
2733
|
+
? target.scope.protocols.map(protocol => ({ protocol }))
|
|
2734
|
+
: [];
|
|
2735
|
+
|
|
2736
|
+
const handlerGeneration = this._engineGeneration;
|
|
2737
|
+
|
|
2738
|
+
// Capture the link for identity-based staleness detection.
|
|
2739
|
+
const capturedPushLink = this._activeLinks.get(target.linkKey);
|
|
2740
|
+
const isPushStale = (): boolean =>
|
|
2741
|
+
this._engineGeneration !== handlerGeneration ||
|
|
2742
|
+
!this._activeLinks.has(target.linkKey) ||
|
|
2743
|
+
(capturedPushLink !== undefined && this._activeLinks.get(target.linkKey) !== capturedPushLink);
|
|
2744
|
+
|
|
2745
|
+
// Subscribe to the local DWN's EventLog.
|
|
2746
|
+
const subscriptionHandler = async (subMessage: SubscriptionMessage): Promise<void> => {
|
|
2747
|
+
if (isPushStale()) {
|
|
2748
|
+
return;
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
if (subMessage.type !== 'event') {
|
|
2752
|
+
return;
|
|
2753
|
+
}
|
|
2754
|
+
|
|
2755
|
+
// Subset scope filtering: only push events that match the link scope.
|
|
2756
|
+
// Events outside the scope are not this link's responsibility.
|
|
2757
|
+
const pushLinkKey = target.linkKey;
|
|
2758
|
+
const pushLink = this._activeLinks.get(pushLinkKey);
|
|
2759
|
+
if (pushLink) {
|
|
2760
|
+
const scopeClassification = classifySyncEventScope(subMessage.event, pushLink.scope);
|
|
2761
|
+
if (scopeClassification === 'out-of-scope') {
|
|
2762
|
+
return;
|
|
2763
|
+
}
|
|
2764
|
+
if (scopeClassification === 'unknown') {
|
|
2765
|
+
this.markLinkNeedsReconcile(pushLinkKey, pushLink, 'push-scope-unclassified');
|
|
2766
|
+
return;
|
|
2767
|
+
}
|
|
2768
|
+
}
|
|
2769
|
+
|
|
2770
|
+
// Accumulate the message CID for a debounced push.
|
|
2771
|
+
const targetKey = pushLinkKey;
|
|
2772
|
+
const cid = await Message.getCid(subMessage.event.message);
|
|
2773
|
+
if (cid === undefined || isPushStale()) {
|
|
2774
|
+
return;
|
|
2775
|
+
}
|
|
2776
|
+
|
|
2777
|
+
// Echo-loop suppression: skip CIDs that were recently pulled from this
|
|
2778
|
+
// specific remote. A message pulled from Provider A is only suppressed
|
|
2779
|
+
// for push to A — it still fans out to Provider B and C.
|
|
2780
|
+
if (this.isRecentlyPulled(cid, dwnUrl)) {
|
|
2781
|
+
return;
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2784
|
+
const pushRuntime = this.getOrCreatePushRuntime(targetKey, {
|
|
2785
|
+
did,
|
|
2786
|
+
dwnUrl,
|
|
2787
|
+
delegateDid,
|
|
2788
|
+
protocol,
|
|
2789
|
+
permissionGrantIds: target.permissionGrantIds,
|
|
2790
|
+
});
|
|
2791
|
+
pushRuntime.entries.push({ cid });
|
|
2792
|
+
|
|
2793
|
+
// Immediate-first: if no push is in flight and no batch timer is
|
|
2794
|
+
// pending, push immediately. Otherwise, the pending batch timer
|
|
2795
|
+
// or the post-flush drain will pick up the new entry.
|
|
2796
|
+
if (!pushRuntime.flushing && !pushRuntime.timer) {
|
|
2797
|
+
void this.flushPendingPushesForLink(targetKey);
|
|
2798
|
+
}
|
|
2799
|
+
};
|
|
2800
|
+
|
|
2801
|
+
// Subscribe to the local DWN EventLog from "now" — opportunistic push
|
|
2802
|
+
// does not replay from a stored cursor. Any writes missed during outages
|
|
2803
|
+
// are recovered by the post-repair reconciliation path.
|
|
2804
|
+
const response = await this.agent.dwn.processRequest({
|
|
2805
|
+
author : did,
|
|
2806
|
+
target : did,
|
|
2807
|
+
messageType : DwnInterface.MessagesSubscribe,
|
|
2808
|
+
granteeDid : delegateDid,
|
|
2809
|
+
messageParams : { filters, permissionGrantIds: toMessagesPermissionGrantIds(target.permissionGrantIds) },
|
|
2810
|
+
subscriptionHandler : subscriptionHandler as any,
|
|
2811
|
+
});
|
|
2812
|
+
|
|
2813
|
+
const reply = response.reply as MessagesSubscribeReply;
|
|
2814
|
+
if (reply.status.code !== 200 || !reply.subscription) {
|
|
2815
|
+
throw new Error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
|
|
2816
|
+
}
|
|
2817
|
+
|
|
2818
|
+
const close = async (): Promise<void> => { await reply.subscription!.close(); };
|
|
2819
|
+
this._localSubscriptions.push({
|
|
2820
|
+
linkKey: target.linkKey,
|
|
2821
|
+
did,
|
|
2822
|
+
dwnUrl,
|
|
2823
|
+
delegateDid,
|
|
2824
|
+
close,
|
|
2825
|
+
});
|
|
2826
|
+
}
|
|
2827
|
+
|
|
2828
|
+
/**
|
|
2829
|
+
* Flushes accumulated push CIDs to remote DWNs.
|
|
2830
|
+
*/
|
|
2831
|
+
private async flushPendingPushes(): Promise<void> {
|
|
2832
|
+
await Promise.all([...this._pushRuntimes.keys()].map(async (linkKey) => {
|
|
2833
|
+
await this.flushPendingPushesForLink(linkKey);
|
|
2834
|
+
}));
|
|
2835
|
+
}
|
|
2836
|
+
|
|
2837
|
+
private async flushPendingPushesForLink(linkKey: string): Promise<void> {
|
|
2838
|
+
const batch = this.takePushFlushBatch(linkKey);
|
|
2839
|
+
if (!batch) { return; }
|
|
2840
|
+
|
|
2841
|
+
const { pushRuntime, pushEntries, isStale } = batch;
|
|
2842
|
+
const { did, dwnUrl, delegateDid, protocol, permissionGrantIds, retryCount } = pushRuntime;
|
|
2843
|
+
|
|
2844
|
+
try {
|
|
2845
|
+
const result = await pushMessages({
|
|
2846
|
+
did,
|
|
2847
|
+
dwnUrl,
|
|
2848
|
+
delegateDid,
|
|
2849
|
+
permissionGrantIds,
|
|
2850
|
+
messageCids : pushEntries.map((entry) => entry.cid),
|
|
2851
|
+
agent : this.agent,
|
|
2852
|
+
});
|
|
2853
|
+
|
|
2854
|
+
await this.handlePushBatchResult(linkKey, batch, result);
|
|
2855
|
+
} catch (error: any) {
|
|
2856
|
+
if (isStale()) { return; }
|
|
2857
|
+
console.error(`SyncEngineLevel: Push batch failed for ${did} -> ${dwnUrl}`, error);
|
|
2858
|
+
this.requeueOrReconcile(linkKey, {
|
|
2859
|
+
did,
|
|
2860
|
+
dwnUrl,
|
|
2861
|
+
delegateDid,
|
|
2862
|
+
protocol,
|
|
2863
|
+
permissionGrantIds,
|
|
2864
|
+
entries : pushEntries,
|
|
2865
|
+
retryCount : retryCount + 1,
|
|
2866
|
+
});
|
|
2867
|
+
} finally {
|
|
2868
|
+
this.finishPushFlush(linkKey, pushRuntime);
|
|
2869
|
+
}
|
|
2870
|
+
}
|
|
2871
|
+
|
|
2872
|
+
private takePushFlushBatch(linkKey: string): PushFlushBatch | undefined {
|
|
2873
|
+
// Guard: bail if this link was hot-removed or is no longer live. Without
|
|
2874
|
+
// this, a stale debounce timer or retry callback could send pushes after
|
|
2875
|
+
// the DID was removed or the link entered repair/terminal state.
|
|
2876
|
+
const flushLink = this._activeLinks.get(linkKey);
|
|
2877
|
+
if (flushLink?.status !== 'live') {
|
|
2878
|
+
const staleRuntime = this._pushRuntimes.get(linkKey);
|
|
2879
|
+
if (staleRuntime?.timer) {
|
|
2880
|
+
clearTimeout(staleRuntime.timer);
|
|
2881
|
+
}
|
|
2882
|
+
this._pushRuntimes.delete(linkKey);
|
|
2883
|
+
return undefined;
|
|
2884
|
+
}
|
|
2885
|
+
|
|
2886
|
+
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
2887
|
+
if (!pushRuntime) {
|
|
2888
|
+
return undefined;
|
|
2889
|
+
}
|
|
2890
|
+
|
|
2891
|
+
const { entries: pushEntries, retryCount } = pushRuntime;
|
|
2892
|
+
pushRuntime.entries = [];
|
|
2893
|
+
|
|
2894
|
+
if (pushEntries.length === 0) {
|
|
2895
|
+
if (!pushRuntime.timer && !pushRuntime.flushing && retryCount === 0) {
|
|
2896
|
+
this._pushRuntimes.delete(linkKey);
|
|
2897
|
+
}
|
|
2898
|
+
return undefined;
|
|
2899
|
+
}
|
|
2900
|
+
|
|
2901
|
+
// Capture the current active link identity so we can detect
|
|
2902
|
+
// remove+re-add during the await pushMessages() call.
|
|
2903
|
+
const isStale = (): boolean =>
|
|
2904
|
+
!this._activeLinks.has(linkKey) ||
|
|
2905
|
+
(flushLink !== undefined && this._activeLinks.get(linkKey) !== flushLink);
|
|
2906
|
+
|
|
2907
|
+
pushRuntime.flushing = true;
|
|
2908
|
+
return { pushRuntime, pushEntries, isStale };
|
|
2909
|
+
}
|
|
2910
|
+
|
|
2911
|
+
private async handlePushBatchResult(
|
|
2912
|
+
linkKey: string,
|
|
2913
|
+
batch: PushFlushBatch,
|
|
2914
|
+
result: PushResult,
|
|
2915
|
+
): Promise<void> {
|
|
2916
|
+
if (batch.isStale()) { return; }
|
|
2917
|
+
|
|
2918
|
+
this.clearSucceededPushFailures(result.succeeded, batch.pushRuntime.dwnUrl);
|
|
2919
|
+
await this.recordPermanentPushFailures(batch.pushRuntime, result.permanentlyFailed);
|
|
2920
|
+
|
|
2921
|
+
if (result.failed.length > 0) {
|
|
2922
|
+
this.requeueFailedPushes(linkKey, batch, result.failed);
|
|
2923
|
+
return;
|
|
2924
|
+
}
|
|
2925
|
+
|
|
2926
|
+
this.cleanupSuccessfulPushRuntime(linkKey, batch.pushRuntime);
|
|
2927
|
+
}
|
|
2928
|
+
|
|
2929
|
+
private clearSucceededPushFailures(cids: string[], dwnUrl: string): void {
|
|
2930
|
+
for (const cid of cids) {
|
|
2931
|
+
this.clearFailedMessage(cid, dwnUrl).catch(() => { /* teardown race */ });
|
|
2932
|
+
}
|
|
2933
|
+
}
|
|
2934
|
+
|
|
2935
|
+
private async recordPermanentPushFailures(
|
|
2936
|
+
pushRuntime: PushRuntimeState,
|
|
2937
|
+
permanentlyFailed: PushResult['permanentlyFailed'],
|
|
2938
|
+
): Promise<void> {
|
|
2939
|
+
for (const entry of permanentlyFailed) {
|
|
2940
|
+
await this.recordDeadLetter({
|
|
2941
|
+
messageCid : entry.cid,
|
|
2942
|
+
tenantDid : pushRuntime.did,
|
|
2943
|
+
remoteEndpoint : pushRuntime.dwnUrl,
|
|
2944
|
+
protocol : pushRuntime.protocol,
|
|
2945
|
+
category : 'push-permanent',
|
|
2946
|
+
errorCode : String(entry.statusCode ?? ''),
|
|
2947
|
+
errorDetail : entry.detail ?? 'permanent push failure',
|
|
2948
|
+
});
|
|
2949
|
+
}
|
|
2950
|
+
}
|
|
2951
|
+
|
|
2952
|
+
private requeueFailedPushes(linkKey: string, batch: PushFlushBatch, failedCids: string[]): void {
|
|
2953
|
+
if (batch.isStale()) { return; }
|
|
2954
|
+
|
|
2955
|
+
const { did, dwnUrl, delegateDid, protocol, permissionGrantIds, retryCount } = batch.pushRuntime;
|
|
2956
|
+
const failedSet = new Set(failedCids);
|
|
2957
|
+
const failedEntries = batch.pushEntries.filter((entry) => failedSet.has(entry.cid));
|
|
2958
|
+
this.requeueOrReconcile(linkKey, {
|
|
2959
|
+
did,
|
|
2960
|
+
dwnUrl,
|
|
2961
|
+
delegateDid,
|
|
2962
|
+
protocol,
|
|
2963
|
+
permissionGrantIds,
|
|
2964
|
+
entries : failedEntries,
|
|
2965
|
+
retryCount : retryCount + 1,
|
|
2966
|
+
});
|
|
2967
|
+
}
|
|
2968
|
+
|
|
2969
|
+
private cleanupSuccessfulPushRuntime(linkKey: string, pushRuntime: PushRuntimeState): void {
|
|
2970
|
+
// Successful push — reset retry count so subsequent unrelated batches on
|
|
2971
|
+
// this link start with a fresh budget.
|
|
2972
|
+
pushRuntime.retryCount = 0;
|
|
2973
|
+
if (!pushRuntime.timer && pushRuntime.entries.length === 0) {
|
|
2974
|
+
this._pushRuntimes.delete(linkKey);
|
|
2975
|
+
}
|
|
2976
|
+
}
|
|
2977
|
+
|
|
2978
|
+
private finishPushFlush(linkKey: string, pushRuntime: PushRuntimeState): void {
|
|
2979
|
+
pushRuntime.flushing = false;
|
|
2980
|
+
|
|
2981
|
+
// If new entries accumulated while this push was in flight, schedule a
|
|
2982
|
+
// short drain to flush them. This gives a brief batching window for burst
|
|
2983
|
+
// writes while keeping single-write latency low.
|
|
2984
|
+
const rt = this._pushRuntimes.get(linkKey);
|
|
2985
|
+
if (rt && rt.entries.length > 0 && !rt.timer) {
|
|
2986
|
+
rt.timer = setTimeout((): void => {
|
|
2987
|
+
rt.timer = undefined;
|
|
2988
|
+
void this.flushPendingPushesForLink(linkKey);
|
|
2989
|
+
}, PUSH_DEBOUNCE_MS);
|
|
2990
|
+
}
|
|
2991
|
+
}
|
|
2992
|
+
|
|
2993
|
+
/** Push retry backoff schedule: immediate, 250ms, 1s, 2s, then give up. */
|
|
2994
|
+
private static readonly PUSH_RETRY_BACKOFF_MS = [0, 250, 1000, 2000];
|
|
2995
|
+
private static readonly ROOT_CONVERGENCE_CLEARABLE_DEAD_LETTER_CATEGORIES: ReadonlySet<DeadLetterCategory> =
|
|
2996
|
+
new Set(['push-permanent', 'push-exhausted', 'pull-processing', 'pull-scope-rejected']);
|
|
2997
|
+
|
|
2998
|
+
/**
|
|
2999
|
+
* Re-queues a failed push batch for retry, or marks the link
|
|
3000
|
+
* `needsReconcile` if retries are exhausted. Bounded to prevent
|
|
3001
|
+
* infinite retry loops.
|
|
3002
|
+
*/
|
|
3003
|
+
private requeueOrReconcile(targetKey: string, pending: {
|
|
3004
|
+
did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
|
|
3005
|
+
permissionGrantIds?: NonEmptyStringArray;
|
|
3006
|
+
entries: PushRuntimeEntry[];
|
|
3007
|
+
retryCount: number;
|
|
3008
|
+
}): void {
|
|
3009
|
+
const maxRetries = SyncEngineLevel.PUSH_RETRY_BACKOFF_MS.length;
|
|
3010
|
+
const pushRuntime = this.getOrCreatePushRuntime(targetKey, pending);
|
|
3011
|
+
|
|
3012
|
+
if (pending.retryCount >= maxRetries) {
|
|
3013
|
+
// Retry budget exhausted — record each CID as a dead letter and mark
|
|
3014
|
+
// the link dirty for reconciliation.
|
|
3015
|
+
for (const entry of pending.entries) {
|
|
3016
|
+
void this.recordDeadLetter({
|
|
3017
|
+
messageCid : entry.cid,
|
|
3018
|
+
tenantDid : pending.did,
|
|
3019
|
+
remoteEndpoint : pending.dwnUrl,
|
|
3020
|
+
protocol : pending.protocol,
|
|
3021
|
+
category : 'push-exhausted',
|
|
3022
|
+
errorDetail : `push retries exhausted after ${maxRetries} attempts`,
|
|
3023
|
+
});
|
|
3024
|
+
}
|
|
3025
|
+
if (pushRuntime.timer) {
|
|
3026
|
+
clearTimeout(pushRuntime.timer);
|
|
3027
|
+
}
|
|
3028
|
+
this._pushRuntimes.delete(targetKey);
|
|
3029
|
+
const link = this._activeLinks.get(targetKey);
|
|
3030
|
+
if (link) {
|
|
3031
|
+
this.markLinkNeedsReconcile(targetKey, link, 'push-retry-exhausted');
|
|
3032
|
+
}
|
|
3033
|
+
return;
|
|
3034
|
+
}
|
|
3035
|
+
|
|
3036
|
+
pushRuntime.entries.push(...pending.entries);
|
|
3037
|
+
pushRuntime.retryCount = pending.retryCount;
|
|
3038
|
+
const delayMs = SyncEngineLevel.PUSH_RETRY_BACKOFF_MS[pending.retryCount] ?? 2000;
|
|
3039
|
+
if (pushRuntime.timer) {
|
|
3040
|
+
clearTimeout(pushRuntime.timer);
|
|
3041
|
+
}
|
|
3042
|
+
pushRuntime.timer = setTimeout((): void => {
|
|
3043
|
+
pushRuntime.timer = undefined;
|
|
3044
|
+
void this.flushPendingPushesForLink(targetKey);
|
|
3045
|
+
}, delayMs);
|
|
3046
|
+
}
|
|
3047
|
+
|
|
3048
|
+
private markLinkNeedsReconcile(linkKey: string, link: ReplicationLinkState, reason: string): void {
|
|
3049
|
+
if (link.needsReconcile) {
|
|
3050
|
+
this.scheduleReconcile(linkKey);
|
|
3051
|
+
return;
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
link.needsReconcile = true;
|
|
3055
|
+
void this.ledger.saveLink(link).then(() => {
|
|
3056
|
+
this.emitEvent({
|
|
3057
|
+
type : 'reconcile:needed',
|
|
3058
|
+
tenantDid : link.tenantDid,
|
|
3059
|
+
remoteEndpoint : link.remoteEndpoint,
|
|
3060
|
+
...syncEventScope(link.scope),
|
|
3061
|
+
reason,
|
|
3062
|
+
});
|
|
3063
|
+
this.scheduleReconcile(linkKey);
|
|
3064
|
+
}).catch((error: unknown) => {
|
|
3065
|
+
console.error(`SyncEngineLevel: Failed to mark link for reconciliation ${link.tenantDid} -> ${link.remoteEndpoint}`, error);
|
|
3066
|
+
});
|
|
3067
|
+
}
|
|
3068
|
+
|
|
3069
|
+
private createLinkReconciler(shouldContinue?: () => boolean): SyncLinkReconciler {
|
|
3070
|
+
return new SyncLinkReconciler({
|
|
3071
|
+
getLocalRoot : async (did, delegateDid, protocol, permissionGrantIds) => this.getLocalRoot(did, delegateDid, protocol, permissionGrantIds),
|
|
3072
|
+
getRemoteRoot : async (did, dwnUrl, delegateDid, protocol, permissionGrantIds) =>
|
|
3073
|
+
this.getRemoteRoot(did, dwnUrl, delegateDid, protocol, permissionGrantIds),
|
|
3074
|
+
diffWithRemote : async (target) => this.diffWithRemote(target),
|
|
3075
|
+
pullMessages : async (params) => this.pullMessages(params),
|
|
3076
|
+
pushMessages : async (params) => this.pushMessages(params),
|
|
3077
|
+
shouldContinue,
|
|
3078
|
+
});
|
|
3079
|
+
}
|
|
3080
|
+
|
|
3081
|
+
private getReconcileProtocols(scope: SyncScope): (string | undefined)[] {
|
|
3082
|
+
return protocolsForSyncScope(scope) ?? [undefined];
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
private getAuthorizationGrantIds(authorization: SyncAuthorization): NonEmptyStringArray | undefined {
|
|
3086
|
+
return authorization.kind === 'delegate' ? authorization.permissionGrantIds : undefined;
|
|
3087
|
+
}
|
|
3088
|
+
|
|
3089
|
+
private async reconcileProjectionTarget(
|
|
3090
|
+
target: ProjectionReconcileTarget,
|
|
3091
|
+
options?: ProjectionReconcileOptions,
|
|
3092
|
+
shouldContinue?: () => boolean,
|
|
3093
|
+
): Promise<ProjectionReconcileResult> {
|
|
3094
|
+
if (target.scope.kind === 'recordsProjection') {
|
|
3095
|
+
return this.reconcileRecordsProjectionTarget(target, target.scope, options, shouldContinue);
|
|
3096
|
+
}
|
|
3097
|
+
|
|
3098
|
+
if (target.scope.kind === 'protocolSet' && target.scope.protocols.length > 1) {
|
|
3099
|
+
return this.reconcileProtocolSetProjectionTarget(target, options, shouldContinue);
|
|
3100
|
+
}
|
|
3101
|
+
|
|
3102
|
+
let converged = true;
|
|
3103
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
3104
|
+
const reconciler = this.createLinkReconciler(shouldContinue);
|
|
3105
|
+
|
|
3106
|
+
for (const protocol of this.getReconcileProtocols(target.scope)) {
|
|
3107
|
+
const outcome = await reconciler.reconcile({
|
|
3108
|
+
did : target.did,
|
|
3109
|
+
dwnUrl : target.dwnUrl,
|
|
3110
|
+
delegateDid : target.delegateDid,
|
|
3111
|
+
protocol,
|
|
3112
|
+
permissionGrantIds,
|
|
3113
|
+
}, options);
|
|
3114
|
+
if (outcome.aborted) {
|
|
3115
|
+
return { aborted: true };
|
|
3116
|
+
}
|
|
3117
|
+
if (options?.verifyConvergence === true && outcome.converged !== true) {
|
|
3118
|
+
converged = false;
|
|
3119
|
+
}
|
|
3120
|
+
}
|
|
3121
|
+
|
|
3122
|
+
return options?.verifyConvergence === true ? { converged } : {};
|
|
3123
|
+
}
|
|
3124
|
+
|
|
3125
|
+
private async reconcileRecordsProjectionTarget(
|
|
3126
|
+
target: ProjectionReconcileTarget,
|
|
3127
|
+
scope: RecordsProjectionSyncScope,
|
|
3128
|
+
options?: ProjectionReconcileOptions,
|
|
3129
|
+
shouldContinue?: () => boolean,
|
|
3130
|
+
): Promise<ProjectionReconcileResult> {
|
|
3131
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
3132
|
+
const localRoot = await this.getLocalProjectedRoot(target.did, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
3133
|
+
if (shouldContinue?.() === false) { return { aborted: true }; }
|
|
3134
|
+
|
|
3135
|
+
const remoteRoot = await this.getRemoteProjectedRoot(target.did, target.dwnUrl, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
3136
|
+
if (shouldContinue?.() === false) { return { aborted: true }; }
|
|
3137
|
+
|
|
3138
|
+
if (localRoot !== remoteRoot) {
|
|
3139
|
+
const diff = await this.diffProjectedWithRemote({
|
|
3140
|
+
did : target.did,
|
|
3141
|
+
dwnUrl : target.dwnUrl,
|
|
3142
|
+
delegateDid : target.delegateDid,
|
|
3143
|
+
scopes : scope.scopes,
|
|
3144
|
+
permissionGrantIds,
|
|
3145
|
+
});
|
|
3146
|
+
if (shouldContinue?.() === false) { return { aborted: true }; }
|
|
3147
|
+
|
|
3148
|
+
const aborted = await this.applyProjectedDiff(target, scope, diff, permissionGrantIds, options, shouldContinue);
|
|
3149
|
+
if (aborted) { return { aborted: true }; }
|
|
3150
|
+
}
|
|
3151
|
+
|
|
3152
|
+
if (options?.verifyConvergence !== true) {
|
|
3153
|
+
return {};
|
|
3154
|
+
}
|
|
3155
|
+
|
|
3156
|
+
const postLocalRoot = await this.getLocalProjectedRoot(target.did, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
3157
|
+
if (shouldContinue?.() === false) { return { aborted: true }; }
|
|
3158
|
+
|
|
3159
|
+
const postRemoteRoot = await this.getRemoteProjectedRoot(target.did, target.dwnUrl, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
3160
|
+
if (shouldContinue?.() === false) { return { aborted: true }; }
|
|
3161
|
+
|
|
3162
|
+
return { converged: postLocalRoot === postRemoteRoot };
|
|
3163
|
+
}
|
|
3164
|
+
|
|
3165
|
+
private async reconcileProtocolSetProjectionTarget(
|
|
3166
|
+
target: ProjectionReconcileTarget,
|
|
3167
|
+
options?: ProjectionReconcileOptions,
|
|
3168
|
+
shouldContinue?: () => boolean,
|
|
3169
|
+
): Promise<ProjectionReconcileResult> {
|
|
3170
|
+
if (target.scope.kind !== 'protocolSet') {
|
|
3171
|
+
return {};
|
|
3172
|
+
}
|
|
3173
|
+
|
|
3174
|
+
const scope = target.scope;
|
|
3175
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
3176
|
+
const diffPlan = await this.collectProtocolSetDiffPlan(target, scope, permissionGrantIds, shouldContinue);
|
|
3177
|
+
if (!diffPlan) {
|
|
3178
|
+
return { aborted: true };
|
|
3179
|
+
}
|
|
3180
|
+
|
|
3181
|
+
if (diffPlan.changedProtocols.length === 0) {
|
|
3182
|
+
return options?.verifyConvergence === true ? { converged: true } : {};
|
|
3183
|
+
}
|
|
3184
|
+
|
|
3185
|
+
const aborted = await this.applyProtocolSetDiffPlan(target, scope, diffPlan, permissionGrantIds, options, shouldContinue);
|
|
3186
|
+
if (aborted) {
|
|
3187
|
+
return { aborted: true };
|
|
3188
|
+
}
|
|
3189
|
+
|
|
3190
|
+
if (options?.verifyConvergence !== true) {
|
|
3191
|
+
return {};
|
|
3192
|
+
}
|
|
3193
|
+
|
|
3194
|
+
return this.verifyProtocolSetConvergence(target, diffPlan.changedProtocols, permissionGrantIds, shouldContinue);
|
|
3195
|
+
}
|
|
3196
|
+
|
|
3197
|
+
private async collectProtocolSetDiffPlan(
|
|
3198
|
+
target: ProjectionReconcileTarget,
|
|
3199
|
+
scope: ProtocolSetScope,
|
|
3200
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3201
|
+
shouldContinue?: () => boolean,
|
|
3202
|
+
): Promise<ProtocolSetDiffPlan | undefined> {
|
|
3203
|
+
const plan: ProtocolSetDiffPlan = { changedProtocols: [], onlyRemote: [], onlyLocal: [] };
|
|
3204
|
+
|
|
3205
|
+
for (const protocol of scope.protocols) {
|
|
3206
|
+
const roots = await this.getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue);
|
|
3207
|
+
if (!roots) { return undefined; }
|
|
3208
|
+
|
|
3209
|
+
if (roots.localRoot === roots.remoteRoot) {
|
|
3210
|
+
continue;
|
|
3211
|
+
}
|
|
3212
|
+
|
|
3213
|
+
plan.changedProtocols.push(protocol);
|
|
3214
|
+
const diff = await this.diffWithRemote({
|
|
3215
|
+
did : target.did,
|
|
3216
|
+
dwnUrl : target.dwnUrl,
|
|
3217
|
+
delegateDid : target.delegateDid,
|
|
3218
|
+
protocol,
|
|
3219
|
+
permissionGrantIds,
|
|
3220
|
+
});
|
|
3221
|
+
if (shouldContinue?.() === false) { return undefined; }
|
|
3222
|
+
|
|
3223
|
+
plan.onlyRemote.push(...diff.onlyRemote);
|
|
3224
|
+
plan.onlyLocal.push(...diff.onlyLocal);
|
|
3225
|
+
}
|
|
3226
|
+
|
|
3227
|
+
return plan;
|
|
3228
|
+
}
|
|
3229
|
+
|
|
3230
|
+
private async getProtocolRoots(
|
|
3231
|
+
target: ProjectionReconcileTarget,
|
|
3232
|
+
protocol: string,
|
|
3233
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3234
|
+
shouldContinue?: () => boolean,
|
|
3235
|
+
): Promise<{ localRoot: string; remoteRoot: string } | undefined> {
|
|
3236
|
+
const localRoot = await this.getLocalRoot(target.did, target.delegateDid, protocol, permissionGrantIds);
|
|
3237
|
+
if (shouldContinue?.() === false) { return undefined; }
|
|
3238
|
+
|
|
3239
|
+
const remoteRoot = await this.getRemoteRoot(target.did, target.dwnUrl, target.delegateDid, protocol, permissionGrantIds);
|
|
3240
|
+
if (shouldContinue?.() === false) { return undefined; }
|
|
3241
|
+
|
|
3242
|
+
return { localRoot, remoteRoot };
|
|
3243
|
+
}
|
|
3244
|
+
|
|
3245
|
+
private async applyProtocolSetDiffPlan(
|
|
3246
|
+
target: ProjectionReconcileTarget,
|
|
3247
|
+
scope: ProtocolSetScope,
|
|
3248
|
+
diffPlan: ProtocolSetDiffPlan,
|
|
3249
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3250
|
+
options?: ProjectionReconcileOptions,
|
|
3251
|
+
shouldContinue?: () => boolean,
|
|
3252
|
+
): Promise<boolean> {
|
|
3253
|
+
// Keep the remote diff combined across protocols so topologicalSort can
|
|
3254
|
+
// order composed protocol configs before records that use them. Any future
|
|
3255
|
+
// chunking for large protocol sets must preserve this global dependency
|
|
3256
|
+
// order instead of reverting to independent per-protocol chunks.
|
|
3257
|
+
if (
|
|
3258
|
+
options?.direction !== 'push' &&
|
|
3259
|
+
diffPlan.onlyRemote.length > 0 &&
|
|
3260
|
+
await this.pullRemoteDiffEntries(target, scope, diffPlan.onlyRemote, permissionGrantIds, shouldContinue)
|
|
3261
|
+
) {
|
|
3262
|
+
return true;
|
|
3263
|
+
}
|
|
3264
|
+
|
|
3265
|
+
if (options?.direction === 'pull' || diffPlan.onlyLocal.length === 0) {
|
|
3266
|
+
return false;
|
|
3267
|
+
}
|
|
3268
|
+
|
|
3269
|
+
return this.pushLocalDiffEntries(target, diffPlan.onlyLocal, permissionGrantIds, shouldContinue);
|
|
3270
|
+
}
|
|
3271
|
+
|
|
3272
|
+
private async applyProjectedDiff(
|
|
3273
|
+
target: ProjectionReconcileTarget,
|
|
3274
|
+
scope: RecordsProjectionSyncScope,
|
|
3275
|
+
diff: ProjectionDiffResult,
|
|
3276
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3277
|
+
options?: ProjectionReconcileOptions,
|
|
3278
|
+
shouldContinue?: () => boolean,
|
|
3279
|
+
): Promise<boolean> {
|
|
3280
|
+
if (await this.pullProjectedRemoteDiff(target, scope, diff, permissionGrantIds, options, shouldContinue)) {
|
|
3281
|
+
return true;
|
|
3282
|
+
}
|
|
3283
|
+
|
|
3284
|
+
return this.pushProjectedLocalDiff(target, diff.onlyLocal, permissionGrantIds, options, shouldContinue);
|
|
3285
|
+
}
|
|
3286
|
+
|
|
3287
|
+
private async pullProjectedRemoteDiff(
|
|
3288
|
+
target: ProjectionReconcileTarget,
|
|
3289
|
+
scope: RecordsProjectionSyncScope,
|
|
3290
|
+
diff: ProjectionDiffResult,
|
|
3291
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3292
|
+
options?: ProjectionReconcileOptions,
|
|
3293
|
+
shouldContinue?: () => boolean,
|
|
3294
|
+
): Promise<boolean> {
|
|
3295
|
+
if (options?.direction === 'push' || diff.onlyRemote.length === 0) {
|
|
3296
|
+
return false;
|
|
3297
|
+
}
|
|
3298
|
+
|
|
3299
|
+
return this.pullRemoteDiffEntries(target, scope, diff.onlyRemote, permissionGrantIds, shouldContinue, diff.dependencies ?? []);
|
|
3300
|
+
}
|
|
3301
|
+
|
|
3302
|
+
private async pushProjectedLocalDiff(
|
|
3303
|
+
target: ProjectionReconcileTarget,
|
|
3304
|
+
onlyLocal: string[],
|
|
3305
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3306
|
+
options?: ProjectionReconcileOptions,
|
|
3307
|
+
shouldContinue?: () => boolean,
|
|
3308
|
+
): Promise<boolean> {
|
|
3309
|
+
if (options?.direction === 'pull' || onlyLocal.length === 0) {
|
|
3310
|
+
return false;
|
|
3311
|
+
}
|
|
3312
|
+
|
|
3313
|
+
return this.pushLocalDiffEntries(target, onlyLocal, permissionGrantIds, shouldContinue);
|
|
3314
|
+
}
|
|
3315
|
+
|
|
3316
|
+
private async pullRemoteDiffEntries(
|
|
3317
|
+
target: ProjectionReconcileTarget,
|
|
3318
|
+
scope: SyncScope,
|
|
3319
|
+
onlyRemote: MessagesSyncDiffEntry[],
|
|
3320
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3321
|
+
shouldContinue?: () => boolean,
|
|
3322
|
+
dependencies: MessagesSyncDependencyEntry[] = [],
|
|
3323
|
+
): Promise<boolean> {
|
|
3324
|
+
const primaryEntries = SyncEngineLevel.dedupeRemoteEntries(onlyRemote);
|
|
3325
|
+
try {
|
|
3326
|
+
let verifiedInitialWrites: RecordsWriteMessage[] = [];
|
|
3327
|
+
if (scope.kind === 'recordsProjection') {
|
|
3328
|
+
verifiedInitialWrites = await this.pullProjectedDependencyHints(
|
|
3329
|
+
target,
|
|
3330
|
+
scope,
|
|
3331
|
+
primaryEntries,
|
|
3332
|
+
dependencies,
|
|
3333
|
+
permissionGrantIds,
|
|
3334
|
+
shouldContinue,
|
|
3335
|
+
);
|
|
3336
|
+
}
|
|
3337
|
+
|
|
3338
|
+
const { prefetched, needsFetchCids } = partitionRemoteEntries(primaryEntries);
|
|
3339
|
+
await this.pullMessages({
|
|
3340
|
+
did : target.did,
|
|
3341
|
+
dwnUrl : target.dwnUrl,
|
|
3342
|
+
delegateDid : target.delegateDid,
|
|
3343
|
+
scope,
|
|
3344
|
+
permissionGrantIds,
|
|
3345
|
+
messageCids : needsFetchCids,
|
|
3346
|
+
prefetched,
|
|
3347
|
+
verifiedInitialWrites,
|
|
3348
|
+
shouldContinue,
|
|
3349
|
+
});
|
|
3350
|
+
} catch (error) {
|
|
3351
|
+
if (error instanceof SyncPullAbortedError) {
|
|
3352
|
+
return true;
|
|
3353
|
+
}
|
|
3354
|
+
throw error;
|
|
3355
|
+
}
|
|
3356
|
+
return shouldContinue?.() === false;
|
|
3357
|
+
}
|
|
3358
|
+
|
|
3359
|
+
private async pullProjectedDependencyHints(
|
|
3360
|
+
target: ProjectionReconcileTarget,
|
|
3361
|
+
scope: RecordsProjectionSyncScope,
|
|
3362
|
+
primaryEntries: MessagesSyncDiffEntry[],
|
|
3363
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3364
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3365
|
+
shouldContinue?: () => boolean,
|
|
3366
|
+
): Promise<RecordsWriteMessage[]> {
|
|
3367
|
+
const verified = await this.verifyProjectedDependencies(target.did, scope, primaryEntries, dependencies);
|
|
3368
|
+
if (verified.length === 0) {
|
|
3369
|
+
return [];
|
|
3370
|
+
}
|
|
3371
|
+
|
|
3372
|
+
await this.pullMessages({
|
|
3373
|
+
did : target.did,
|
|
3374
|
+
dwnUrl : target.dwnUrl,
|
|
3375
|
+
delegateDid : target.delegateDid,
|
|
3376
|
+
scope : SyncEngineLevel.protocolSetScopeForProjectedDependencies(verified),
|
|
3377
|
+
permissionGrantIds,
|
|
3378
|
+
messageCids : [],
|
|
3379
|
+
prefetched : verified,
|
|
3380
|
+
shouldContinue,
|
|
3381
|
+
});
|
|
3382
|
+
|
|
3383
|
+
return SyncEngineLevel.recordsInitialWritesFromVerifiedDependencies(verified);
|
|
3384
|
+
}
|
|
3385
|
+
|
|
3386
|
+
private async verifyProjectedDependencies(
|
|
3387
|
+
tenantDid: string,
|
|
3388
|
+
scope: RecordsProjectionSyncScope,
|
|
3389
|
+
primaryEntries: MessagesSyncDiffEntry[],
|
|
3390
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3391
|
+
): Promise<MessagesSyncDependencyEntry[]> {
|
|
3392
|
+
const primaryByCid = SyncEngineLevel.indexEntriesWithMessage(primaryEntries);
|
|
3393
|
+
const initialWritesByRoot = await this.collectProjectedRecordsInitialWriteDependencies(
|
|
3394
|
+
scope,
|
|
3395
|
+
primaryByCid,
|
|
3396
|
+
dependencies,
|
|
3397
|
+
);
|
|
3398
|
+
const protocolConfigs = await this.verifyProjectedProtocolConfigDependencies(
|
|
3399
|
+
tenantDid,
|
|
3400
|
+
scope,
|
|
3401
|
+
primaryEntries,
|
|
3402
|
+
dependencies,
|
|
3403
|
+
initialWritesByRoot,
|
|
3404
|
+
);
|
|
3405
|
+
|
|
3406
|
+
return SyncEngineLevel.dedupeDependencyEntries([
|
|
3407
|
+
...protocolConfigs,
|
|
3408
|
+
...initialWritesByRoot.values(),
|
|
3409
|
+
]);
|
|
3410
|
+
}
|
|
3411
|
+
|
|
3412
|
+
private async verifyProjectedProtocolConfigDependencies(
|
|
3413
|
+
tenantDid: string,
|
|
3414
|
+
scope: RecordsProjectionSyncScope,
|
|
3415
|
+
primaryEntries: MessagesSyncDiffEntry[],
|
|
3416
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3417
|
+
initialWritesByRoot: Map<string, AuthenticatedRecordsInitialWriteDependency> = new Map(),
|
|
3418
|
+
): Promise<MessagesSyncDependencyEntry[]> {
|
|
3419
|
+
// Projected sync dependency entries are untrusted server hints. Before any
|
|
3420
|
+
// config is applied, bind it to an accepted primary record by CID,
|
|
3421
|
+
// tenant authorship, signature, timestamp, scope, and protocol closure;
|
|
3422
|
+
// malformed or unrelated hints are ignored.
|
|
3423
|
+
const primaryByCid = SyncEngineLevel.indexEntriesWithMessage(primaryEntries);
|
|
3424
|
+
const candidatesByRoot = await this.collectProjectedProtocolConfigCandidates(
|
|
3425
|
+
tenantDid,
|
|
3426
|
+
scope,
|
|
3427
|
+
primaryByCid,
|
|
3428
|
+
dependencies,
|
|
3429
|
+
initialWritesByRoot,
|
|
3430
|
+
);
|
|
3431
|
+
const verified = new Map<string, MessagesSyncDependencyEntry>();
|
|
3432
|
+
|
|
3433
|
+
for (const [rootMessageCid, rootCandidates] of candidatesByRoot) {
|
|
3434
|
+
const primary = primaryByCid.get(rootMessageCid);
|
|
3435
|
+
const rootRecordsWrite = primary === undefined
|
|
3436
|
+
? undefined
|
|
3437
|
+
: SyncEngineLevel.protocolConfigRootRecordsWrite(primary.message, initialWritesByRoot.get(rootMessageCid)?.message);
|
|
3438
|
+
const rootProtocol = SyncEngineLevel.recordsWriteProtocol(rootRecordsWrite);
|
|
3439
|
+
if (rootProtocol === undefined) {
|
|
3440
|
+
continue;
|
|
3441
|
+
}
|
|
3442
|
+
|
|
3443
|
+
for (const dependency of SyncEngineLevel.filterProtocolConfigClosure(rootProtocol, rootCandidates)) {
|
|
3444
|
+
verified.set(dependency.messageCid, dependency);
|
|
3445
|
+
}
|
|
3446
|
+
}
|
|
3447
|
+
|
|
3448
|
+
return [...verified.values()];
|
|
3449
|
+
}
|
|
3450
|
+
|
|
3451
|
+
private static indexEntriesWithMessage(
|
|
3452
|
+
entries: MessagesSyncDiffEntry[],
|
|
3453
|
+
): Map<string, SyncDiffEntryWithMessage> {
|
|
3454
|
+
const entriesByCid = new Map<string, SyncDiffEntryWithMessage>();
|
|
3455
|
+
for (const entry of entries) {
|
|
3456
|
+
if (SyncEngineLevel.hasMessage(entry)) {
|
|
3457
|
+
entriesByCid.set(entry.messageCid, entry);
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
3460
|
+
return entriesByCid;
|
|
3461
|
+
}
|
|
3462
|
+
|
|
3463
|
+
private static recordsInitialWritesFromVerifiedDependencies(
|
|
3464
|
+
entries: MessagesSyncDependencyEntry[],
|
|
3465
|
+
): RecordsWriteMessage[] {
|
|
3466
|
+
const initialWrites: RecordsWriteMessage[] = [];
|
|
3467
|
+
for (const entry of entries) {
|
|
3468
|
+
if (SyncEngineLevel.hasMessage(entry) && SyncEngineLevel.isRecordsWriteMessage(entry.message)) {
|
|
3469
|
+
initialWrites.push(entry.message);
|
|
3470
|
+
}
|
|
3471
|
+
}
|
|
3472
|
+
return initialWrites;
|
|
3473
|
+
}
|
|
3474
|
+
|
|
3475
|
+
private async collectProjectedRecordsInitialWriteDependencies(
|
|
3476
|
+
scope: RecordsProjectionSyncScope,
|
|
3477
|
+
primaryByCid: Map<string, SyncDiffEntryWithMessage>,
|
|
3478
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3479
|
+
): Promise<Map<string, AuthenticatedRecordsInitialWriteDependency>> {
|
|
3480
|
+
const dependenciesByRoot = new Map<string, AuthenticatedRecordsInitialWriteDependency>();
|
|
3481
|
+
for (const dependency of dependencies) {
|
|
3482
|
+
const verified = await this.verifyRecordsInitialWriteCandidate(scope, primaryByCid, dependency);
|
|
3483
|
+
if (verified === undefined) {
|
|
3484
|
+
continue;
|
|
3485
|
+
}
|
|
3486
|
+
|
|
3487
|
+
dependenciesByRoot.set(verified.rootMessageCid, verified.dependency);
|
|
3488
|
+
}
|
|
3489
|
+
return dependenciesByRoot;
|
|
3490
|
+
}
|
|
3491
|
+
|
|
3492
|
+
private async verifyRecordsInitialWriteCandidate(
|
|
3493
|
+
scope: RecordsProjectionSyncScope,
|
|
3494
|
+
primaryByCid: Map<string, SyncDiffEntryWithMessage>,
|
|
3495
|
+
dependency: MessagesSyncDependencyEntry,
|
|
3496
|
+
): Promise<VerifiedRecordsInitialWriteCandidate | undefined> {
|
|
3497
|
+
if (dependency.dependencyClass !== 'recordsInitialWrite' ||
|
|
3498
|
+
!SyncEngineLevel.hasMessage(dependency) ||
|
|
3499
|
+
SyncEngineLevel.hasDependencyPayloadBytes(dependency)) {
|
|
3500
|
+
return undefined;
|
|
3501
|
+
}
|
|
3502
|
+
|
|
3503
|
+
const primary = primaryByCid.get(dependency.rootMessageCid);
|
|
3504
|
+
if (primary === undefined ||
|
|
3505
|
+
!SyncEngineLevel.isRecordsDeleteMessage(primary.message) ||
|
|
3506
|
+
!await SyncEngineLevel.projectedDependencyCidsMatch({
|
|
3507
|
+
dependencyCid : dependency.messageCid,
|
|
3508
|
+
dependencyMessage : dependency.message,
|
|
3509
|
+
primaryCid : primary.messageCid,
|
|
3510
|
+
primaryMessage : primary.message,
|
|
3511
|
+
})) {
|
|
3512
|
+
return undefined;
|
|
3513
|
+
}
|
|
3514
|
+
|
|
3515
|
+
const initialWrite = await this.toAuthenticatedRecordsInitialWriteDependency(dependency);
|
|
3516
|
+
if (initialWrite === undefined ||
|
|
3517
|
+
initialWrite.message.recordId !== SyncEngineLevel.recordsDeleteRecordId(primary.message) ||
|
|
3518
|
+
classifySyncMessageScope({ message: primary.message, initialWrite: initialWrite.message, scope }) !== 'in-scope') {
|
|
3519
|
+
return undefined;
|
|
3520
|
+
}
|
|
3521
|
+
|
|
3522
|
+
return { dependency: initialWrite, rootMessageCid: dependency.rootMessageCid };
|
|
3523
|
+
}
|
|
3524
|
+
|
|
3525
|
+
private async toAuthenticatedRecordsInitialWriteDependency(
|
|
3526
|
+
dependency: SyncDependencyEntryWithMessage,
|
|
3527
|
+
): Promise<AuthenticatedRecordsInitialWriteDependency | undefined> {
|
|
3528
|
+
if (!SyncEngineLevel.isRecordsWriteMessage(dependency.message)) {
|
|
3529
|
+
return undefined;
|
|
3530
|
+
}
|
|
3531
|
+
|
|
3532
|
+
try {
|
|
3533
|
+
const recordsWrite = await RecordsWrite.parse(dependency.message);
|
|
3534
|
+
await authenticate(recordsWrite.message.authorization, this.agent.did, recordsWrite.message.attestation);
|
|
3535
|
+
return await recordsWrite.isInitialWrite()
|
|
3536
|
+
? { ...dependency, message: recordsWrite.message }
|
|
3537
|
+
: undefined;
|
|
3538
|
+
} catch {
|
|
3539
|
+
return undefined;
|
|
3540
|
+
}
|
|
3541
|
+
}
|
|
3542
|
+
|
|
3543
|
+
private async collectProjectedProtocolConfigCandidates(
|
|
3544
|
+
tenantDid: string,
|
|
3545
|
+
scope: RecordsProjectionSyncScope,
|
|
3546
|
+
primaryByCid: Map<string, SyncDiffEntryWithMessage>,
|
|
3547
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3548
|
+
initialWritesByRoot: Map<string, AuthenticatedRecordsInitialWriteDependency>,
|
|
3549
|
+
): Promise<Map<string, AuthenticatedProtocolConfigDependency[]>> {
|
|
3550
|
+
const candidatesByRoot = new Map<string, AuthenticatedProtocolConfigDependency[]>();
|
|
3551
|
+
for (const dependency of dependencies) {
|
|
3552
|
+
const verified = await this.verifyProtocolConfigCandidate(tenantDid, scope, primaryByCid, dependency, initialWritesByRoot);
|
|
3553
|
+
if (verified === undefined) {
|
|
3554
|
+
continue;
|
|
3555
|
+
}
|
|
3556
|
+
|
|
3557
|
+
const rootCandidates = candidatesByRoot.get(verified.rootMessageCid) ?? [];
|
|
3558
|
+
rootCandidates.push(verified.dependency);
|
|
3559
|
+
candidatesByRoot.set(verified.rootMessageCid, rootCandidates);
|
|
3560
|
+
}
|
|
3561
|
+
return candidatesByRoot;
|
|
3562
|
+
}
|
|
3563
|
+
|
|
3564
|
+
private async verifyProtocolConfigCandidate(
|
|
3565
|
+
tenantDid: string,
|
|
3566
|
+
scope: RecordsProjectionSyncScope,
|
|
3567
|
+
primaryByCid: Map<string, SyncDiffEntryWithMessage>,
|
|
3568
|
+
dependency: MessagesSyncDependencyEntry,
|
|
3569
|
+
initialWritesByRoot: Map<string, AuthenticatedRecordsInitialWriteDependency>,
|
|
3570
|
+
): Promise<VerifiedProtocolConfigCandidate | undefined> {
|
|
3571
|
+
if (dependency.dependencyClass !== 'protocolsConfigure' || !SyncEngineLevel.hasMessage(dependency)) {
|
|
3572
|
+
return undefined;
|
|
3573
|
+
}
|
|
3574
|
+
|
|
3575
|
+
const primary = primaryByCid.get(dependency.rootMessageCid);
|
|
3576
|
+
if (primary === undefined) {
|
|
3577
|
+
return undefined;
|
|
3578
|
+
}
|
|
3579
|
+
|
|
3580
|
+
const verifiedDependency = await this.verifyProtocolConfigCandidateMessage(
|
|
3581
|
+
tenantDid,
|
|
3582
|
+
scope,
|
|
3583
|
+
primary,
|
|
3584
|
+
dependency,
|
|
3585
|
+
initialWritesByRoot.get(dependency.rootMessageCid)?.message,
|
|
3586
|
+
);
|
|
3587
|
+
return verifiedDependency === undefined
|
|
3588
|
+
? undefined
|
|
3589
|
+
: { dependency: verifiedDependency, rootMessageCid: dependency.rootMessageCid };
|
|
3590
|
+
}
|
|
3591
|
+
|
|
3592
|
+
private async verifyProtocolConfigCandidateMessage(
|
|
3593
|
+
tenantDid: string,
|
|
3594
|
+
scope: RecordsProjectionSyncScope,
|
|
3595
|
+
primary: SyncDiffEntryWithMessage,
|
|
3596
|
+
dependency: SyncDependencyEntryWithMessage,
|
|
3597
|
+
initialWrite: RecordsWriteMessage | undefined,
|
|
3598
|
+
): Promise<AuthenticatedProtocolConfigDependency | undefined> {
|
|
3599
|
+
// Protocol authorization is temporal: a record is governed by the protocol
|
|
3600
|
+
// definition active at its creation timestamp. Future configs may add
|
|
3601
|
+
// unrelated `uses` dependencies, so they must not widen this primary's
|
|
3602
|
+
// dependency closure.
|
|
3603
|
+
if (!await SyncEngineLevel.projectedDependencyCidsMatch({
|
|
3604
|
+
dependencyCid : dependency.messageCid,
|
|
3605
|
+
dependencyMessage : dependency.message,
|
|
3606
|
+
primaryCid : primary.messageCid,
|
|
3607
|
+
primaryMessage : primary.message,
|
|
3608
|
+
})) {
|
|
3609
|
+
return undefined;
|
|
3610
|
+
}
|
|
3611
|
+
|
|
3612
|
+
const authenticatedDependency = await this.toAuthenticatedProtocolConfigDependency(tenantDid, dependency);
|
|
3613
|
+
if (authenticatedDependency === undefined) {
|
|
3614
|
+
return undefined;
|
|
3615
|
+
}
|
|
3616
|
+
|
|
3617
|
+
const rootRecordsWrite = SyncEngineLevel.protocolConfigRootRecordsWrite(primary.message, initialWrite);
|
|
3618
|
+
const primaryIsInScope = rootRecordsWrite !== undefined &&
|
|
3619
|
+
classifySyncMessageScope({ message: primary.message, initialWrite, scope }) === 'in-scope';
|
|
3620
|
+
if (!primaryIsInScope ||
|
|
3621
|
+
!SyncEngineLevel.protocolsConfigureIsNotNewerThanRecordsWrite(authenticatedDependency.message, rootRecordsWrite)) {
|
|
3622
|
+
return undefined;
|
|
3623
|
+
}
|
|
3624
|
+
|
|
3625
|
+
return authenticatedDependency;
|
|
3626
|
+
}
|
|
3627
|
+
|
|
3628
|
+
private async toAuthenticatedProtocolConfigDependency(
|
|
3629
|
+
tenantDid: string,
|
|
3630
|
+
dependency: SyncDependencyEntryWithMessage,
|
|
3631
|
+
): Promise<AuthenticatedProtocolConfigDependency | undefined> {
|
|
3632
|
+
const config = await this.toAuthenticatedTenantProtocolConfig(tenantDid, dependency.message);
|
|
3633
|
+
if (config === undefined) {
|
|
3634
|
+
return undefined;
|
|
3635
|
+
}
|
|
3636
|
+
|
|
3637
|
+
return { ...dependency, message: config };
|
|
3638
|
+
}
|
|
3639
|
+
|
|
3640
|
+
private static async projectedDependencyCidsMatch({
|
|
3641
|
+
dependencyCid,
|
|
3642
|
+
dependencyMessage,
|
|
3643
|
+
primaryCid,
|
|
3644
|
+
primaryMessage,
|
|
3645
|
+
}: {
|
|
3646
|
+
dependencyCid: string;
|
|
3647
|
+
dependencyMessage: GenericMessage;
|
|
3648
|
+
primaryCid: string;
|
|
3649
|
+
primaryMessage: GenericMessage;
|
|
3650
|
+
}): Promise<boolean> {
|
|
3651
|
+
return await Message.getCid(primaryMessage) === primaryCid &&
|
|
3652
|
+
await Message.getCid(dependencyMessage) === dependencyCid;
|
|
3653
|
+
}
|
|
3654
|
+
|
|
3655
|
+
private static recordsWriteProtocol(message: GenericMessage | undefined): string | undefined {
|
|
3656
|
+
if (!SyncEngineLevel.isRecordsWriteProtocolMessage(message)) {
|
|
3657
|
+
return undefined;
|
|
3658
|
+
}
|
|
3659
|
+
|
|
3660
|
+
const { protocol } = message.descriptor;
|
|
3661
|
+
return typeof protocol === 'string' ? protocol : undefined;
|
|
3662
|
+
}
|
|
3663
|
+
|
|
3664
|
+
private static recordsDeleteRecordId(message: GenericMessage): string | undefined {
|
|
3665
|
+
if (!SyncEngineLevel.isRecordsDeleteMessage(message)) {
|
|
3666
|
+
return undefined;
|
|
3667
|
+
}
|
|
3668
|
+
|
|
3669
|
+
const recordId = (message.descriptor as Record<string, unknown>).recordId;
|
|
3670
|
+
return typeof recordId === 'string' ? recordId : undefined;
|
|
3671
|
+
}
|
|
3672
|
+
|
|
3673
|
+
private static protocolConfigRootRecordsWrite(
|
|
3674
|
+
primary: GenericMessage,
|
|
3675
|
+
initialWrite: RecordsWriteMessage | undefined,
|
|
3676
|
+
): RecordsWriteMessage | undefined {
|
|
3677
|
+
if (SyncEngineLevel.isRecordsWriteMessage(primary)) {
|
|
3678
|
+
return primary;
|
|
3679
|
+
}
|
|
3680
|
+
|
|
3681
|
+
return SyncEngineLevel.isRecordsDeleteMessage(primary) ? initialWrite : undefined;
|
|
3682
|
+
}
|
|
3683
|
+
|
|
3684
|
+
private static protocolsConfigureProtocol(message: ProtocolsConfigureMessage): string {
|
|
3685
|
+
return message.descriptor.definition.protocol;
|
|
3686
|
+
}
|
|
3687
|
+
|
|
3688
|
+
private static protocolsConfigureProtocolFromGenericMessage(message: GenericMessage): string | undefined {
|
|
3689
|
+
if (!SyncEngineLevel.isProtocolsConfigureDefinitionMessage(message)) {
|
|
3690
|
+
return undefined;
|
|
3691
|
+
}
|
|
1926
3692
|
|
|
1927
|
-
|
|
1928
|
-
|
|
3693
|
+
return message.descriptor.definition.protocol;
|
|
3694
|
+
}
|
|
1929
3695
|
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
delegateDid,
|
|
1937
|
-
protocol,
|
|
1938
|
-
cached : true,
|
|
1939
|
-
});
|
|
1940
|
-
permissionGrantId = grant.grant.id;
|
|
1941
|
-
}
|
|
3696
|
+
private static protocolsConfigureIsNotNewerThanRecordsWrite(
|
|
3697
|
+
protocolsConfigureMessage: GenericMessage,
|
|
3698
|
+
recordsWriteMessage: GenericMessage,
|
|
3699
|
+
): boolean {
|
|
3700
|
+
return protocolsConfigureMessage.descriptor.messageTimestamp <= recordsWriteMessage.descriptor.messageTimestamp;
|
|
3701
|
+
}
|
|
1942
3702
|
|
|
1943
|
-
|
|
3703
|
+
private static protocolsConfigureUses(message: ProtocolsConfigureMessage): string[] {
|
|
3704
|
+
const uses = message.descriptor.definition?.uses;
|
|
3705
|
+
return uses === undefined
|
|
3706
|
+
? []
|
|
3707
|
+
: Object.values(uses).filter((protocol): protocol is string => typeof protocol === 'string');
|
|
3708
|
+
}
|
|
1944
3709
|
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
(capturedPushLink !== undefined && this._activeLinks.get(target.linkKey) !== capturedPushLink);
|
|
3710
|
+
private static hasMessage<T extends MessagesSyncDiffEntry>(
|
|
3711
|
+
entry: T | undefined,
|
|
3712
|
+
): entry is T & { message: GenericMessage } {
|
|
3713
|
+
return entry?.message !== undefined;
|
|
3714
|
+
}
|
|
1951
3715
|
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
}
|
|
3716
|
+
private static isRecordsWriteProtocolMessage(message: GenericMessage | undefined): message is RecordsWriteProtocolMessage {
|
|
3717
|
+
return message?.descriptor.interface === DwnInterfaceName.Records &&
|
|
3718
|
+
message.descriptor.method === DwnMethodName.Write;
|
|
3719
|
+
}
|
|
1957
3720
|
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
3721
|
+
private static isRecordsWriteMessage(message: GenericMessage | undefined): message is RecordsWriteMessage {
|
|
3722
|
+
return SyncEngineLevel.isRecordsWriteProtocolMessage(message) &&
|
|
3723
|
+
'recordId' in message &&
|
|
3724
|
+
typeof message.recordId === 'string' &&
|
|
3725
|
+
'contextId' in message &&
|
|
3726
|
+
typeof message.contextId === 'string';
|
|
3727
|
+
}
|
|
1961
3728
|
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
if (pushLink && !isEventInScope(subMessage.event.message, pushLink.scope)) {
|
|
1967
|
-
return;
|
|
1968
|
-
}
|
|
3729
|
+
private static isRecordsDeleteMessage(message: GenericMessage): boolean {
|
|
3730
|
+
return message.descriptor.interface === DwnInterfaceName.Records &&
|
|
3731
|
+
message.descriptor.method === DwnMethodName.Delete;
|
|
3732
|
+
}
|
|
1969
3733
|
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
3734
|
+
private static isProtocolsConfigureDefinitionMessage(message: GenericMessage): message is ProtocolsConfigureMessage {
|
|
3735
|
+
return message.descriptor.interface === DwnInterfaceName.Protocols &&
|
|
3736
|
+
message.descriptor.method === DwnMethodName.Configure &&
|
|
3737
|
+
message.authorization !== undefined &&
|
|
3738
|
+
SyncEngineLevel.hasProtocolsConfigureDefinition(message.descriptor);
|
|
3739
|
+
}
|
|
1976
3740
|
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
}
|
|
3741
|
+
private static hasProtocolsConfigureDefinition(
|
|
3742
|
+
descriptor: MaybeProtocolsConfigureDefinitionDescriptor,
|
|
3743
|
+
): descriptor is ProtocolsConfigureDefinitionDescriptor {
|
|
3744
|
+
return SyncEngineLevel.isProtocolsConfigureDefinition(descriptor.definition);
|
|
3745
|
+
}
|
|
1983
3746
|
|
|
1984
|
-
|
|
1985
|
-
|
|
3747
|
+
private static isProtocolsConfigureDefinition(
|
|
3748
|
+
definition: unknown,
|
|
3749
|
+
): definition is ProtocolsConfigureDefinition {
|
|
3750
|
+
return typeof definition === 'object' &&
|
|
3751
|
+
definition !== null &&
|
|
3752
|
+
'protocol' in definition &&
|
|
3753
|
+
typeof definition.protocol === 'string';
|
|
3754
|
+
}
|
|
3755
|
+
|
|
3756
|
+
private static filterProtocolConfigClosure(
|
|
3757
|
+
primaryProtocol: string,
|
|
3758
|
+
candidates: AuthenticatedProtocolConfigDependency[],
|
|
3759
|
+
): MessagesSyncDependencyEntry[] {
|
|
3760
|
+
// Start from the primary record's protocol and walk only protocols named by
|
|
3761
|
+
// accepted, signed config definitions. This keeps composed-protocol support
|
|
3762
|
+
// narrow: the governing config can admit its `uses` targets, but arbitrary
|
|
3763
|
+
// protocol config hints cannot enter the apply set.
|
|
3764
|
+
const candidatesByProtocol = SyncEngineLevel.groupGoverningProtocolConfigCandidatesByProtocol(candidates);
|
|
3765
|
+
const visitedProtocols = new Set<string>();
|
|
3766
|
+
const pendingProtocols = [primaryProtocol];
|
|
3767
|
+
const accepted = new Map<string, MessagesSyncDependencyEntry>();
|
|
3768
|
+
|
|
3769
|
+
for (
|
|
3770
|
+
let protocol = SyncEngineLevel.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols);
|
|
3771
|
+
protocol !== undefined;
|
|
3772
|
+
protocol = SyncEngineLevel.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols)
|
|
3773
|
+
) {
|
|
3774
|
+
SyncEngineLevel.acceptProtocolConfigCandidates({
|
|
3775
|
+
protocol,
|
|
3776
|
+
candidatesByProtocol,
|
|
3777
|
+
visitedProtocols,
|
|
3778
|
+
pendingProtocols,
|
|
3779
|
+
accepted,
|
|
1986
3780
|
});
|
|
1987
|
-
|
|
3781
|
+
}
|
|
1988
3782
|
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
// or the post-flush drain will pick up the new entry.
|
|
1992
|
-
if (!pushRuntime.flushing && !pushRuntime.timer) {
|
|
1993
|
-
void this.flushPendingPushesForLink(targetKey);
|
|
1994
|
-
}
|
|
1995
|
-
};
|
|
3783
|
+
return [...accepted.values()];
|
|
3784
|
+
}
|
|
1996
3785
|
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
const
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
});
|
|
3786
|
+
private static groupGoverningProtocolConfigCandidatesByProtocol(
|
|
3787
|
+
candidates: AuthenticatedProtocolConfigDependency[],
|
|
3788
|
+
): Map<string, AuthenticatedProtocolConfigDependency> {
|
|
3789
|
+
const candidatesByProtocol = new Map<string, AuthenticatedProtocolConfigDependency>();
|
|
3790
|
+
for (const candidate of candidates) {
|
|
3791
|
+
const protocol = SyncEngineLevel.protocolsConfigureProtocol(candidate.message);
|
|
3792
|
+
const existing = candidatesByProtocol.get(protocol);
|
|
3793
|
+
if (existing !== undefined && SyncEngineLevel.isProtocolConfigCandidateAtLeastAsNew(existing, candidate)) {
|
|
3794
|
+
continue;
|
|
3795
|
+
}
|
|
2008
3796
|
|
|
2009
|
-
|
|
2010
|
-
if (reply.status.code !== 200 || !reply.subscription) {
|
|
2011
|
-
throw new Error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
|
|
3797
|
+
candidatesByProtocol.set(protocol, candidate);
|
|
2012
3798
|
}
|
|
2013
|
-
|
|
2014
|
-
this._localSubscriptions.push({
|
|
2015
|
-
linkKey : target.linkKey ?? buildLegacyCursorKey(did, dwnUrl, protocol),
|
|
2016
|
-
did,
|
|
2017
|
-
dwnUrl,
|
|
2018
|
-
delegateDid,
|
|
2019
|
-
protocol,
|
|
2020
|
-
close : async (): Promise<void> => { await reply.subscription!.close(); },
|
|
2021
|
-
});
|
|
3799
|
+
return candidatesByProtocol;
|
|
2022
3800
|
}
|
|
2023
3801
|
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
3802
|
+
private static isProtocolConfigCandidateAtLeastAsNew(
|
|
3803
|
+
existing: AuthenticatedProtocolConfigDependency,
|
|
3804
|
+
candidate: AuthenticatedProtocolConfigDependency,
|
|
3805
|
+
): boolean {
|
|
3806
|
+
const existingTimestamp = existing.message.descriptor.messageTimestamp;
|
|
3807
|
+
const candidateTimestamp = candidate.message.descriptor.messageTimestamp;
|
|
3808
|
+
if (existingTimestamp !== candidateTimestamp) {
|
|
3809
|
+
return existingTimestamp > candidateTimestamp;
|
|
3810
|
+
}
|
|
3811
|
+
return lexicographicalCompare(existing.messageCid, candidate.messageCid) >= 0;
|
|
2031
3812
|
}
|
|
2032
3813
|
|
|
2033
|
-
private
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
3814
|
+
private static takeNextUnvisitedProtocol(
|
|
3815
|
+
pendingProtocols: string[],
|
|
3816
|
+
visitedProtocols: Set<string>,
|
|
3817
|
+
): string | undefined {
|
|
3818
|
+
while (pendingProtocols.length > 0) {
|
|
3819
|
+
const protocol = pendingProtocols.shift()!;
|
|
3820
|
+
if (visitedProtocols.has(protocol)) {
|
|
3821
|
+
continue;
|
|
3822
|
+
}
|
|
3823
|
+
visitedProtocols.add(protocol);
|
|
3824
|
+
return protocol;
|
|
2039
3825
|
}
|
|
3826
|
+
return undefined;
|
|
3827
|
+
}
|
|
2040
3828
|
|
|
2041
|
-
|
|
2042
|
-
|
|
3829
|
+
private static acceptProtocolConfigCandidates({
|
|
3830
|
+
protocol,
|
|
3831
|
+
candidatesByProtocol,
|
|
3832
|
+
visitedProtocols,
|
|
3833
|
+
pendingProtocols,
|
|
3834
|
+
accepted,
|
|
3835
|
+
}: {
|
|
3836
|
+
protocol: string;
|
|
3837
|
+
candidatesByProtocol: Map<string, AuthenticatedProtocolConfigDependency>;
|
|
3838
|
+
visitedProtocols: Set<string>;
|
|
3839
|
+
pendingProtocols: string[];
|
|
3840
|
+
accepted: Map<string, MessagesSyncDependencyEntry>;
|
|
3841
|
+
}): void {
|
|
3842
|
+
const candidate = candidatesByProtocol.get(protocol);
|
|
3843
|
+
if (candidate === undefined) {
|
|
2043
3844
|
return;
|
|
2044
3845
|
}
|
|
2045
3846
|
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
const { did, dwnUrl, delegateDid, protocol, entries: pushEntries, retryCount } = pushRuntime;
|
|
2054
|
-
pushRuntime.entries = [];
|
|
3847
|
+
accepted.set(candidate.messageCid, candidate);
|
|
3848
|
+
SyncEngineLevel.queueUnvisitedProtocols(
|
|
3849
|
+
SyncEngineLevel.protocolsConfigureUses(candidate.message),
|
|
3850
|
+
visitedProtocols,
|
|
3851
|
+
pendingProtocols,
|
|
3852
|
+
);
|
|
3853
|
+
}
|
|
2055
3854
|
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
3855
|
+
private static queueUnvisitedProtocols(
|
|
3856
|
+
protocols: string[],
|
|
3857
|
+
visitedProtocols: Set<string>,
|
|
3858
|
+
pendingProtocols: string[],
|
|
3859
|
+
): void {
|
|
3860
|
+
for (const protocol of protocols) {
|
|
3861
|
+
if (!visitedProtocols.has(protocol)) {
|
|
3862
|
+
pendingProtocols.push(protocol);
|
|
2059
3863
|
}
|
|
2060
|
-
return;
|
|
2061
3864
|
}
|
|
3865
|
+
}
|
|
2062
3866
|
|
|
2063
|
-
|
|
2064
|
-
|
|
3867
|
+
private static protocolSetScopeForProjectedDependencies(
|
|
3868
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3869
|
+
): Extract<SyncScope, { kind: 'protocolSet' }> {
|
|
3870
|
+
// Verification above is the security boundary. This protocolSet scope only
|
|
3871
|
+
// routes already-verified config dependencies through the existing
|
|
3872
|
+
// pull/apply path, which expects every prefetched message to be accepted by
|
|
3873
|
+
// the supplied sync scope before it reaches processRawMessage().
|
|
3874
|
+
const protocols = SyncEngineLevel.dedupeStrings(
|
|
3875
|
+
dependencies
|
|
3876
|
+
.flatMap(dependency => SyncEngineLevel.projectedDependencyProtocols(dependency))
|
|
3877
|
+
).sort(lexicographicalCompare);
|
|
3878
|
+
if (protocols.length === 0) {
|
|
3879
|
+
throw new Error('SyncEngineLevel: projected dependency hints contained no protocols.');
|
|
3880
|
+
}
|
|
2065
3881
|
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
permissionsApi : this._permissionsApi,
|
|
2072
|
-
});
|
|
3882
|
+
return {
|
|
3883
|
+
kind : 'protocolSet',
|
|
3884
|
+
protocols : protocols as NonEmptyStringArray,
|
|
3885
|
+
};
|
|
3886
|
+
}
|
|
2073
3887
|
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
3888
|
+
private static projectedDependencyProtocols(dependency: MessagesSyncDependencyEntry): string[] {
|
|
3889
|
+
if (dependency.message === undefined) {
|
|
3890
|
+
return [];
|
|
3891
|
+
}
|
|
2077
3892
|
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
3893
|
+
const protocol = dependency.dependencyClass === 'protocolsConfigure'
|
|
3894
|
+
? SyncEngineLevel.protocolsConfigureProtocolFromGenericMessage(dependency.message)
|
|
3895
|
+
: SyncEngineLevel.recordsWriteProtocol(dependency.message);
|
|
3896
|
+
return protocol === undefined ? [] : [protocol];
|
|
3897
|
+
}
|
|
2083
3898
|
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
tenantDid : did,
|
|
2089
|
-
remoteEndpoint : dwnUrl,
|
|
2090
|
-
protocol,
|
|
2091
|
-
category : 'push-permanent',
|
|
2092
|
-
errorCode : String(entry.statusCode ?? ''),
|
|
2093
|
-
errorDetail : entry.detail ?? 'permanent push failure',
|
|
2094
|
-
});
|
|
2095
|
-
}
|
|
3899
|
+
private static hasDependencyPayloadBytes(dependency: MessagesSyncDependencyEntry): boolean {
|
|
3900
|
+
if (dependency.encodedData !== undefined) {
|
|
3901
|
+
return true;
|
|
3902
|
+
}
|
|
2096
3903
|
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
const failedEntries = pushEntries.filter((entry) => failedSet.has(entry.cid));
|
|
2101
|
-
this.requeueOrReconcile(linkKey, {
|
|
2102
|
-
did, dwnUrl, delegateDid, protocol,
|
|
2103
|
-
entries : failedEntries,
|
|
2104
|
-
retryCount : retryCount + 1,
|
|
2105
|
-
});
|
|
2106
|
-
} else {
|
|
2107
|
-
// Successful push — reset retry count so subsequent unrelated
|
|
2108
|
-
// batches on this link start with a fresh budget.
|
|
2109
|
-
pushRuntime.retryCount = 0;
|
|
2110
|
-
if (!pushRuntime.timer && pushRuntime.entries.length === 0) {
|
|
2111
|
-
this._pushRuntimes.delete(linkKey);
|
|
2112
|
-
}
|
|
2113
|
-
}
|
|
2114
|
-
} catch (error: any) {
|
|
2115
|
-
if (isFlushStale()) { return; }
|
|
2116
|
-
console.error(`SyncEngineLevel: Push batch failed for ${did} -> ${dwnUrl}`, error);
|
|
2117
|
-
this.requeueOrReconcile(linkKey, {
|
|
2118
|
-
did, dwnUrl, delegateDid, protocol,
|
|
2119
|
-
entries : pushEntries,
|
|
2120
|
-
retryCount : retryCount + 1,
|
|
2121
|
-
});
|
|
2122
|
-
} finally {
|
|
2123
|
-
pushRuntime.flushing = false;
|
|
3904
|
+
const message = dependency.message as Record<string, unknown> | undefined;
|
|
3905
|
+
return message !== undefined && 'encodedData' in message;
|
|
3906
|
+
}
|
|
2124
3907
|
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
rt.timer = undefined;
|
|
2132
|
-
void this.flushPendingPushesForLink(linkKey);
|
|
2133
|
-
}, PUSH_DEBOUNCE_MS);
|
|
2134
|
-
}
|
|
3908
|
+
private static dedupeDependencyEntries(
|
|
3909
|
+
dependencies: Iterable<MessagesSyncDependencyEntry>,
|
|
3910
|
+
): MessagesSyncDependencyEntry[] {
|
|
3911
|
+
const deduped = new Map<string, MessagesSyncDependencyEntry>();
|
|
3912
|
+
for (const dependency of dependencies) {
|
|
3913
|
+
deduped.set(dependency.messageCid, dependency);
|
|
2135
3914
|
}
|
|
3915
|
+
return [...deduped.values()];
|
|
2136
3916
|
}
|
|
2137
3917
|
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
3918
|
+
private async pushLocalDiffEntries(
|
|
3919
|
+
target: ProjectionReconcileTarget,
|
|
3920
|
+
onlyLocal: string[],
|
|
3921
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3922
|
+
shouldContinue?: () => boolean,
|
|
3923
|
+
): Promise<boolean> {
|
|
3924
|
+
await this.pushMessages({
|
|
3925
|
+
did : target.did,
|
|
3926
|
+
dwnUrl : target.dwnUrl,
|
|
3927
|
+
delegateDid : target.delegateDid,
|
|
3928
|
+
permissionGrantIds,
|
|
3929
|
+
messageCids : SyncEngineLevel.dedupeStrings(onlyLocal),
|
|
3930
|
+
});
|
|
3931
|
+
return shouldContinue?.() === false;
|
|
3932
|
+
}
|
|
2153
3933
|
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
}
|
|
2166
|
-
}
|
|
2167
|
-
if (pushRuntime.timer) {
|
|
2168
|
-
clearTimeout(pushRuntime.timer);
|
|
2169
|
-
}
|
|
2170
|
-
this._pushRuntimes.delete(targetKey);
|
|
2171
|
-
const link = this._activeLinks.get(targetKey);
|
|
2172
|
-
if (link && !link.needsReconcile) {
|
|
2173
|
-
link.needsReconcile = true;
|
|
2174
|
-
void this.ledger.saveLink(link).then(() => {
|
|
2175
|
-
this.emitEvent({ type: 'reconcile:needed', tenantDid: pending.did, remoteEndpoint: pending.dwnUrl, protocol: pending.protocol, reason: 'push-retry-exhausted' });
|
|
2176
|
-
this.scheduleReconcile(targetKey);
|
|
2177
|
-
});
|
|
3934
|
+
private async verifyProtocolSetConvergence(
|
|
3935
|
+
target: ProjectionReconcileTarget,
|
|
3936
|
+
changedProtocols: string[],
|
|
3937
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3938
|
+
shouldContinue?: () => boolean,
|
|
3939
|
+
): Promise<ProjectionReconcileResult> {
|
|
3940
|
+
for (const protocol of changedProtocols) {
|
|
3941
|
+
const roots = await this.getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue);
|
|
3942
|
+
if (!roots) { return { aborted: true }; }
|
|
3943
|
+
|
|
3944
|
+
if (roots.localRoot !== roots.remoteRoot) {
|
|
3945
|
+
return { converged: false };
|
|
2178
3946
|
}
|
|
2179
|
-
return;
|
|
2180
3947
|
}
|
|
2181
3948
|
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
3949
|
+
return { converged: true };
|
|
3950
|
+
}
|
|
3951
|
+
|
|
3952
|
+
private static dedupeRemoteEntries(entries: MessagesSyncDiffEntry[]): MessagesSyncDiffEntry[] {
|
|
3953
|
+
const seen = new Set<string>();
|
|
3954
|
+
const unique: MessagesSyncDiffEntry[] = [];
|
|
3955
|
+
for (const entry of entries) {
|
|
3956
|
+
if (seen.has(entry.messageCid)) {
|
|
3957
|
+
continue;
|
|
3958
|
+
}
|
|
3959
|
+
seen.add(entry.messageCid);
|
|
3960
|
+
unique.push(entry);
|
|
2187
3961
|
}
|
|
2188
|
-
|
|
2189
|
-
pushRuntime.timer = undefined;
|
|
2190
|
-
void this.flushPendingPushesForLink(targetKey);
|
|
2191
|
-
}, delayMs);
|
|
3962
|
+
return unique;
|
|
2192
3963
|
}
|
|
2193
3964
|
|
|
2194
|
-
private
|
|
2195
|
-
return new
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
3965
|
+
private static dedupeStrings(values: string[]): string[] {
|
|
3966
|
+
return [...new Set(values)];
|
|
3967
|
+
}
|
|
3968
|
+
|
|
3969
|
+
private async clearRootConvergenceDeadLettersForScope(
|
|
3970
|
+
tenantDid: string,
|
|
3971
|
+
remoteEndpoint: string,
|
|
3972
|
+
scope: SyncScope,
|
|
3973
|
+
): Promise<void> {
|
|
3974
|
+
if (scope.kind === 'recordsProjection' || (scope.kind === 'protocolSet' && scope.protocols.length > 1)) {
|
|
3975
|
+
// Batched multi-protocol and projected pulls pass the full scope to
|
|
3976
|
+
// pullMessages, so pull dead letters can be recorded without a single
|
|
3977
|
+
// protocol bucket.
|
|
3978
|
+
await this.clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint);
|
|
3979
|
+
}
|
|
3980
|
+
|
|
3981
|
+
for (const protocol of this.getReconcileProtocols(scope)) {
|
|
3982
|
+
await this.clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint, protocol);
|
|
3983
|
+
}
|
|
2203
3984
|
}
|
|
2204
3985
|
|
|
2205
3986
|
// ---------------------------------------------------------------------------
|
|
@@ -2277,21 +4058,30 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2277
4058
|
// closure's captured `link` reference may no longer be the active
|
|
2278
4059
|
// link object. Bail before mutating the replacement's state.
|
|
2279
4060
|
const isStaleLink = (): boolean => this._activeLinks.get(linkKey) !== link;
|
|
4061
|
+
const shouldContinue = (): boolean =>
|
|
4062
|
+
this._engineGeneration === generation &&
|
|
4063
|
+
!isStaleLink() &&
|
|
4064
|
+
link.status === 'live';
|
|
2280
4065
|
|
|
2281
|
-
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid,
|
|
4066
|
+
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, scope, authorization } = link;
|
|
4067
|
+
const eventScope = syncEventScope(scope);
|
|
2282
4068
|
|
|
2283
4069
|
try {
|
|
2284
|
-
const reconcileOutcome = await this.
|
|
2285
|
-
|
|
2286
|
-
|
|
4070
|
+
const reconcileOutcome = await this.reconcileProjectionTarget({
|
|
4071
|
+
did,
|
|
4072
|
+
dwnUrl,
|
|
4073
|
+
delegateDid,
|
|
4074
|
+
scope,
|
|
4075
|
+
authorization,
|
|
4076
|
+
}, { verifyConvergence: true }, shouldContinue);
|
|
2287
4077
|
if (reconcileOutcome.aborted || isStaleLink()) { return; }
|
|
2288
4078
|
|
|
2289
4079
|
if (reconcileOutcome.converged) {
|
|
2290
4080
|
await this.ledger.clearNeedsReconcile(link);
|
|
2291
|
-
// SMT roots match
|
|
2292
|
-
//
|
|
2293
|
-
|
|
2294
|
-
this.emitEvent({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
4081
|
+
// SMT roots match, so transport/apply failures for this link may no
|
|
4082
|
+
// longer be current. Closure failures are not cleared by root equality.
|
|
4083
|
+
await this.clearRootConvergenceDeadLettersForScope(did, dwnUrl, scope);
|
|
4084
|
+
this.emitEvent({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope });
|
|
2295
4085
|
} else {
|
|
2296
4086
|
// Roots still differ — retry after a delay. This can happen when
|
|
2297
4087
|
// pushMessages() had permanent failures, pullMessages() partially
|
|
@@ -2301,136 +4091,46 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2301
4091
|
} catch (error: any) {
|
|
2302
4092
|
if (isStaleLink()) { return; }
|
|
2303
4093
|
console.error(`SyncEngineLevel: Reconciliation failed for ${did} -> ${dwnUrl}`, error);
|
|
2304
|
-
// Schedule retry with longer delay.
|
|
2305
|
-
this.scheduleReconcile(linkKey, 5000);
|
|
2306
|
-
}
|
|
2307
|
-
}
|
|
2308
|
-
|
|
2309
|
-
private getOrCreatePushRuntime(linkKey: string, params: {
|
|
2310
|
-
did: string;
|
|
2311
|
-
dwnUrl: string;
|
|
2312
|
-
delegateDid?: string;
|
|
2313
|
-
protocol?: string;
|
|
2314
|
-
}): PushRuntimeState {
|
|
2315
|
-
let pushRuntime = this._pushRuntimes.get(linkKey);
|
|
2316
|
-
if (!pushRuntime) {
|
|
2317
|
-
pushRuntime = {
|
|
2318
|
-
...params,
|
|
2319
|
-
entries : [],
|
|
2320
|
-
retryCount : 0,
|
|
2321
|
-
};
|
|
2322
|
-
this._pushRuntimes.set(linkKey, pushRuntime);
|
|
2323
|
-
}
|
|
2324
|
-
|
|
2325
|
-
return pushRuntime;
|
|
2326
|
-
}
|
|
2327
|
-
|
|
2328
|
-
// ---------------------------------------------------------------------------
|
|
2329
|
-
// Cursor persistence
|
|
2330
|
-
// ---------------------------------------------------------------------------
|
|
2331
|
-
|
|
2332
|
-
/**
|
|
2333
|
-
* Build the runtime key for a replication link.
|
|
2334
|
-
*
|
|
2335
|
-
* Live-mode subscription methods (`openLivePullSubscription`,
|
|
2336
|
-
* `openLocalPushSubscription`) receive `linkKey` directly and never
|
|
2337
|
-
* call this. The remaining callers are poll-mode `sync()` and the
|
|
2338
|
-
* live-mode startup/error paths that already have `link.scopeId`.
|
|
2339
|
-
*
|
|
2340
|
-
* The `undefined` fallback (which produces a legacy cursor key) exists
|
|
2341
|
-
* only for the no-protocol full-tenant targets in poll mode.
|
|
2342
|
-
*/
|
|
2343
|
-
private buildLinkKey(did: string, dwnUrl: string, scopeIdOrProtocol?: string): string {
|
|
2344
|
-
return scopeIdOrProtocol ? buildLinkId(did, dwnUrl, scopeIdOrProtocol) : buildLegacyCursorKey(did, dwnUrl);
|
|
2345
|
-
}
|
|
2346
|
-
|
|
2347
|
-
/**
|
|
2348
|
-
* @deprecated Used by poll-mode sync and one-time migration only. Live mode
|
|
2349
|
-
* uses ReplicationLedger checkpoints. Handles migration from old string cursors:
|
|
2350
|
-
* if the stored value is a bare string (pre-ProgressToken format), it is treated
|
|
2351
|
-
* as absent — the sync engine will do a full SMT reconciliation on first startup
|
|
2352
|
-
* after upgrade, which is correct and safe.
|
|
2353
|
-
*/
|
|
2354
|
-
private async getCursor(key: string): Promise<ProgressToken | undefined> {
|
|
2355
|
-
const cursors = this._db.sublevel('syncCursors');
|
|
2356
|
-
try {
|
|
2357
|
-
const raw = await cursors.get(key);
|
|
2358
|
-
try {
|
|
2359
|
-
const parsed = JSON.parse(raw);
|
|
2360
|
-
if (parsed && typeof parsed === 'object' &&
|
|
2361
|
-
typeof parsed.streamId === 'string' && parsed.streamId.length > 0 &&
|
|
2362
|
-
typeof parsed.epoch === 'string' && parsed.epoch.length > 0 &&
|
|
2363
|
-
typeof parsed.position === 'string' && parsed.position.length > 0 &&
|
|
2364
|
-
typeof parsed.messageCid === 'string' && parsed.messageCid.length > 0) {
|
|
2365
|
-
return parsed as ProgressToken;
|
|
2366
|
-
}
|
|
2367
|
-
} catch {
|
|
2368
|
-
// Not valid JSON (old string cursor) — fall through to delete.
|
|
2369
|
-
}
|
|
2370
|
-
// Entry exists but is unparseable or has invalid/empty fields. Delete it
|
|
2371
|
-
// so subsequent startups don't re-check it on every launch.
|
|
2372
|
-
await this.deleteLegacyCursor(key);
|
|
2373
|
-
return undefined;
|
|
2374
|
-
} catch (error) {
|
|
2375
|
-
const e = error as { code: string };
|
|
2376
|
-
if (e.code === 'LEVEL_NOT_FOUND') {
|
|
2377
|
-
return undefined;
|
|
2378
|
-
}
|
|
2379
|
-
throw error;
|
|
4094
|
+
// Schedule retry with longer delay.
|
|
4095
|
+
this.scheduleReconcile(linkKey, 5000);
|
|
2380
4096
|
}
|
|
2381
4097
|
}
|
|
2382
4098
|
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
4099
|
+
private getOrCreatePushRuntime(linkKey: string, params: {
|
|
4100
|
+
did: string;
|
|
4101
|
+
dwnUrl: string;
|
|
4102
|
+
delegateDid?: string;
|
|
4103
|
+
protocol?: string;
|
|
4104
|
+
permissionGrantIds?: NonEmptyStringArray;
|
|
4105
|
+
}): PushRuntimeState {
|
|
4106
|
+
let pushRuntime = this._pushRuntimes.get(linkKey);
|
|
4107
|
+
if (!pushRuntime) {
|
|
4108
|
+
pushRuntime = {
|
|
4109
|
+
...params,
|
|
4110
|
+
entries : [],
|
|
4111
|
+
retryCount : 0,
|
|
4112
|
+
};
|
|
4113
|
+
this._pushRuntimes.set(linkKey, pushRuntime);
|
|
2396
4114
|
}
|
|
4115
|
+
|
|
4116
|
+
return pushRuntime;
|
|
2397
4117
|
}
|
|
2398
4118
|
|
|
2399
4119
|
// ---------------------------------------------------------------------------
|
|
2400
|
-
//
|
|
4120
|
+
// Cursor persistence
|
|
2401
4121
|
// ---------------------------------------------------------------------------
|
|
2402
4122
|
|
|
2403
4123
|
/**
|
|
2404
|
-
*
|
|
2405
|
-
*
|
|
2406
|
-
*
|
|
4124
|
+
* Build the runtime key for a replication link.
|
|
4125
|
+
*
|
|
4126
|
+
* Live-mode subscription methods (`openLivePullSubscription`,
|
|
4127
|
+
* `openLocalPushSubscription`) receive `linkKey` directly and never
|
|
4128
|
+
* call this. The remaining callers are poll-mode `sync()` and the
|
|
4129
|
+
* live-mode startup/error paths that already have a projection ID and
|
|
4130
|
+
* authorization epoch.
|
|
2407
4131
|
*/
|
|
2408
|
-
private
|
|
2409
|
-
|
|
2410
|
-
return undefined;
|
|
2411
|
-
}
|
|
2412
|
-
|
|
2413
|
-
// Check for inline base64url-encoded data (small records from EventLog).
|
|
2414
|
-
// Delete the transport-level field so the DWN schema validator does not
|
|
2415
|
-
// reject the message for having unevaluated properties.
|
|
2416
|
-
const encodedData = (event.message as any).encodedData as string | undefined;
|
|
2417
|
-
if (encodedData) {
|
|
2418
|
-
delete (event.message as any).encodedData;
|
|
2419
|
-
const bytes = Encoder.base64UrlToBytes(encodedData);
|
|
2420
|
-
return new ReadableStream<Uint8Array>({
|
|
2421
|
-
start(controller): void {
|
|
2422
|
-
controller.enqueue(bytes);
|
|
2423
|
-
controller.close();
|
|
2424
|
-
}
|
|
2425
|
-
});
|
|
2426
|
-
}
|
|
2427
|
-
|
|
2428
|
-
// Check for a pre-existing data stream (e.g. from a direct message read).
|
|
2429
|
-
if ((event as any).data) {
|
|
2430
|
-
return (event as any).data;
|
|
2431
|
-
}
|
|
2432
|
-
|
|
2433
|
-
return undefined;
|
|
4132
|
+
private buildLinkKey(did: string, dwnUrl: string, projectionId: string, authorizationEpoch: string): string {
|
|
4133
|
+
return buildLinkId(did, dwnUrl, projectionId, authorizationEpoch);
|
|
2434
4134
|
}
|
|
2435
4135
|
|
|
2436
4136
|
// ---------------------------------------------------------------------------
|
|
@@ -2492,7 +4192,12 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2492
4192
|
*
|
|
2493
4193
|
* Returns a hex-encoded root hash string.
|
|
2494
4194
|
*/
|
|
2495
|
-
private async getLocalRoot(
|
|
4195
|
+
private async getLocalRoot(
|
|
4196
|
+
did: string,
|
|
4197
|
+
delegateDid?: string,
|
|
4198
|
+
protocol?: string,
|
|
4199
|
+
permissionGrantIds?: string[],
|
|
4200
|
+
): Promise<string> {
|
|
2496
4201
|
const si = this.stateIndex;
|
|
2497
4202
|
if (si) {
|
|
2498
4203
|
const rootHash = protocol === undefined
|
|
@@ -2502,16 +4207,15 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2502
4207
|
}
|
|
2503
4208
|
|
|
2504
4209
|
// Remote mode fallback: go through processRequest → RPC.
|
|
2505
|
-
const permissionGrantId = await this.getSyncPermissionGrantId(did, delegateDid, protocol);
|
|
2506
4210
|
const response = await this.agent.dwn.processRequest({
|
|
2507
4211
|
author : did,
|
|
2508
4212
|
target : did,
|
|
2509
4213
|
messageType : DwnInterface.MessagesSync,
|
|
2510
4214
|
granteeDid : delegateDid,
|
|
2511
4215
|
messageParams : {
|
|
2512
|
-
action: 'root',
|
|
4216
|
+
action : 'root',
|
|
2513
4217
|
protocol,
|
|
2514
|
-
|
|
4218
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
|
|
2515
4219
|
}
|
|
2516
4220
|
});
|
|
2517
4221
|
const reply = response.reply as MessagesSyncReply;
|
|
@@ -2522,9 +4226,13 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2522
4226
|
* Get the SMT root hash from a remote DWN via a MessagesSync 'root' action.
|
|
2523
4227
|
* Returns a hex-encoded root hash string.
|
|
2524
4228
|
*/
|
|
2525
|
-
private async getRemoteRoot(
|
|
2526
|
-
|
|
2527
|
-
|
|
4229
|
+
private async getRemoteRoot(
|
|
4230
|
+
did: string,
|
|
4231
|
+
dwnUrl: string,
|
|
4232
|
+
delegateDid?: string,
|
|
4233
|
+
protocol?: string,
|
|
4234
|
+
permissionGrantIds?: string[],
|
|
4235
|
+
): Promise<string> {
|
|
2528
4236
|
const syncMessage = await this.agent.dwn.processRequest({
|
|
2529
4237
|
store : false,
|
|
2530
4238
|
author : did,
|
|
@@ -2532,9 +4240,71 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2532
4240
|
messageType : DwnInterface.MessagesSync,
|
|
2533
4241
|
granteeDid : delegateDid,
|
|
2534
4242
|
messageParams : {
|
|
2535
|
-
action: 'root',
|
|
4243
|
+
action : 'root',
|
|
2536
4244
|
protocol,
|
|
2537
|
-
|
|
4245
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
|
|
4246
|
+
}
|
|
4247
|
+
});
|
|
4248
|
+
|
|
4249
|
+
const reply = await this.agent.rpc.sendDwnRequest({
|
|
4250
|
+
dwnUrl,
|
|
4251
|
+
targetDid : did,
|
|
4252
|
+
message : syncMessage.message,
|
|
4253
|
+
}) as MessagesSyncReply;
|
|
4254
|
+
|
|
4255
|
+
return reply.root ?? '';
|
|
4256
|
+
}
|
|
4257
|
+
|
|
4258
|
+
private async getLocalProjectedRoot(
|
|
4259
|
+
did: string,
|
|
4260
|
+
delegateDid: string | undefined,
|
|
4261
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
|
|
4262
|
+
permissionGrantIds?: string[],
|
|
4263
|
+
): Promise<string> {
|
|
4264
|
+
if (this.stateIndex) {
|
|
4265
|
+
// Local projected roots use the already-derived scope directly. The
|
|
4266
|
+
// remote root/diff request still re-authorizes the invoked grant set.
|
|
4267
|
+
return RecordsProjection.getRootHex({
|
|
4268
|
+
tenant : did,
|
|
4269
|
+
messageStore : this.agent.dwn.node.storage.messageStore,
|
|
4270
|
+
scopes,
|
|
4271
|
+
});
|
|
4272
|
+
}
|
|
4273
|
+
|
|
4274
|
+
const response = await this.agent.dwn.processRequest({
|
|
4275
|
+
author : did,
|
|
4276
|
+
target : did,
|
|
4277
|
+
messageType : DwnInterface.MessagesSync,
|
|
4278
|
+
granteeDid : delegateDid,
|
|
4279
|
+
messageParams : {
|
|
4280
|
+
action : 'root',
|
|
4281
|
+
projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
|
|
4282
|
+
projectionScopes : [...scopes],
|
|
4283
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
|
|
4284
|
+
}
|
|
4285
|
+
});
|
|
4286
|
+
const reply = response.reply as MessagesSyncReply;
|
|
4287
|
+
return reply.root ?? '';
|
|
4288
|
+
}
|
|
4289
|
+
|
|
4290
|
+
private async getRemoteProjectedRoot(
|
|
4291
|
+
did: string,
|
|
4292
|
+
dwnUrl: string,
|
|
4293
|
+
delegateDid: string | undefined,
|
|
4294
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
|
|
4295
|
+
permissionGrantIds?: string[],
|
|
4296
|
+
): Promise<string> {
|
|
4297
|
+
const syncMessage = await this.agent.dwn.processRequest({
|
|
4298
|
+
store : false,
|
|
4299
|
+
author : did,
|
|
4300
|
+
target : did,
|
|
4301
|
+
messageType : DwnInterface.MessagesSync,
|
|
4302
|
+
granteeDid : delegateDid,
|
|
4303
|
+
messageParams : {
|
|
4304
|
+
action : 'root',
|
|
4305
|
+
projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
|
|
4306
|
+
projectionScopes : [...scopes],
|
|
4307
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
|
|
2538
4308
|
}
|
|
2539
4309
|
});
|
|
2540
4310
|
|
|
@@ -2565,55 +4335,102 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2565
4335
|
*
|
|
2566
4336
|
* This replaces `walkTreeDiff()` which required one HTTP call per tree node.
|
|
2567
4337
|
*/
|
|
2568
|
-
private async diffWithRemote({ did, dwnUrl, delegateDid, protocol }: {
|
|
4338
|
+
private async diffWithRemote({ did, dwnUrl, delegateDid, protocol, permissionGrantIds }: {
|
|
2569
4339
|
did: string;
|
|
2570
4340
|
dwnUrl: string;
|
|
2571
4341
|
delegateDid?: string;
|
|
2572
4342
|
protocol?: string;
|
|
2573
|
-
|
|
4343
|
+
permissionGrantIds?: string[];
|
|
4344
|
+
}): Promise<ProjectionDiffResult> {
|
|
2574
4345
|
// Step 1: Collect local subtree hashes at BATCHED_DIFF_DEPTH directly from StateIndex.
|
|
2575
|
-
const localHashes = await this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH);
|
|
4346
|
+
const localHashes = await this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH, permissionGrantIds);
|
|
2576
4347
|
|
|
2577
4348
|
// Step 2: Send a single 'diff' request to the remote with our hashes.
|
|
2578
|
-
const
|
|
4349
|
+
const messageParams: DwnMessageParams[DwnInterface.MessagesSync] = {
|
|
4350
|
+
action : 'diff',
|
|
4351
|
+
protocol,
|
|
4352
|
+
hashes : localHashes,
|
|
4353
|
+
depth : BATCHED_DIFF_DEPTH,
|
|
4354
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
|
|
4355
|
+
};
|
|
4356
|
+
|
|
4357
|
+
// Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
|
|
4358
|
+
// Reuse the same grant set from step 2.
|
|
4359
|
+
return this.diffRemoteMessages(
|
|
4360
|
+
{ did, dwnUrl, delegateDid },
|
|
4361
|
+
messageParams,
|
|
4362
|
+
prefix => this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantIds),
|
|
4363
|
+
'diff',
|
|
4364
|
+
);
|
|
4365
|
+
}
|
|
4366
|
+
|
|
4367
|
+
private async diffProjectedWithRemote({ did, dwnUrl, delegateDid, scopes, permissionGrantIds }: {
|
|
4368
|
+
did: string;
|
|
4369
|
+
dwnUrl: string;
|
|
4370
|
+
delegateDid?: string;
|
|
4371
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]];
|
|
4372
|
+
permissionGrantIds?: string[];
|
|
4373
|
+
}): Promise<ProjectionDiffResult> {
|
|
4374
|
+
const localHashes = await this.collectLocalProjectedSubtreeHashes(
|
|
4375
|
+
did,
|
|
4376
|
+
scopes,
|
|
4377
|
+
BATCHED_DIFF_DEPTH,
|
|
4378
|
+
delegateDid,
|
|
4379
|
+
permissionGrantIds,
|
|
4380
|
+
);
|
|
4381
|
+
|
|
4382
|
+
const messageParams: DwnMessageParams[DwnInterface.MessagesSync] = {
|
|
4383
|
+
action : 'diff',
|
|
4384
|
+
projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
|
|
4385
|
+
projectionScopes : [...scopes],
|
|
4386
|
+
hashes : localHashes,
|
|
4387
|
+
depth : BATCHED_DIFF_DEPTH,
|
|
4388
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
|
|
4389
|
+
};
|
|
4390
|
+
|
|
4391
|
+
return this.diffRemoteMessages(
|
|
4392
|
+
{ did, dwnUrl, delegateDid },
|
|
4393
|
+
messageParams,
|
|
4394
|
+
prefix => this.getLocalProjectedLeaves(did, prefix, delegateDid, scopes, permissionGrantIds),
|
|
4395
|
+
'projected diff',
|
|
4396
|
+
);
|
|
4397
|
+
}
|
|
2579
4398
|
|
|
4399
|
+
private async diffRemoteMessages(
|
|
4400
|
+
target: Pick<ProjectionReconcileTarget, 'did' | 'dwnUrl' | 'delegateDid'>,
|
|
4401
|
+
messageParams: DwnMessageParams[DwnInterface.MessagesSync],
|
|
4402
|
+
getLocalLeavesForPrefix: (prefix: string) => Promise<string[]>,
|
|
4403
|
+
operationName: string,
|
|
4404
|
+
): Promise<ProjectionDiffResult> {
|
|
2580
4405
|
const syncMessage = await this.agent.dwn.processRequest({
|
|
2581
|
-
store
|
|
2582
|
-
author
|
|
2583
|
-
target
|
|
2584
|
-
messageType
|
|
2585
|
-
granteeDid
|
|
2586
|
-
messageParams
|
|
2587
|
-
action : 'diff',
|
|
2588
|
-
protocol,
|
|
2589
|
-
hashes : localHashes,
|
|
2590
|
-
depth : BATCHED_DIFF_DEPTH,
|
|
2591
|
-
permissionGrantId,
|
|
2592
|
-
}
|
|
4406
|
+
store : false,
|
|
4407
|
+
author : target.did,
|
|
4408
|
+
target : target.did,
|
|
4409
|
+
messageType : DwnInterface.MessagesSync,
|
|
4410
|
+
granteeDid : target.delegateDid,
|
|
4411
|
+
messageParams,
|
|
2593
4412
|
});
|
|
2594
4413
|
|
|
2595
4414
|
const reply = await this.agent.rpc.sendDwnRequest({
|
|
2596
|
-
dwnUrl,
|
|
2597
|
-
targetDid : did,
|
|
4415
|
+
dwnUrl : target.dwnUrl,
|
|
4416
|
+
targetDid : target.did,
|
|
2598
4417
|
message : syncMessage.message,
|
|
2599
4418
|
}) as MessagesSyncReply;
|
|
2600
4419
|
|
|
2601
4420
|
if (reply.status.code !== 200) {
|
|
2602
|
-
throw new Error(`SyncEngineLevel:
|
|
4421
|
+
throw new Error(`SyncEngineLevel: ${operationName} failed with ${reply.status.code}: ${reply.status.detail}`);
|
|
2603
4422
|
}
|
|
2604
4423
|
|
|
2605
|
-
// Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
|
|
2606
|
-
// Reuse the same grant ID from step 2 (avoids redundant lookup).
|
|
2607
|
-
const permissionGrantIdForLeaves = permissionGrantId;
|
|
2608
4424
|
const onlyLocalCids: string[] = [];
|
|
2609
4425
|
for (const prefix of reply.onlyLocal ?? []) {
|
|
2610
|
-
const leaves = await
|
|
4426
|
+
const leaves = await getLocalLeavesForPrefix(prefix);
|
|
2611
4427
|
onlyLocalCids.push(...leaves);
|
|
2612
4428
|
}
|
|
2613
4429
|
|
|
2614
4430
|
return {
|
|
2615
|
-
|
|
2616
|
-
|
|
4431
|
+
dependencies : reply.dependencies ?? [],
|
|
4432
|
+
onlyRemote : reply.onlyRemote ?? [],
|
|
4433
|
+
onlyLocal : onlyLocalCids,
|
|
2617
4434
|
};
|
|
2618
4435
|
}
|
|
2619
4436
|
|
|
@@ -2629,6 +4446,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2629
4446
|
did: string,
|
|
2630
4447
|
protocol: string | undefined,
|
|
2631
4448
|
depth: number,
|
|
4449
|
+
permissionGrantIds?: string[],
|
|
2632
4450
|
): Promise<Record<string, string>> {
|
|
2633
4451
|
const result: Record<string, string> = {};
|
|
2634
4452
|
const defaultHash = await this.getDefaultHashHex(depth);
|
|
@@ -2646,7 +4464,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2646
4464
|
hexHash = hashToHex(hash);
|
|
2647
4465
|
} else {
|
|
2648
4466
|
// Remote mode fallback.
|
|
2649
|
-
hexHash = await this.getLocalSubtreeHash(did, prefix, undefined, protocol);
|
|
4467
|
+
hexHash = await this.getLocalSubtreeHash(did, prefix, undefined, protocol, permissionGrantIds);
|
|
2650
4468
|
}
|
|
2651
4469
|
|
|
2652
4470
|
if (hexHash === defaultHash) {
|
|
@@ -2670,6 +4488,52 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2670
4488
|
return result;
|
|
2671
4489
|
}
|
|
2672
4490
|
|
|
4491
|
+
private async collectLocalProjectedSubtreeHashes(
|
|
4492
|
+
did: string,
|
|
4493
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
|
|
4494
|
+
depth: number,
|
|
4495
|
+
delegateDid?: string,
|
|
4496
|
+
permissionGrantIds?: string[],
|
|
4497
|
+
): Promise<Record<string, string>> {
|
|
4498
|
+
const result: Record<string, string> = {};
|
|
4499
|
+
const snapshot = this.stateIndex
|
|
4500
|
+
? await RecordsProjection.createSnapshot({
|
|
4501
|
+
tenant : did,
|
|
4502
|
+
messageStore : this.agent.dwn.node.storage.messageStore,
|
|
4503
|
+
scopes,
|
|
4504
|
+
})
|
|
4505
|
+
: undefined;
|
|
4506
|
+
|
|
4507
|
+
try {
|
|
4508
|
+
const walk = async (prefix: string, currentDepth: number): Promise<void> => {
|
|
4509
|
+
const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
|
|
4510
|
+
const hexHash = snapshot
|
|
4511
|
+
? hashToHex(await snapshot.getSubtreeHash(bitPath))
|
|
4512
|
+
: await this.getLocalProjectedSubtreeHash(did, prefix, delegateDid, scopes, permissionGrantIds);
|
|
4513
|
+
const defaultHash = await this.getDefaultHashHex(currentDepth);
|
|
4514
|
+
|
|
4515
|
+
if (hexHash === defaultHash) {
|
|
4516
|
+
return;
|
|
4517
|
+
}
|
|
4518
|
+
|
|
4519
|
+
if (currentDepth >= depth) {
|
|
4520
|
+
result[prefix] = hexHash;
|
|
4521
|
+
return;
|
|
4522
|
+
}
|
|
4523
|
+
|
|
4524
|
+
await Promise.all([
|
|
4525
|
+
walk(prefix + '0', currentDepth + 1),
|
|
4526
|
+
walk(prefix + '1', currentDepth + 1),
|
|
4527
|
+
]);
|
|
4528
|
+
};
|
|
4529
|
+
|
|
4530
|
+
await walk('', 0);
|
|
4531
|
+
return result;
|
|
4532
|
+
} finally {
|
|
4533
|
+
await snapshot?.close();
|
|
4534
|
+
}
|
|
4535
|
+
}
|
|
4536
|
+
|
|
2673
4537
|
/**
|
|
2674
4538
|
* Get the subtree hash at a given bit prefix from the local DWN.
|
|
2675
4539
|
*
|
|
@@ -2677,7 +4541,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2677
4541
|
* In remote mode: constructs a signed MessagesSync message and routes through RPC.
|
|
2678
4542
|
*/
|
|
2679
4543
|
private async getLocalSubtreeHash(
|
|
2680
|
-
did: string, prefix: string, delegateDid?: string, protocol?: string,
|
|
4544
|
+
did: string, prefix: string, delegateDid?: string, protocol?: string, permissionGrantIds?: string[]
|
|
2681
4545
|
): Promise<string> {
|
|
2682
4546
|
const si = this.stateIndex;
|
|
2683
4547
|
if (si) {
|
|
@@ -2695,10 +4559,34 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2695
4559
|
messageType : DwnInterface.MessagesSync,
|
|
2696
4560
|
granteeDid : delegateDid,
|
|
2697
4561
|
messageParams : {
|
|
2698
|
-
action: 'subtree',
|
|
4562
|
+
action : 'subtree',
|
|
2699
4563
|
prefix,
|
|
2700
4564
|
protocol,
|
|
2701
|
-
|
|
4565
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
|
|
4566
|
+
}
|
|
4567
|
+
});
|
|
4568
|
+
const reply = response.reply as MessagesSyncReply;
|
|
4569
|
+
return reply.hash ?? '';
|
|
4570
|
+
}
|
|
4571
|
+
|
|
4572
|
+
private async getLocalProjectedSubtreeHash(
|
|
4573
|
+
did: string,
|
|
4574
|
+
prefix: string,
|
|
4575
|
+
delegateDid: string | undefined,
|
|
4576
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
|
|
4577
|
+
permissionGrantIds?: string[],
|
|
4578
|
+
): Promise<string> {
|
|
4579
|
+
const response = await this.agent.dwn.processRequest({
|
|
4580
|
+
author : did,
|
|
4581
|
+
target : did,
|
|
4582
|
+
messageType : DwnInterface.MessagesSync,
|
|
4583
|
+
granteeDid : delegateDid,
|
|
4584
|
+
messageParams : {
|
|
4585
|
+
action : 'subtree',
|
|
4586
|
+
prefix,
|
|
4587
|
+
projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
|
|
4588
|
+
projectionScopes : [...scopes],
|
|
4589
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
|
|
2702
4590
|
}
|
|
2703
4591
|
});
|
|
2704
4592
|
const reply = response.reply as MessagesSyncReply;
|
|
@@ -2712,7 +4600,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2712
4600
|
* In remote mode: constructs a signed MessagesSync message and routes through RPC.
|
|
2713
4601
|
*/
|
|
2714
4602
|
private async getLocalLeaves(
|
|
2715
|
-
did: string, prefix: string, delegateDid?: string, protocol?: string,
|
|
4603
|
+
did: string, prefix: string, delegateDid?: string, protocol?: string, permissionGrantIds?: string[]
|
|
2716
4604
|
): Promise<string[]> {
|
|
2717
4605
|
const si = this.stateIndex;
|
|
2718
4606
|
if (si) {
|
|
@@ -2729,10 +4617,43 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2729
4617
|
messageType : DwnInterface.MessagesSync,
|
|
2730
4618
|
granteeDid : delegateDid,
|
|
2731
4619
|
messageParams : {
|
|
2732
|
-
action: 'leaves',
|
|
4620
|
+
action : 'leaves',
|
|
2733
4621
|
prefix,
|
|
2734
4622
|
protocol,
|
|
2735
|
-
|
|
4623
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
|
|
4624
|
+
}
|
|
4625
|
+
});
|
|
4626
|
+
const reply = response.reply as MessagesSyncReply;
|
|
4627
|
+
return reply.entries ?? [];
|
|
4628
|
+
}
|
|
4629
|
+
|
|
4630
|
+
private async getLocalProjectedLeaves(
|
|
4631
|
+
did: string,
|
|
4632
|
+
prefix: string,
|
|
4633
|
+
delegateDid: string | undefined,
|
|
4634
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
|
|
4635
|
+
permissionGrantIds?: string[],
|
|
4636
|
+
): Promise<string[]> {
|
|
4637
|
+
if (this.stateIndex) {
|
|
4638
|
+
return RecordsProjection.getLeaves({
|
|
4639
|
+
tenant : did,
|
|
4640
|
+
messageStore : this.agent.dwn.node.storage.messageStore,
|
|
4641
|
+
scopes,
|
|
4642
|
+
prefix : SyncEngineLevel.parseBitPrefix(prefix),
|
|
4643
|
+
});
|
|
4644
|
+
}
|
|
4645
|
+
|
|
4646
|
+
const response = await this.agent.dwn.processRequest({
|
|
4647
|
+
author : did,
|
|
4648
|
+
target : did,
|
|
4649
|
+
messageType : DwnInterface.MessagesSync,
|
|
4650
|
+
granteeDid : delegateDid,
|
|
4651
|
+
messageParams : {
|
|
4652
|
+
action : 'leaves',
|
|
4653
|
+
prefix,
|
|
4654
|
+
projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
|
|
4655
|
+
projectionScopes : [...scopes],
|
|
4656
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
|
|
2736
4657
|
}
|
|
2737
4658
|
});
|
|
2738
4659
|
const reply = response.reply as MessagesSyncReply;
|
|
@@ -2751,33 +4672,161 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2751
4672
|
* they are processed directly without additional HTTP round-trips.
|
|
2752
4673
|
* Only `messageCids` that were NOT prefetched are fetched individually.
|
|
2753
4674
|
*/
|
|
2754
|
-
private async pullMessages({
|
|
4675
|
+
private async pullMessages({
|
|
4676
|
+
did,
|
|
4677
|
+
dwnUrl,
|
|
4678
|
+
delegateDid,
|
|
4679
|
+
protocol,
|
|
4680
|
+
scope,
|
|
4681
|
+
permissionGrantIds,
|
|
4682
|
+
messageCids,
|
|
4683
|
+
prefetched,
|
|
4684
|
+
verifiedInitialWrites,
|
|
4685
|
+
shouldContinue,
|
|
4686
|
+
}: {
|
|
2755
4687
|
did: string;
|
|
2756
4688
|
dwnUrl: string;
|
|
2757
4689
|
delegateDid?: string;
|
|
2758
4690
|
protocol?: string;
|
|
4691
|
+
scope?: SyncScope;
|
|
4692
|
+
permissionGrantIds?: string[];
|
|
2759
4693
|
messageCids: string[];
|
|
2760
4694
|
prefetched?: MessagesSyncDiffEntry[];
|
|
4695
|
+
verifiedInitialWrites?: RecordsWriteMessage[];
|
|
4696
|
+
shouldContinue?: () => boolean;
|
|
2761
4697
|
}): Promise<void> {
|
|
4698
|
+
const acceptanceScope: SyncScope = scope ?? (protocol === undefined
|
|
4699
|
+
? { kind: 'full' }
|
|
4700
|
+
: { kind: 'protocolSet', protocols: [protocol] });
|
|
4701
|
+
const rejectedPullEntries = new Map<string, Extract<PullAcceptanceResult, { accepted: false }>>();
|
|
2762
4702
|
const failedCids = await pullMessages({
|
|
2763
|
-
did,
|
|
2764
|
-
|
|
2765
|
-
|
|
4703
|
+
did,
|
|
4704
|
+
dwnUrl,
|
|
4705
|
+
delegateDid,
|
|
4706
|
+
permissionGrantIds,
|
|
4707
|
+
messageCids,
|
|
4708
|
+
prefetched,
|
|
4709
|
+
shouldContinue,
|
|
4710
|
+
agent : this.agent,
|
|
4711
|
+
acceptEntry : async (entry, entries) => {
|
|
4712
|
+
const result = await this.acceptPulledSyncEntry(did, acceptanceScope, entry, entries, verifiedInitialWrites);
|
|
4713
|
+
if (!result.accepted) {
|
|
4714
|
+
rejectedPullEntries.set(await getMessageCid(entry.message), result);
|
|
4715
|
+
}
|
|
4716
|
+
return result.accepted;
|
|
4717
|
+
},
|
|
2766
4718
|
});
|
|
2767
4719
|
|
|
2768
4720
|
// Record permanently failed pull entries in the dead letter store.
|
|
2769
4721
|
for (const cid of failedCids) {
|
|
4722
|
+
const rejection = rejectedPullEntries.get(cid);
|
|
2770
4723
|
await this.recordDeadLetter({
|
|
2771
4724
|
messageCid : cid,
|
|
2772
4725
|
tenantDid : did,
|
|
2773
4726
|
remoteEndpoint : dwnUrl,
|
|
2774
4727
|
protocol,
|
|
2775
|
-
category : 'pull-processing',
|
|
2776
|
-
|
|
4728
|
+
category : rejection ? 'pull-scope-rejected' : 'pull-processing',
|
|
4729
|
+
errorCode : rejection?.classification,
|
|
4730
|
+
errorDetail : rejection
|
|
4731
|
+
? `pulled message rejected by ${rejection.classification} sync scope gate`
|
|
4732
|
+
: 'pull processing failed after retry passes exhausted',
|
|
2777
4733
|
});
|
|
2778
4734
|
}
|
|
2779
4735
|
}
|
|
2780
4736
|
|
|
4737
|
+
private async acceptPulledSyncEntry(
|
|
4738
|
+
did: string,
|
|
4739
|
+
scope: SyncScope,
|
|
4740
|
+
entry: SyncMessageEntry,
|
|
4741
|
+
entries: SyncMessageEntry[],
|
|
4742
|
+
verifiedInitialWrites: RecordsWriteMessage[] = [],
|
|
4743
|
+
): Promise<PullAcceptanceResult> {
|
|
4744
|
+
if (scope.kind === 'full') {
|
|
4745
|
+
return { accepted: true };
|
|
4746
|
+
}
|
|
4747
|
+
|
|
4748
|
+
const initialWrite = await this.resolvePulledDeleteInitialWrite(did, entry.message, entries, verifiedInitialWrites);
|
|
4749
|
+
const classification = classifySyncMessageScope({
|
|
4750
|
+
message: entry.message,
|
|
4751
|
+
initialWrite,
|
|
4752
|
+
scope,
|
|
4753
|
+
});
|
|
4754
|
+
|
|
4755
|
+
if (classification === 'in-scope') {
|
|
4756
|
+
return { accepted: true };
|
|
4757
|
+
}
|
|
4758
|
+
|
|
4759
|
+
const messageCid = await getMessageCid(entry.message);
|
|
4760
|
+
console.warn(`SyncEngineLevel: refusing to apply ${classification} pulled message ${messageCid}`);
|
|
4761
|
+
return { accepted: false, classification };
|
|
4762
|
+
}
|
|
4763
|
+
|
|
4764
|
+
private async resolvePulledDeleteInitialWrite(
|
|
4765
|
+
did: string,
|
|
4766
|
+
message: GenericMessage,
|
|
4767
|
+
entries: SyncMessageEntry[],
|
|
4768
|
+
verifiedInitialWrites: RecordsWriteMessage[] = [],
|
|
4769
|
+
): Promise<RecordsWriteMessage | undefined> {
|
|
4770
|
+
const descriptor = message.descriptor as Record<string, unknown>;
|
|
4771
|
+
if (
|
|
4772
|
+
descriptor.interface !== DwnInterfaceName.Records ||
|
|
4773
|
+
descriptor.method !== DwnMethodName.Delete ||
|
|
4774
|
+
typeof descriptor.recordId !== 'string'
|
|
4775
|
+
) {
|
|
4776
|
+
return undefined;
|
|
4777
|
+
}
|
|
4778
|
+
|
|
4779
|
+
if (!this.agent.dwn.isRemoteMode) {
|
|
4780
|
+
const localInitialWrite = await RecordsWrite.fetchInitialRecordsWriteMessage(
|
|
4781
|
+
this.agent.dwn.node.storage.messageStore,
|
|
4782
|
+
did,
|
|
4783
|
+
descriptor.recordId,
|
|
4784
|
+
);
|
|
4785
|
+
if (localInitialWrite) {
|
|
4786
|
+
return localInitialWrite;
|
|
4787
|
+
}
|
|
4788
|
+
}
|
|
4789
|
+
|
|
4790
|
+
const verifiedInitialWrite = SyncEngineLevel.findInitialWriteByRecordId(descriptor.recordId, verifiedInitialWrites);
|
|
4791
|
+
if (verifiedInitialWrite !== undefined) {
|
|
4792
|
+
return verifiedInitialWrite;
|
|
4793
|
+
}
|
|
4794
|
+
|
|
4795
|
+
// Batch entries are only used when the initial write has not been applied
|
|
4796
|
+
// locally yet. Verified dependency hints cover the projected remote-mode
|
|
4797
|
+
// path where the initial write was applied in the previous pull batch and
|
|
4798
|
+
// no embedded local message store is available. Batch entries are still
|
|
4799
|
+
// parsed as initial RecordsWrite messages, and processRawMessage
|
|
4800
|
+
// authenticates the delete before any local mutation occurs.
|
|
4801
|
+
return this.findInitialWriteInPullBatch(descriptor.recordId, entries);
|
|
4802
|
+
}
|
|
4803
|
+
|
|
4804
|
+
private static findInitialWriteByRecordId(
|
|
4805
|
+
recordId: string,
|
|
4806
|
+
initialWrites: RecordsWriteMessage[],
|
|
4807
|
+
): RecordsWriteMessage | undefined {
|
|
4808
|
+
return initialWrites.find(initialWrite => initialWrite.recordId === recordId);
|
|
4809
|
+
}
|
|
4810
|
+
|
|
4811
|
+
private async findInitialWriteInPullBatch(
|
|
4812
|
+
recordId: string,
|
|
4813
|
+
entries: SyncMessageEntry[],
|
|
4814
|
+
): Promise<RecordsWriteMessage | undefined> {
|
|
4815
|
+
for (const entry of entries) {
|
|
4816
|
+
if (entry.message.descriptor.interface !== DwnInterfaceName.Records ||
|
|
4817
|
+
entry.message.descriptor.method !== DwnMethodName.Write) {
|
|
4818
|
+
continue;
|
|
4819
|
+
}
|
|
4820
|
+
|
|
4821
|
+
const candidate = entry.message as RecordsWriteMessage;
|
|
4822
|
+
if (candidate.recordId === recordId && await RecordsWrite.isInitialWrite(candidate)) {
|
|
4823
|
+
return candidate;
|
|
4824
|
+
}
|
|
4825
|
+
}
|
|
4826
|
+
|
|
4827
|
+
return undefined;
|
|
4828
|
+
}
|
|
4829
|
+
|
|
2781
4830
|
// ---------------------------------------------------------------------------
|
|
2782
4831
|
// Echo-loop suppression
|
|
2783
4832
|
// ---------------------------------------------------------------------------
|
|
@@ -2828,17 +4877,16 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2828
4877
|
* Reads missing messages from the local DWN and pushes them to the remote DWN
|
|
2829
4878
|
* in dependency order (topological sort).
|
|
2830
4879
|
*/
|
|
2831
|
-
private async pushMessages({ did, dwnUrl, delegateDid,
|
|
4880
|
+
private async pushMessages({ did, dwnUrl, delegateDid, permissionGrantIds, messageCids }: {
|
|
2832
4881
|
did: string;
|
|
2833
4882
|
dwnUrl: string;
|
|
2834
4883
|
delegateDid?: string;
|
|
2835
|
-
|
|
4884
|
+
permissionGrantIds?: string[];
|
|
2836
4885
|
messageCids: string[];
|
|
2837
4886
|
}): Promise<PushResult> {
|
|
2838
4887
|
return pushMessages({
|
|
2839
|
-
did, dwnUrl, delegateDid,
|
|
2840
|
-
agent
|
|
2841
|
-
permissionsApi : this._permissionsApi,
|
|
4888
|
+
did, dwnUrl, delegateDid, permissionGrantIds, messageCids,
|
|
4889
|
+
agent: this.agent,
|
|
2842
4890
|
});
|
|
2843
4891
|
}
|
|
2844
4892
|
|
|
@@ -2867,14 +4915,20 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2867
4915
|
* When `protocol` is undefined (full-tenant link), clears entries that
|
|
2868
4916
|
* also have no protocol.
|
|
2869
4917
|
*/
|
|
2870
|
-
private async clearDeadLettersForLink(
|
|
4918
|
+
private async clearDeadLettersForLink(
|
|
4919
|
+
tenantDid: string,
|
|
4920
|
+
remoteEndpoint: string,
|
|
4921
|
+
protocol?: string,
|
|
4922
|
+
options: { categories?: ReadonlySet<DeadLetterCategory> } = {},
|
|
4923
|
+
): Promise<void> {
|
|
2871
4924
|
const batch: { type: 'del'; key: string }[] = [];
|
|
2872
4925
|
try {
|
|
2873
4926
|
for await (const [key, value] of this._deadLetters.iterator()) {
|
|
2874
4927
|
const entry = JSON.parse(value) as DeadLetterEntry;
|
|
2875
4928
|
if (entry.tenantDid === tenantDid &&
|
|
2876
4929
|
entry.remoteEndpoint === remoteEndpoint &&
|
|
2877
|
-
entry.protocol === protocol
|
|
4930
|
+
entry.protocol === protocol &&
|
|
4931
|
+
(options.categories === undefined || options.categories.has(entry.category))) {
|
|
2878
4932
|
batch.push({ type: 'del', key });
|
|
2879
4933
|
}
|
|
2880
4934
|
}
|
|
@@ -2887,6 +4941,20 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2887
4941
|
}
|
|
2888
4942
|
}
|
|
2889
4943
|
|
|
4944
|
+
private async clearRootConvergenceDeadLetters(
|
|
4945
|
+
tenantDid: string,
|
|
4946
|
+
remoteEndpoint: string,
|
|
4947
|
+
protocol?: string,
|
|
4948
|
+
): Promise<void> {
|
|
4949
|
+
try {
|
|
4950
|
+
await this.clearDeadLettersForLink(tenantDid, remoteEndpoint, protocol, {
|
|
4951
|
+
categories: SyncEngineLevel.ROOT_CONVERGENCE_CLEARABLE_DEAD_LETTER_CATEGORIES,
|
|
4952
|
+
});
|
|
4953
|
+
} catch (error) {
|
|
4954
|
+
console.warn(`SyncEngineLevel: Failed to clear root-convergence dead letters for ${tenantDid} -> ${remoteEndpoint}`, error);
|
|
4955
|
+
}
|
|
4956
|
+
}
|
|
4957
|
+
|
|
2890
4958
|
/**
|
|
2891
4959
|
* Build a compound dead letter key. Different remotes can fail the same CID
|
|
2892
4960
|
* for different reasons, so the key includes the remote endpoint.
|
|
@@ -2929,7 +4997,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2929
4997
|
}
|
|
2930
4998
|
}
|
|
2931
4999
|
// Deterministic ordering: newest first so apps see the most recent failures.
|
|
2932
|
-
entries.sort((a, b) => b.failedAt
|
|
5000
|
+
entries.sort((a, b) => lexicographicalCompare(b.failedAt, a.failedAt));
|
|
2933
5001
|
return entries;
|
|
2934
5002
|
}
|
|
2935
5003
|
|
|
@@ -2984,44 +5052,87 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2984
5052
|
|
|
2985
5053
|
public async getSyncHealth(): Promise<SyncHealthSummary> {
|
|
2986
5054
|
let failedMessageCount = 0;
|
|
2987
|
-
|
|
5055
|
+
let closureFailureCount = 0;
|
|
5056
|
+
for await (const [, value] of this._deadLetters.iterator()) {
|
|
2988
5057
|
failedMessageCount++;
|
|
5058
|
+
const entry = JSON.parse(value) as DeadLetterEntry;
|
|
5059
|
+
if (entry.category === 'closure') {
|
|
5060
|
+
closureFailureCount++;
|
|
5061
|
+
}
|
|
2989
5062
|
}
|
|
2990
5063
|
|
|
2991
|
-
//
|
|
2992
|
-
//
|
|
2993
|
-
//
|
|
5064
|
+
// Superseded authorization epochs can leave durable link state behind. Only
|
|
5065
|
+
// links that still belong to the current registered projection/epoch should
|
|
5066
|
+
// affect health. Endpoint-level orphan cleanup is a separate GC concern.
|
|
5067
|
+
const currentLinkIdentityKeys = await this.getCurrentDurableLinkIdentityKeys();
|
|
2994
5068
|
let degradedLinkCount = 0;
|
|
2995
5069
|
const allLinks = await this.ledger.getAllLinks();
|
|
2996
5070
|
for (const link of allLinks) {
|
|
2997
|
-
|
|
5071
|
+
const isCurrentLink = currentLinkIdentityKeys === undefined || currentLinkIdentityKeys.has(this.getDurableLinkIdentityKey(link));
|
|
5072
|
+
if (isCurrentLink && SyncEngineLevel.isUnhealthyLinkStatus(link.status)) {
|
|
2998
5073
|
degradedLinkCount++;
|
|
2999
5074
|
}
|
|
3000
5075
|
}
|
|
3001
5076
|
|
|
3002
5077
|
return {
|
|
3003
|
-
connectivity: this.connectivityState,
|
|
3004
|
-
failedMessageCount,
|
|
3005
|
-
|
|
5078
|
+
connectivity : this.connectivityState,
|
|
5079
|
+
failedMessageCount : failedMessageCount,
|
|
5080
|
+
closureFailureCount : closureFailureCount,
|
|
5081
|
+
degradedLinkCount : degradedLinkCount,
|
|
5082
|
+
syncHealthy : failedMessageCount === 0 && degradedLinkCount === 0,
|
|
3006
5083
|
};
|
|
3007
5084
|
}
|
|
3008
5085
|
|
|
5086
|
+
private async getCurrentDurableLinkIdentityKeys(): Promise<Set<string> | undefined> {
|
|
5087
|
+
try {
|
|
5088
|
+
const identityKeys = new Set<string>();
|
|
5089
|
+
for await (const [did, options] of this._db.sublevel('registeredIdentities').iterator()) {
|
|
5090
|
+
let parsed: SyncIdentityOptions;
|
|
5091
|
+
try {
|
|
5092
|
+
parsed = JSON.parse(options) as SyncIdentityOptions;
|
|
5093
|
+
} catch (error: unknown) {
|
|
5094
|
+
console.warn(`SyncEngineLevel: Corrupt sync options for ${did}, skipping health target:`, error);
|
|
5095
|
+
continue;
|
|
5096
|
+
}
|
|
5097
|
+
|
|
5098
|
+
const scope = syncScopeFromProtocols(parsed.protocols);
|
|
5099
|
+
const resolutions = await this.buildSyncTargetResolutions(did, scope, parsed);
|
|
5100
|
+
for (const resolution of resolutions) {
|
|
5101
|
+
const projectionId = await computeProjectionId(did, resolution.scope);
|
|
5102
|
+
identityKeys.add(SyncEngineLevel.durableLinkIdentityKey(did, projectionId, resolution.authorizationEpoch));
|
|
5103
|
+
}
|
|
5104
|
+
}
|
|
5105
|
+
return identityKeys;
|
|
5106
|
+
} catch (error: unknown) {
|
|
5107
|
+
console.warn('SyncEngineLevel: Failed to resolve current durable link identity keys for health; falling back to all durable links', error);
|
|
5108
|
+
return undefined;
|
|
5109
|
+
}
|
|
5110
|
+
}
|
|
5111
|
+
|
|
5112
|
+
private getDurableLinkIdentityKey(link: ReplicationLinkState): string {
|
|
5113
|
+
return SyncEngineLevel.durableLinkIdentityKey(link.tenantDid, link.projectionId, link.authorizationEpoch);
|
|
5114
|
+
}
|
|
5115
|
+
|
|
5116
|
+
private static durableLinkIdentityKey(tenantDid: string, projectionId: string, authorizationEpoch: string): string {
|
|
5117
|
+
return `${tenantDid}^${projectionId}^${authorizationEpoch}`;
|
|
5118
|
+
}
|
|
5119
|
+
|
|
5120
|
+
private static isUnhealthyLinkStatus(status: ReplicationLinkState['status']): boolean {
|
|
5121
|
+
return status === 'repairing' || status === 'degraded_poll' || status === 'terminal_incomplete';
|
|
5122
|
+
}
|
|
5123
|
+
|
|
3009
5124
|
// ---------------------------------------------------------------------------
|
|
3010
5125
|
// Sync targets
|
|
3011
5126
|
// ---------------------------------------------------------------------------
|
|
3012
5127
|
|
|
3013
5128
|
/**
|
|
3014
|
-
* Returns the list of sync targets:
|
|
5129
|
+
* Returns the list of sync targets: one canonical projection target per
|
|
5130
|
+
* registered DID and resolved DWN endpoint.
|
|
3015
5131
|
* Results are cached for up to 30 seconds to avoid redundant DID resolution
|
|
3016
5132
|
* on every sync tick. The cache is invalidated when identities are registered,
|
|
3017
5133
|
* unregistered, or updated.
|
|
3018
5134
|
*/
|
|
3019
|
-
private async getSyncTargets(): Promise<{
|
|
3020
|
-
did: string;
|
|
3021
|
-
dwnUrl: string;
|
|
3022
|
-
delegateDid?: string;
|
|
3023
|
-
protocol?: string;
|
|
3024
|
-
}[]> {
|
|
5135
|
+
private async getSyncTargets(): Promise<SyncTarget[]> {
|
|
3025
5136
|
// Return cached targets if still valid.
|
|
3026
5137
|
if (this._syncTargetsCache
|
|
3027
5138
|
&& (Date.now() - this._syncTargetsCache.timestamp) < SyncEngineLevel.SYNC_TARGETS_CACHE_TTL_MS) {
|
|
@@ -3033,7 +5144,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
3033
5144
|
// make our result stale.
|
|
3034
5145
|
const generationAtStart = this._syncTargetsCacheGeneration;
|
|
3035
5146
|
|
|
3036
|
-
const targets:
|
|
5147
|
+
const targets: SyncTarget[] = [];
|
|
3037
5148
|
let hasRegisteredIdentities = false;
|
|
3038
5149
|
let anyEndpointMissing = false;
|
|
3039
5150
|
|
|
@@ -3047,8 +5158,6 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
3047
5158
|
continue;
|
|
3048
5159
|
}
|
|
3049
5160
|
|
|
3050
|
-
const { protocols, delegateDid } = parsed;
|
|
3051
|
-
|
|
3052
5161
|
const dwnEndpointUrls = await this.agent.dwn.getDwnEndpointUrlsForTarget(did);
|
|
3053
5162
|
if (dwnEndpointUrls.length === 0) {
|
|
3054
5163
|
anyEndpointMissing = true;
|
|
@@ -3056,14 +5165,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
3056
5165
|
}
|
|
3057
5166
|
|
|
3058
5167
|
for (const dwnUrl of dwnEndpointUrls) {
|
|
3059
|
-
|
|
3060
|
-
// Sync all protocols (global tree).
|
|
3061
|
-
targets.push({ did, delegateDid, dwnUrl });
|
|
3062
|
-
} else {
|
|
3063
|
-
for (const protocol of protocols) {
|
|
3064
|
-
targets.push({ did, delegateDid, dwnUrl, protocol });
|
|
3065
|
-
}
|
|
3066
|
-
}
|
|
5168
|
+
targets.push(...await this.buildSyncTargetsForEndpoint(did, dwnUrl, parsed));
|
|
3067
5169
|
}
|
|
3068
5170
|
}
|
|
3069
5171
|
|
|
@@ -3081,22 +5183,4 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
3081
5183
|
return targets;
|
|
3082
5184
|
}
|
|
3083
5185
|
|
|
3084
|
-
/**
|
|
3085
|
-
* Gets the permission grant ID for MessagesSync if a delegateDid is provided.
|
|
3086
|
-
* Returns undefined if no delegate is in use (owner access).
|
|
3087
|
-
*/
|
|
3088
|
-
private async getSyncPermissionGrantId(did: string, delegateDid?: string, protocol?: string): Promise<string | undefined> {
|
|
3089
|
-
if (!delegateDid) {
|
|
3090
|
-
return undefined;
|
|
3091
|
-
}
|
|
3092
|
-
|
|
3093
|
-
const messagesSyncGrant = await this._permissionsApi.getPermissionForRequest({
|
|
3094
|
-
connectedDid : did,
|
|
3095
|
-
messageType : DwnInterface.MessagesSync,
|
|
3096
|
-
delegateDid,
|
|
3097
|
-
protocol,
|
|
3098
|
-
cached : true
|
|
3099
|
-
});
|
|
3100
|
-
return messagesSyncGrant.grant.id;
|
|
3101
|
-
}
|
|
3102
5186
|
}
|