@enbox/agent 0.7.6 → 0.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.mjs +9 -11
- package/dist/browser.mjs.map +4 -4
- package/dist/esm/dwn-api.js +3 -2
- package/dist/esm/dwn-api.js.map +1 -1
- package/dist/esm/enbox-connect-protocol.js +5 -5
- package/dist/esm/enbox-connect-protocol.js.map +1 -1
- package/dist/esm/hd-identity-vault.js +187 -177
- package/dist/esm/hd-identity-vault.js.map +1 -1
- package/dist/esm/index.js +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/permissions-api.js +7 -34
- package/dist/esm/permissions-api.js.map +1 -1
- package/dist/esm/sync-closure-resolver.js +229 -110
- package/dist/esm/sync-closure-resolver.js.map +1 -1
- package/dist/esm/sync-closure-types.js +24 -7
- package/dist/esm/sync-closure-types.js.map +1 -1
- package/dist/esm/sync-engine-level.js +1961 -764
- package/dist/esm/sync-engine-level.js.map +1 -1
- package/dist/esm/sync-link-id.js +4 -13
- package/dist/esm/sync-link-id.js.map +1 -1
- package/dist/esm/sync-link-reconciler.js +26 -8
- package/dist/esm/sync-link-reconciler.js.map +1 -1
- package/dist/esm/sync-messages.js +218 -154
- package/dist/esm/sync-messages.js.map +1 -1
- package/dist/esm/sync-permission-grants.js +208 -0
- package/dist/esm/sync-permission-grants.js.map +1 -0
- package/dist/esm/sync-replication-ledger.js +23 -40
- package/dist/esm/sync-replication-ledger.js.map +1 -1
- package/dist/esm/sync-scope-acceptance.js +126 -0
- package/dist/esm/sync-scope-acceptance.js.map +1 -0
- package/dist/esm/sync-topological-sort.js +57 -15
- package/dist/esm/sync-topological-sort.js.map +1 -1
- package/dist/esm/types/sync.js +130 -22
- package/dist/esm/types/sync.js.map +1 -1
- package/dist/types/dwn-api.d.ts.map +1 -1
- package/dist/types/hd-identity-vault.d.ts +25 -0
- package/dist/types/hd-identity-vault.d.ts.map +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/permissions-api.d.ts +1 -2
- package/dist/types/permissions-api.d.ts.map +1 -1
- package/dist/types/sync-closure-resolver.d.ts.map +1 -1
- package/dist/types/sync-closure-types.d.ts +14 -3
- package/dist/types/sync-closure-types.d.ts.map +1 -1
- package/dist/types/sync-engine-level.d.ts +127 -25
- package/dist/types/sync-engine-level.d.ts.map +1 -1
- package/dist/types/sync-link-id.d.ts +3 -9
- package/dist/types/sync-link-id.d.ts.map +1 -1
- package/dist/types/sync-link-reconciler.d.ts +12 -2
- package/dist/types/sync-link-reconciler.d.ts.map +1 -1
- package/dist/types/sync-messages.d.ts +16 -13
- package/dist/types/sync-messages.d.ts.map +1 -1
- package/dist/types/sync-permission-grants.d.ts +52 -0
- package/dist/types/sync-permission-grants.d.ts.map +1 -0
- package/dist/types/sync-replication-ledger.d.ts +5 -13
- package/dist/types/sync-replication-ledger.d.ts.map +1 -1
- package/dist/types/sync-scope-acceptance.d.ts +28 -0
- package/dist/types/sync-scope-acceptance.d.ts.map +1 -0
- package/dist/types/sync-topological-sort.d.ts +2 -1
- package/dist/types/sync-topological-sort.d.ts.map +1 -1
- package/dist/types/types/identity-vault.d.ts +9 -0
- package/dist/types/types/identity-vault.d.ts.map +1 -1
- package/dist/types/types/permissions.d.ts +2 -0
- package/dist/types/types/permissions.d.ts.map +1 -1
- package/dist/types/types/sync.d.ts +137 -75
- package/dist/types/types/sync.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/dwn-api.ts +3 -2
- package/src/enbox-connect-protocol.ts +5 -5
- package/src/hd-identity-vault.ts +244 -212
- package/src/index.ts +10 -1
- package/src/permissions-api.ts +11 -42
- package/src/sync-closure-resolver.ts +306 -126
- package/src/sync-closure-types.ts +38 -9
- package/src/sync-engine-level.ts +2560 -797
- package/src/sync-link-id.ts +9 -14
- package/src/sync-link-reconciler.ts +43 -10
- package/src/sync-messages.ts +263 -159
- package/src/sync-permission-grants.ts +297 -0
- package/src/sync-replication-ledger.ts +55 -50
- package/src/sync-scope-acceptance.ts +186 -0
- package/src/sync-topological-sort.ts +89 -21
- package/src/types/identity-vault.ts +8 -1
- package/src/types/permissions.ts +2 -0
- package/src/types/sync.ts +235 -62
package/src/sync-engine-level.ts
CHANGED
|
@@ -1,31 +1,36 @@
|
|
|
1
1
|
import type { AbstractLevel } from 'abstract-level';
|
|
2
2
|
|
|
3
3
|
import type { DwnSubscriptionHandler, ResubscribeFactory } from '@enbox/dwn-clients';
|
|
4
|
-
import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncDiffEntry, MessagesSyncReply, ProgressToken, StateIndex, SubscriptionMessage } from '@enbox/dwn-sdk-js';
|
|
4
|
+
import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncDependencyEntry, MessagesSyncDiffEntry, MessagesSyncReply, ProgressToken, ProtocolsConfigureMessage, RecordsProjectionScope, RecordsWriteMessage, StateIndex, SubscriptionMessage } from '@enbox/dwn-sdk-js';
|
|
5
5
|
|
|
6
6
|
import ms from 'ms';
|
|
7
7
|
|
|
8
8
|
import { Level } from 'level';
|
|
9
9
|
import { sleep } from '@enbox/common';
|
|
10
|
-
import { Encoder, hashToHex, initDefaultHashes, Message } from '@enbox/dwn-sdk-js';
|
|
10
|
+
import { authenticate, DwnInterfaceName, DwnMethodName, Encoder, hashToHex, initDefaultHashes, Message, ProtocolsConfigure, RECORDS_PROJECTION_ROOT_VERSION, RecordsProjection, RecordsWrite } from '@enbox/dwn-sdk-js';
|
|
11
11
|
|
|
12
12
|
import type { ClosureEvaluationContext } from './sync-closure-types.js';
|
|
13
13
|
import type { EnboxPlatformAgent } from './types/agent.js';
|
|
14
14
|
import type { PermissionsApi } from './types/permissions.js';
|
|
15
|
-
import type {
|
|
16
|
-
|
|
17
|
-
import {
|
|
18
|
-
import { MAX_PENDING_TOKENS } from './types/sync.js';
|
|
19
|
-
import { ReplicationLedger } from './sync-replication-ledger.js';
|
|
20
|
-
import { createClosureContext, invalidateClosureCache } from './sync-closure-types.js';
|
|
15
|
+
import type { SyncMessageEntry } from './sync-messages.js';
|
|
16
|
+
import type { SyncScopeClassification } from './sync-scope-acceptance.js';
|
|
17
|
+
import type { DeadLetterCategory, DeadLetterEntry, NonEmptyStringArray, PushResult, ReplicationLinkState, StartSyncParams, SyncAuthorization, SyncConnectivityState, SyncEngine, SyncEvent, SyncEventListener, SyncEventScope, SyncHealthSummary, SyncIdentityOptions, SyncMode, SyncScope } from './types/sync.js';
|
|
21
18
|
|
|
22
19
|
import { AgentPermissionsApi } from './permissions-api.js';
|
|
20
|
+
import type { DwnMessageParams } from './types/dwn.js';
|
|
21
|
+
|
|
22
|
+
import { buildLinkId } from './sync-link-id.js';
|
|
23
23
|
import { DwnInterface } from './types/dwn.js';
|
|
24
|
+
import { evaluateClosure } from './sync-closure-resolver.js';
|
|
24
25
|
import { isRecordsWrite } from './utils.js';
|
|
25
|
-
import {
|
|
26
|
+
import { ReplicationLedger } from './sync-replication-ledger.js';
|
|
26
27
|
import { topologicalSort } from './sync-topological-sort.js';
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
28
|
+
import { classifySyncEventScope, classifySyncMessageScope } from './sync-scope-acceptance.js';
|
|
29
|
+
import { computeAuthorizationEpoch, computeProjectionId, lexicographicalCompare, MAX_PENDING_TOKENS, protocolsForSyncScope, singleProtocolForSyncScope, syncScopeFromProtocols } from './types/sync.js';
|
|
30
|
+
import { createClosureContext, invalidateClosureCache, isTerminalClosureFailureCode } from './sync-closure-types.js';
|
|
31
|
+
import { fetchRemoteMessages, getMessageCid, pullMessages, pushMessages, SyncPullAbortedError } from './sync-messages.js';
|
|
32
|
+
import { partitionRemoteEntries, SyncLinkReconciler } from './sync-link-reconciler.js';
|
|
33
|
+
import { permissionGrantIdsFromEntries, resolveMessagesSyncScopes, toMessagesPermissionGrantIds, toSyncAuthorizationGrants } from './sync-permission-grants.js';
|
|
29
34
|
|
|
30
35
|
export type SyncEngineLevelParams = {
|
|
31
36
|
agent?: EnboxPlatformAgent;
|
|
@@ -34,7 +39,7 @@ export type SyncEngineLevelParams = {
|
|
|
34
39
|
};
|
|
35
40
|
|
|
36
41
|
/**
|
|
37
|
-
* Maximum bit prefix depth for the per-node tree walk
|
|
42
|
+
* Maximum bit prefix depth for the per-node tree walk fallback.
|
|
38
43
|
* At depth 16, each subtree covers ~1/65536 of the key space.
|
|
39
44
|
*/
|
|
40
45
|
const MAX_DIFF_DEPTH = 16;
|
|
@@ -62,7 +67,6 @@ type LiveSubscription = {
|
|
|
62
67
|
did: string;
|
|
63
68
|
dwnUrl: string;
|
|
64
69
|
delegateDid?: string;
|
|
65
|
-
protocol?: string;
|
|
66
70
|
close: () => Promise<void>;
|
|
67
71
|
};
|
|
68
72
|
|
|
@@ -72,10 +76,103 @@ type LocalSubscription = {
|
|
|
72
76
|
did: string;
|
|
73
77
|
dwnUrl: string;
|
|
74
78
|
delegateDid?: string;
|
|
75
|
-
protocol?: string;
|
|
76
79
|
close: () => Promise<void>;
|
|
77
80
|
};
|
|
78
81
|
|
|
82
|
+
type SyncTarget = {
|
|
83
|
+
did: string;
|
|
84
|
+
dwnUrl: string;
|
|
85
|
+
delegateDid?: string;
|
|
86
|
+
scope: SyncScope;
|
|
87
|
+
authorization: SyncAuthorization;
|
|
88
|
+
authorizationEpoch: string;
|
|
89
|
+
permissionGrantIds?: NonEmptyStringArray;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
type SyncTargetResolution = Pick<SyncTarget, 'authorization' | 'authorizationEpoch' | 'delegateDid' | 'permissionGrantIds' | 'scope'>;
|
|
93
|
+
|
|
94
|
+
type LinkSyncTarget = SyncTarget & { linkKey: string };
|
|
95
|
+
|
|
96
|
+
enum LinkSubscriptionOpenResult {
|
|
97
|
+
ReadyForLive = 'readyForLive',
|
|
98
|
+
Polling = 'polling',
|
|
99
|
+
Repairing = 'repairing',
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
enum LinkInitializationStatus {
|
|
103
|
+
Active = 'active',
|
|
104
|
+
Failed = 'failed',
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
type LinkInitializationResult =
|
|
108
|
+
| { status: LinkInitializationStatus.Active; durableLinkIdentityKey: string }
|
|
109
|
+
| { status: LinkInitializationStatus.Failed };
|
|
110
|
+
|
|
111
|
+
type PullAcceptanceResult =
|
|
112
|
+
| { accepted: true }
|
|
113
|
+
| { accepted: false; classification: Exclude<SyncScopeClassification, 'in-scope'> };
|
|
114
|
+
|
|
115
|
+
type RecordsWriteProtocolDescriptor = GenericMessage['descriptor'] & { protocol?: unknown };
|
|
116
|
+
type RecordsWriteProtocolMessage = GenericMessage & { descriptor: RecordsWriteProtocolDescriptor };
|
|
117
|
+
type ProtocolsConfigureDefinition = {
|
|
118
|
+
protocol: string;
|
|
119
|
+
uses?: Record<string, unknown>;
|
|
120
|
+
};
|
|
121
|
+
type ProtocolsConfigureDefinitionDescriptor = GenericMessage['descriptor'] & {
|
|
122
|
+
definition: ProtocolsConfigureDefinition;
|
|
123
|
+
};
|
|
124
|
+
type MaybeProtocolsConfigureDefinitionDescriptor = GenericMessage['descriptor'] & {
|
|
125
|
+
definition?: {
|
|
126
|
+
protocol?: string;
|
|
127
|
+
uses?: Record<string, unknown>;
|
|
128
|
+
};
|
|
129
|
+
};
|
|
130
|
+
type SyncDiffEntryWithMessage = MessagesSyncDiffEntry & { message: GenericMessage };
|
|
131
|
+
type SyncDependencyEntryWithMessage = MessagesSyncDependencyEntry & { message: GenericMessage };
|
|
132
|
+
type VerifiedProtocolConfigCandidate = {
|
|
133
|
+
rootMessageCid: string;
|
|
134
|
+
dependency: AuthenticatedProtocolConfigDependency;
|
|
135
|
+
};
|
|
136
|
+
type AuthenticatedProtocolConfigDependency = SyncDependencyEntryWithMessage & { message: ProtocolsConfigureMessage };
|
|
137
|
+
type VerifiedRecordsInitialWriteCandidate = {
|
|
138
|
+
rootMessageCid: string;
|
|
139
|
+
dependency: AuthenticatedRecordsInitialWriteDependency;
|
|
140
|
+
};
|
|
141
|
+
type AuthenticatedRecordsInitialWriteDependency = SyncDependencyEntryWithMessage & { message: RecordsWriteMessage };
|
|
142
|
+
|
|
143
|
+
type ProjectionReconcileTarget = {
|
|
144
|
+
did: string;
|
|
145
|
+
dwnUrl: string;
|
|
146
|
+
delegateDid?: string;
|
|
147
|
+
scope: SyncScope;
|
|
148
|
+
authorization: SyncAuthorization;
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
type ProjectionReconcileOptions = {
|
|
152
|
+
direction?: 'push' | 'pull';
|
|
153
|
+
verifyConvergence?: boolean;
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
type ProjectionReconcileResult = {
|
|
157
|
+
aborted?: boolean;
|
|
158
|
+
converged?: boolean;
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
type ProjectionDiffResult = {
|
|
162
|
+
dependencies?: MessagesSyncDependencyEntry[];
|
|
163
|
+
onlyRemote: MessagesSyncDiffEntry[];
|
|
164
|
+
onlyLocal: string[];
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
type ProtocolSetScope = Extract<SyncScope, { kind: 'protocolSet' }>;
|
|
168
|
+
type RecordsProjectionSyncScope = Extract<SyncScope, { kind: 'recordsProjection' }>;
|
|
169
|
+
|
|
170
|
+
type ProtocolSetDiffPlan = {
|
|
171
|
+
changedProtocols: string[];
|
|
172
|
+
onlyRemote: MessagesSyncDiffEntry[];
|
|
173
|
+
onlyLocal: string[];
|
|
174
|
+
};
|
|
175
|
+
|
|
79
176
|
// ---------------------------------------------------------------------------
|
|
80
177
|
// Per-link in-memory delivery-order tracking (not persisted to ledger)
|
|
81
178
|
// ---------------------------------------------------------------------------
|
|
@@ -95,48 +192,6 @@ type InFlightCommit = {
|
|
|
95
192
|
committed: boolean;
|
|
96
193
|
};
|
|
97
194
|
|
|
98
|
-
/**
|
|
99
|
-
* Checks whether a message's protocolPath and contextId match the link's
|
|
100
|
-
* subset scope prefixes. Returns true if the message is in scope.
|
|
101
|
-
*
|
|
102
|
-
* When the scope has no prefixes (or is kind:'full'), all messages match.
|
|
103
|
-
* When protocolPathPrefixes or contextIdPrefixes are specified, the message
|
|
104
|
-
* must match at least one prefix in each specified set.
|
|
105
|
-
*
|
|
106
|
-
* This is agent-side filtering for subset scopes. The underlying
|
|
107
|
-
* MessagesSubscribe filter only supports protocol-level scoping today —
|
|
108
|
-
* protocolPath/contextId prefix filtering at the EventLog level is a
|
|
109
|
-
* follow-up (requires dwn-sdk-js MessagesFilter extension).
|
|
110
|
-
*/
|
|
111
|
-
function isEventInScope(message: GenericMessage, scope: SyncScope): boolean {
|
|
112
|
-
if (scope.kind === 'full') { return true; }
|
|
113
|
-
if (!scope.protocolPathPrefixes && !scope.contextIdPrefixes) { return true; }
|
|
114
|
-
|
|
115
|
-
const desc = message.descriptor as Record<string, unknown>;
|
|
116
|
-
|
|
117
|
-
// Check protocolPath prefix.
|
|
118
|
-
if (scope.protocolPathPrefixes && scope.protocolPathPrefixes.length > 0) {
|
|
119
|
-
const protocolPath = desc.protocolPath as string | undefined;
|
|
120
|
-
if (!protocolPath) { return false; }
|
|
121
|
-
const matches = scope.protocolPathPrefixes.some(
|
|
122
|
-
prefix => protocolPath === prefix || protocolPath.startsWith(prefix + '/')
|
|
123
|
-
);
|
|
124
|
-
if (!matches) { return false; }
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
// Check contextId prefix.
|
|
128
|
-
if (scope.contextIdPrefixes && scope.contextIdPrefixes.length > 0) {
|
|
129
|
-
const contextId = (message as any).contextId as string | undefined;
|
|
130
|
-
if (!contextId) { return false; }
|
|
131
|
-
const matches = scope.contextIdPrefixes.some(
|
|
132
|
-
prefix => contextId === prefix || contextId.startsWith(prefix + '/')
|
|
133
|
-
);
|
|
134
|
-
if (!matches) { return false; }
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
return true;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
195
|
/**
|
|
141
196
|
* Per-link runtime state held in memory. Not persisted — on crash,
|
|
142
197
|
* replay restarts from `contiguousAppliedToken` (idempotent apply).
|
|
@@ -150,18 +205,59 @@ type LinkRuntimeState = {
|
|
|
150
205
|
inflight: Map<number, InFlightCommit>;
|
|
151
206
|
};
|
|
152
207
|
|
|
208
|
+
type PushRuntimeEntry = { cid: string };
|
|
209
|
+
|
|
153
210
|
type PushRuntimeState = {
|
|
154
211
|
did: string;
|
|
155
212
|
dwnUrl: string;
|
|
156
213
|
delegateDid?: string;
|
|
157
214
|
protocol?: string;
|
|
158
|
-
|
|
215
|
+
permissionGrantIds?: NonEmptyStringArray;
|
|
216
|
+
entries: PushRuntimeEntry[];
|
|
159
217
|
retryCount: number;
|
|
160
218
|
timer?: ReturnType<typeof setTimeout>;
|
|
161
219
|
/** True while a push HTTP request is in flight for this link. */
|
|
162
220
|
flushing?: boolean;
|
|
163
221
|
};
|
|
164
222
|
|
|
223
|
+
type PushFlushBatch = {
|
|
224
|
+
pushRuntime: PushRuntimeState;
|
|
225
|
+
pushEntries: PushRuntimeEntry[];
|
|
226
|
+
isStale: () => boolean;
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
type LivePullContext = {
|
|
230
|
+
did: string;
|
|
231
|
+
dwnUrl: string;
|
|
232
|
+
delegateDid?: string;
|
|
233
|
+
eventScope: SyncEventScope;
|
|
234
|
+
linkKey: string;
|
|
235
|
+
link?: ReplicationLinkState;
|
|
236
|
+
permissionGrantIds?: NonEmptyStringArray;
|
|
237
|
+
isStale: () => boolean;
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
type PullDelivery = {
|
|
241
|
+
runtime?: LinkRuntimeState;
|
|
242
|
+
ordinal: number;
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
function syncEventScope(scope: SyncScope | undefined): SyncEventScope {
|
|
246
|
+
if (scope === undefined) {
|
|
247
|
+
return {};
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const coveredProtocols = protocolsForSyncScope(scope);
|
|
251
|
+
if (coveredProtocols === undefined) {
|
|
252
|
+
return {};
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
const protocols = [...coveredProtocols] as NonEmptyStringArray;
|
|
256
|
+
return protocols.length === 1
|
|
257
|
+
? { protocol: protocols[0], protocols }
|
|
258
|
+
: { protocols };
|
|
259
|
+
}
|
|
260
|
+
|
|
165
261
|
export class SyncEngineLevel implements SyncEngine {
|
|
166
262
|
/**
|
|
167
263
|
* Holds the instance of a `EnboxPlatformAgent` that represents the current execution context for
|
|
@@ -273,13 +369,67 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
273
369
|
}
|
|
274
370
|
}
|
|
275
371
|
|
|
372
|
+
private async buildSyncTargetsForEndpoint(did: string, dwnUrl: string, options: SyncIdentityOptions): Promise<SyncTarget[]> {
|
|
373
|
+
const requestedScope = syncScopeFromProtocols(options.protocols);
|
|
374
|
+
const resolutions = await this.buildSyncTargetResolutions(did, requestedScope, options);
|
|
375
|
+
|
|
376
|
+
return resolutions.map(resolution => ({
|
|
377
|
+
did,
|
|
378
|
+
dwnUrl,
|
|
379
|
+
...resolution,
|
|
380
|
+
}));
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
private async buildSyncTargetResolutions(did: string, requestedScope: SyncScope, options: SyncIdentityOptions): Promise<SyncTargetResolution[]> {
|
|
384
|
+
const { delegateDid } = options;
|
|
385
|
+
|
|
386
|
+
if (delegateDid === undefined) {
|
|
387
|
+
return [{
|
|
388
|
+
scope : requestedScope,
|
|
389
|
+
authorization : { kind: 'owner' },
|
|
390
|
+
authorizationEpoch : await computeAuthorizationEpoch({ kind: 'owner' }),
|
|
391
|
+
}];
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
const resolvedScopes = await resolveMessagesSyncScopes({
|
|
395
|
+
did,
|
|
396
|
+
delegateDid,
|
|
397
|
+
requestedScope,
|
|
398
|
+
messageType : DwnInterface.MessagesSync,
|
|
399
|
+
permissionsApi : this._permissionsApi,
|
|
400
|
+
});
|
|
401
|
+
|
|
402
|
+
return Promise.all(resolvedScopes.map(async ({ scope, permissionGrants }) => {
|
|
403
|
+
const permissionGrantIds = permissionGrantIdsFromEntries(permissionGrants);
|
|
404
|
+
if (permissionGrantIds === undefined) {
|
|
405
|
+
throw new Error(`SyncEngineLevel: delegate ${delegateDid} has no active sync grants for ${did}.`);
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
return {
|
|
409
|
+
scope,
|
|
410
|
+
delegateDid,
|
|
411
|
+
authorization: {
|
|
412
|
+
kind: 'delegate' as const,
|
|
413
|
+
delegateDid,
|
|
414
|
+
permissionGrantIds,
|
|
415
|
+
},
|
|
416
|
+
authorizationEpoch: await computeAuthorizationEpoch({
|
|
417
|
+
kind : 'delegate' as const,
|
|
418
|
+
delegateDid,
|
|
419
|
+
grants : toSyncAuthorizationGrants(permissionGrants),
|
|
420
|
+
}),
|
|
421
|
+
permissionGrantIds,
|
|
422
|
+
};
|
|
423
|
+
}));
|
|
424
|
+
}
|
|
425
|
+
|
|
276
426
|
/**
|
|
277
427
|
* Cached sync targets result from the last {@link getSyncTargets} call.
|
|
278
428
|
* Invalidated on identity registration/unregistration/update.
|
|
279
429
|
* TTL-based: cleared after 30 seconds to pick up DID document changes.
|
|
280
430
|
*/
|
|
281
431
|
private _syncTargetsCache?: {
|
|
282
|
-
targets:
|
|
432
|
+
targets: SyncTarget[];
|
|
283
433
|
timestamp: number;
|
|
284
434
|
};
|
|
285
435
|
|
|
@@ -295,6 +445,9 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
295
445
|
/** TTL for the sync targets cache (30 seconds). */
|
|
296
446
|
private static readonly SYNC_TARGETS_CACHE_TTL_MS = 30_000;
|
|
297
447
|
|
|
448
|
+
/** Backoff schedule for recently published did:dht records. */
|
|
449
|
+
private static readonly DID_RESOLUTION_RETRY_BACKOFF_MS = [2000, 4000, 8000];
|
|
450
|
+
|
|
298
451
|
/** Count of consecutive SMT sync failures (for backoff in poll mode). */
|
|
299
452
|
private _consecutiveFailures = 0;
|
|
300
453
|
|
|
@@ -428,7 +581,12 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
428
581
|
|
|
429
582
|
// If live sync is active, hot-add subscriptions for this identity.
|
|
430
583
|
if (this._syncMode === 'live') {
|
|
431
|
-
await this.addIdentityToLiveSync(did, options);
|
|
584
|
+
const currentIdentityKeys = await this.addIdentityToLiveSync(did, options);
|
|
585
|
+
if (currentIdentityKeys.size > 0) {
|
|
586
|
+
await this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
587
|
+
}
|
|
588
|
+
} else {
|
|
589
|
+
await this.tryPruneSupersededDurableLinksForRegisteredIdentity(did, options);
|
|
432
590
|
}
|
|
433
591
|
}
|
|
434
592
|
|
|
@@ -447,6 +605,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
447
605
|
await registeredIdentities.del(did);
|
|
448
606
|
this._syncTargetsCache = undefined;
|
|
449
607
|
this._syncTargetsCacheGeneration++;
|
|
608
|
+
await this.pruneSupersededDurableLinksForIdentity(did, new Set());
|
|
450
609
|
}
|
|
451
610
|
|
|
452
611
|
public async getIdentityOptions(did: string): Promise<SyncIdentityOptions | undefined> {
|
|
@@ -480,19 +639,17 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
480
639
|
this._syncTargetsCache = undefined;
|
|
481
640
|
this._syncTargetsCacheGeneration++;
|
|
482
641
|
|
|
483
|
-
// Always persist the new delegate to durable links, regardless of
|
|
484
|
-
// sync mode. If sync is stopped or polling, existing persisted links
|
|
485
|
-
// would otherwise keep the old delegateDid. When live sync starts
|
|
486
|
-
// later, initializeLinkTarget() loads the link from LevelDB without
|
|
487
|
-
// normalizing delegateDid, so repair/reconcile paths could use stale
|
|
488
|
-
// delegate data.
|
|
489
|
-
await this.ledger.updateDelegateDid(did, options.delegateDid);
|
|
490
|
-
|
|
491
642
|
// If live sync is active, tear down and rebuild subscriptions with
|
|
492
|
-
// the new options.
|
|
643
|
+
// the new options. Delegate/scope changes derive a new authorization
|
|
644
|
+
// epoch, so existing durable links are not mutated in place.
|
|
493
645
|
if (this._syncMode === 'live' && this.hasActiveLinksForDid(did)) {
|
|
494
646
|
await this.removeIdentityFromLiveSync(did);
|
|
495
|
-
await this.addIdentityToLiveSync(did, options);
|
|
647
|
+
const currentIdentityKeys = await this.addIdentityToLiveSync(did, options);
|
|
648
|
+
if (currentIdentityKeys.size > 0) {
|
|
649
|
+
await this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
650
|
+
}
|
|
651
|
+
} else {
|
|
652
|
+
await this.tryPruneSupersededDurableLinksForRegisteredIdentity(did, options);
|
|
496
653
|
}
|
|
497
654
|
}
|
|
498
655
|
|
|
@@ -526,15 +683,12 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
526
683
|
|
|
527
684
|
const results = await Promise.allSettled([...byUrl.entries()].map(async ([dwnUrl, targets]) => {
|
|
528
685
|
for (const target of targets) {
|
|
529
|
-
const { did, delegateDid, protocol } = target;
|
|
530
686
|
try {
|
|
531
|
-
await this.
|
|
532
|
-
did, dwnUrl, delegateDid, protocol,
|
|
533
|
-
}, { direction });
|
|
687
|
+
await this.reconcileProjectionTarget(target, { direction });
|
|
534
688
|
} catch (error: any) {
|
|
535
689
|
// Skip remaining targets for this DWN endpoint.
|
|
536
690
|
groupsFailed++;
|
|
537
|
-
console.error(`SyncEngineLevel: Error syncing ${did} with ${dwnUrl}`, error);
|
|
691
|
+
console.error(`SyncEngineLevel: Error syncing ${target.did} with ${dwnUrl}`, error);
|
|
538
692
|
return;
|
|
539
693
|
}
|
|
540
694
|
}
|
|
@@ -636,7 +790,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
636
790
|
}
|
|
637
791
|
|
|
638
792
|
// ---------------------------------------------------------------------------
|
|
639
|
-
// Poll-mode sync
|
|
793
|
+
// Poll-mode sync
|
|
640
794
|
// ---------------------------------------------------------------------------
|
|
641
795
|
|
|
642
796
|
private async startPollSync(intervalMilliseconds: number): Promise<void> {
|
|
@@ -768,7 +922,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
768
922
|
}
|
|
769
923
|
|
|
770
924
|
// ---------------------------------------------------------------------------
|
|
771
|
-
// Per-link repair and degraded-poll orchestration
|
|
925
|
+
// Per-link repair and degraded-poll orchestration
|
|
772
926
|
// ---------------------------------------------------------------------------
|
|
773
927
|
|
|
774
928
|
/** Maximum consecutive repair attempts before falling back to degraded_poll. */
|
|
@@ -811,27 +965,19 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
811
965
|
link: ReplicationLinkState,
|
|
812
966
|
options?: { resumeToken?: ProgressToken },
|
|
813
967
|
): Promise<void> {
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
link.connectivity = 'offline';
|
|
817
|
-
await this.ledger.setStatus(link, 'repairing');
|
|
818
|
-
|
|
819
|
-
this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevStatus, to: 'repairing' });
|
|
820
|
-
if (prevConnectivity !== 'offline') {
|
|
821
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevConnectivity, to: 'offline' });
|
|
968
|
+
if (link.status === 'terminal_incomplete') {
|
|
969
|
+
return;
|
|
822
970
|
}
|
|
823
971
|
|
|
972
|
+
await this.setLinkOfflineStatus(link, 'repairing');
|
|
973
|
+
|
|
824
974
|
if (options?.resumeToken) {
|
|
825
975
|
this._repairContext.set(linkKey, { resumeToken: options.resumeToken });
|
|
826
976
|
}
|
|
827
977
|
|
|
828
978
|
// Clear runtime ordinals immediately — stale state must not linger
|
|
829
979
|
// across repair attempts.
|
|
830
|
-
|
|
831
|
-
if (rt) {
|
|
832
|
-
rt.inflight.clear();
|
|
833
|
-
rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
|
|
834
|
-
}
|
|
980
|
+
this.clearLinkRuntimeInflight(linkKey);
|
|
835
981
|
|
|
836
982
|
// Kick off repair with retry scheduling on failure.
|
|
837
983
|
void this.repairLink(linkKey).catch(() => {
|
|
@@ -839,6 +985,68 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
839
985
|
});
|
|
840
986
|
}
|
|
841
987
|
|
|
988
|
+
private async transitionToTerminalIncomplete(
|
|
989
|
+
linkKey: string,
|
|
990
|
+
link: ReplicationLinkState,
|
|
991
|
+
): Promise<void> {
|
|
992
|
+
if (link.status === 'terminal_incomplete') {
|
|
993
|
+
return;
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
await this.setLinkOfflineStatus(link, 'terminal_incomplete');
|
|
997
|
+
|
|
998
|
+
await this.closeLinkSubscriptions(link);
|
|
999
|
+
|
|
1000
|
+
this.clearLinkRuntimeInflight(linkKey);
|
|
1001
|
+
|
|
1002
|
+
const retryTimer = this._repairRetryTimers.get(linkKey);
|
|
1003
|
+
if (retryTimer) {
|
|
1004
|
+
clearTimeout(retryTimer);
|
|
1005
|
+
this._repairRetryTimers.delete(linkKey);
|
|
1006
|
+
}
|
|
1007
|
+
const degradedTimer = this._degradedPollTimers.get(linkKey);
|
|
1008
|
+
if (degradedTimer) {
|
|
1009
|
+
clearInterval(degradedTimer);
|
|
1010
|
+
this._degradedPollTimers.delete(linkKey);
|
|
1011
|
+
}
|
|
1012
|
+
const reconcileTimer = this._reconcileTimers.get(linkKey);
|
|
1013
|
+
if (reconcileTimer) {
|
|
1014
|
+
clearTimeout(reconcileTimer);
|
|
1015
|
+
this._reconcileTimers.delete(linkKey);
|
|
1016
|
+
}
|
|
1017
|
+
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
1018
|
+
if (pushRuntime?.timer) {
|
|
1019
|
+
clearTimeout(pushRuntime.timer);
|
|
1020
|
+
}
|
|
1021
|
+
this._pushRuntimes.delete(linkKey);
|
|
1022
|
+
|
|
1023
|
+
this._repairAttempts.delete(linkKey);
|
|
1024
|
+
this._repairContext.delete(linkKey);
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
private async setLinkOfflineStatus(link: ReplicationLinkState, status: ReplicationLinkState['status']): Promise<void> {
|
|
1028
|
+
const prevStatus = link.status;
|
|
1029
|
+
const prevConnectivity = link.connectivity;
|
|
1030
|
+
link.connectivity = 'offline';
|
|
1031
|
+
await this.ledger.setStatus(link, status);
|
|
1032
|
+
|
|
1033
|
+
const eventScope = syncEventScope(link.scope);
|
|
1034
|
+
this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope, from: prevStatus, to: status });
|
|
1035
|
+
if (prevConnectivity !== 'offline') {
|
|
1036
|
+
this.emitEvent({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope, from: prevConnectivity, to: 'offline' });
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
private clearLinkRuntimeInflight(linkKey: string): void {
|
|
1041
|
+
const rt = this._linkRuntimes.get(linkKey);
|
|
1042
|
+
if (!rt) {
|
|
1043
|
+
return;
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
rt.inflight.clear();
|
|
1047
|
+
rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
|
|
1048
|
+
}
|
|
1049
|
+
|
|
842
1050
|
/**
|
|
843
1051
|
* Schedule a retry for a failed repair. Uses exponential backoff.
|
|
844
1052
|
* No-op if the link is already in `degraded_poll` (timer loop owns retries)
|
|
@@ -925,9 +1133,10 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
925
1133
|
// The old repair closure must not mutate the replacement link's state.
|
|
926
1134
|
const isStaleLink = (): boolean => this._activeLinks.get(linkKey) !== link;
|
|
927
1135
|
|
|
928
|
-
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid,
|
|
1136
|
+
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, scope, authorization } = link;
|
|
1137
|
+
const eventScope = syncEventScope(scope);
|
|
929
1138
|
|
|
930
|
-
this.emitEvent({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1139
|
+
this.emitEvent({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, attempt: (this._repairAttempts.get(linkKey) ?? 0) + 1 });
|
|
931
1140
|
const attempts = (this._repairAttempts.get(linkKey) ?? 0) + 1;
|
|
932
1141
|
this._repairAttempts.set(linkKey, attempts);
|
|
933
1142
|
|
|
@@ -945,9 +1154,13 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
945
1154
|
|
|
946
1155
|
try {
|
|
947
1156
|
// Step 3: Run SMT reconciliation for this link.
|
|
948
|
-
const reconcileOutcome = await this.
|
|
949
|
-
|
|
950
|
-
|
|
1157
|
+
const reconcileOutcome = await this.reconcileProjectionTarget({
|
|
1158
|
+
did,
|
|
1159
|
+
dwnUrl,
|
|
1160
|
+
delegateDid,
|
|
1161
|
+
scope,
|
|
1162
|
+
authorization,
|
|
1163
|
+
}, undefined, () => this._engineGeneration === generation && !isStaleLink());
|
|
951
1164
|
if (reconcileOutcome.aborted) { return; }
|
|
952
1165
|
|
|
953
1166
|
// Step 4: Determine the post-repair pull resume token.
|
|
@@ -972,7 +1185,16 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
972
1185
|
await this.ledger.saveLink(link);
|
|
973
1186
|
if (this._engineGeneration !== generation || isStaleLink()) { return; }
|
|
974
1187
|
|
|
975
|
-
const target = {
|
|
1188
|
+
const target = {
|
|
1189
|
+
did,
|
|
1190
|
+
dwnUrl,
|
|
1191
|
+
delegateDid,
|
|
1192
|
+
scope,
|
|
1193
|
+
authorization,
|
|
1194
|
+
authorizationEpoch : link.authorizationEpoch,
|
|
1195
|
+
permissionGrantIds : this.getAuthorizationGrantIds(authorization),
|
|
1196
|
+
linkKey,
|
|
1197
|
+
};
|
|
976
1198
|
try {
|
|
977
1199
|
await this.openLivePullSubscription(target);
|
|
978
1200
|
} catch (pullErr: any) {
|
|
@@ -1013,16 +1235,15 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1013
1235
|
link.connectivity = 'online';
|
|
1014
1236
|
await this.ledger.setStatus(link, 'live');
|
|
1015
1237
|
|
|
1016
|
-
//
|
|
1017
|
-
//
|
|
1018
|
-
//
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, protocol });
|
|
1238
|
+
// Root convergence proves primary CID membership matches, but it does
|
|
1239
|
+
// not prove dependencies are usable. Keep closure failures until a later
|
|
1240
|
+
// successful apply/closure pass clears the specific CID.
|
|
1241
|
+
await this.clearRootConvergenceDeadLettersForScope(did, dwnUrl, scope);
|
|
1242
|
+
this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope });
|
|
1022
1243
|
if (prevRepairConnectivity !== 'online') {
|
|
1023
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1244
|
+
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: prevRepairConnectivity, to: 'online' });
|
|
1024
1245
|
}
|
|
1025
|
-
this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1246
|
+
this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: 'repairing', to: 'live' });
|
|
1026
1247
|
|
|
1027
1248
|
} catch (error: any) {
|
|
1028
1249
|
// If teardown occurred during repair or the link was replaced by a
|
|
@@ -1030,7 +1251,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1030
1251
|
if (this._engineGeneration !== generation || isStaleLink()) { return; }
|
|
1031
1252
|
|
|
1032
1253
|
console.error(`SyncEngineLevel: Repair failed for ${did} -> ${dwnUrl} (attempt ${attempts})`, error);
|
|
1033
|
-
this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1254
|
+
this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, attempt: attempts, error: String(error.message ?? error) });
|
|
1034
1255
|
|
|
1035
1256
|
if (attempts >= SyncEngineLevel.MAX_REPAIR_ATTEMPTS) {
|
|
1036
1257
|
console.warn(`SyncEngineLevel: Max repair attempts reached for ${did} -> ${dwnUrl}, entering degraded_poll`);
|
|
@@ -1048,21 +1269,26 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1048
1269
|
*/
|
|
1049
1270
|
private async closeLinkSubscriptions(link: ReplicationLinkState): Promise<void> {
|
|
1050
1271
|
const { tenantDid: did, remoteEndpoint: dwnUrl } = link;
|
|
1051
|
-
const linkKey = this.buildLinkKey(did, dwnUrl, link.
|
|
1272
|
+
const linkKey = this.buildLinkKey(did, dwnUrl, link.projectionId, link.authorizationEpoch);
|
|
1052
1273
|
|
|
1053
|
-
|
|
1274
|
+
await this.closeLiveSubscription(linkKey);
|
|
1275
|
+
await this.closeLocalSubscription(linkKey);
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
private async closeLiveSubscription(linkKey: string): Promise<void> {
|
|
1054
1279
|
const pullSub = this._liveSubscriptions.find((s) => s.linkKey === linkKey);
|
|
1055
|
-
if (pullSub) {
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1280
|
+
if (!pullSub) { return; }
|
|
1281
|
+
|
|
1282
|
+
try { await pullSub.close(); } catch { /* best effort */ }
|
|
1283
|
+
this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
|
|
1284
|
+
}
|
|
1059
1285
|
|
|
1060
|
-
|
|
1286
|
+
private async closeLocalSubscription(linkKey: string): Promise<void> {
|
|
1061
1287
|
const pushSub = this._localSubscriptions.find((s) => s.linkKey === linkKey);
|
|
1062
|
-
if (pushSub) {
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1288
|
+
if (!pushSub) { return; }
|
|
1289
|
+
|
|
1290
|
+
try { await pushSub.close(); } catch { /* best effort */ }
|
|
1291
|
+
this._localSubscriptions = this._localSubscriptions.filter(s => s !== pushSub);
|
|
1066
1292
|
}
|
|
1067
1293
|
|
|
1068
1294
|
/**
|
|
@@ -1078,8 +1304,9 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1078
1304
|
const prevDegradedStatus = link.status;
|
|
1079
1305
|
await this.ledger.setStatus(link, 'degraded_poll');
|
|
1080
1306
|
this._repairAttempts.delete(linkKey);
|
|
1081
|
-
|
|
1082
|
-
this.emitEvent({ type: '
|
|
1307
|
+
const eventScope = syncEventScope(link.scope);
|
|
1308
|
+
this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope, from: prevDegradedStatus, to: 'degraded_poll' });
|
|
1309
|
+
this.emitEvent({ type: 'degraded-poll:entered', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope });
|
|
1083
1310
|
|
|
1084
1311
|
// Clear any existing timer for this link.
|
|
1085
1312
|
const existing = this._degradedPollTimers.get(linkKey);
|
|
@@ -1194,7 +1421,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1194
1421
|
type : 'link:connectivity-change',
|
|
1195
1422
|
tenantDid : link.tenantDid,
|
|
1196
1423
|
remoteEndpoint : link.remoteEndpoint,
|
|
1197
|
-
|
|
1424
|
+
...syncEventScope(link.scope),
|
|
1198
1425
|
from : prev,
|
|
1199
1426
|
to : 'offline',
|
|
1200
1427
|
});
|
|
@@ -1319,80 +1546,144 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1319
1546
|
|
|
1320
1547
|
/**
|
|
1321
1548
|
* Initialize a single replication link target: create or resume the durable
|
|
1322
|
-
* link,
|
|
1323
|
-
* transition the link to `'live'`.
|
|
1549
|
+
* link, open pull + push subscriptions, and transition the link to `'live'`.
|
|
1324
1550
|
*/
|
|
1325
|
-
private async initializeLinkTarget(target: {
|
|
1326
|
-
did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
|
|
1327
|
-
}): Promise<void> {
|
|
1551
|
+
private async initializeLinkTarget(target: SyncTarget): Promise<LinkInitializationResult> {
|
|
1328
1552
|
let link: ReplicationLinkState | undefined;
|
|
1329
1553
|
try {
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
link
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
scope : linkScope,
|
|
1337
|
-
delegateDid : target.delegateDid,
|
|
1338
|
-
protocol : target.protocol,
|
|
1339
|
-
});
|
|
1340
|
-
|
|
1341
|
-
const linkKey = this.buildLinkKey(target.did, target.dwnUrl, link.scopeId);
|
|
1554
|
+
link = await this.getOrCreateReplicationLink(target);
|
|
1555
|
+
const linkKey = this.getReplicationLinkKey(target, link);
|
|
1556
|
+
this._activeLinks.set(linkKey, link);
|
|
1557
|
+
if (link.status === 'terminal_incomplete') {
|
|
1558
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1559
|
+
}
|
|
1342
1560
|
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
await this.ledger.saveLink(link);
|
|
1349
|
-
await this.deleteLegacyCursor(legacyKey);
|
|
1350
|
-
}
|
|
1561
|
+
const subscriptionResult = await this.openLinkSubscriptions({ ...target, linkKey });
|
|
1562
|
+
if (subscriptionResult === LinkSubscriptionOpenResult.ReadyForLive) {
|
|
1563
|
+
await this.markLinkLive(target, link, linkKey);
|
|
1564
|
+
} else if (subscriptionResult === LinkSubscriptionOpenResult.Polling) {
|
|
1565
|
+
await this.markLinkPolling(target, link);
|
|
1351
1566
|
}
|
|
1567
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1568
|
+
} catch (error: any) {
|
|
1569
|
+
return this.handleInitializeLinkTargetError(target, link, error);
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1352
1572
|
|
|
1353
|
-
|
|
1573
|
+
private async getOrCreateReplicationLink(target: SyncTarget): Promise<ReplicationLinkState> {
|
|
1574
|
+
return this.ledger.getOrCreateLink({
|
|
1575
|
+
tenantDid : target.did,
|
|
1576
|
+
remoteEndpoint : target.dwnUrl,
|
|
1577
|
+
scope : target.scope,
|
|
1578
|
+
authorization : target.authorization,
|
|
1579
|
+
authorizationEpoch : target.authorizationEpoch,
|
|
1580
|
+
delegateDid : target.delegateDid,
|
|
1581
|
+
});
|
|
1582
|
+
}
|
|
1354
1583
|
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
await this.openLocalPushSubscription(targetWithKey);
|
|
1359
|
-
} catch (pushError) {
|
|
1360
|
-
const pullSub = this._liveSubscriptions.find((s) => s.linkKey === linkKey);
|
|
1361
|
-
if (pullSub) {
|
|
1362
|
-
try { await pullSub.close(); } catch { /* best effort */ }
|
|
1363
|
-
this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
|
|
1364
|
-
}
|
|
1365
|
-
throw pushError;
|
|
1366
|
-
}
|
|
1584
|
+
private getReplicationLinkKey(target: SyncTarget, link: ReplicationLinkState): string {
|
|
1585
|
+
return this.buildLinkKey(target.did, target.dwnUrl, link.projectionId, link.authorizationEpoch);
|
|
1586
|
+
}
|
|
1367
1587
|
|
|
1368
|
-
|
|
1369
|
-
|
|
1588
|
+
private async openLinkSubscriptions(target: LinkSyncTarget): Promise<LinkSubscriptionOpenResult> {
|
|
1589
|
+
if (!SyncEngineLevel.supportsLiveSubscriptions(target.scope)) {
|
|
1590
|
+
return LinkSubscriptionOpenResult.Polling;
|
|
1591
|
+
}
|
|
1370
1592
|
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
}
|
|
1593
|
+
await this.openLivePullSubscription(target);
|
|
1594
|
+
const link = this._activeLinks.get(target.linkKey);
|
|
1595
|
+
if (link?.status === 'repairing') {
|
|
1596
|
+
await this.closeLiveSubscription(target.linkKey);
|
|
1597
|
+
return LinkSubscriptionOpenResult.Repairing;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
try {
|
|
1601
|
+
await this.openLocalPushSubscription(target);
|
|
1602
|
+
} catch (error) {
|
|
1603
|
+
await this.closeLiveSubscription(target.linkKey);
|
|
1604
|
+
throw error;
|
|
1605
|
+
}
|
|
1606
|
+
return LinkSubscriptionOpenResult.ReadyForLive;
|
|
1607
|
+
}
|
|
1387
1608
|
|
|
1388
|
-
|
|
1609
|
+
private static supportsLiveSubscriptions(scope: SyncScope): boolean {
|
|
1610
|
+
// Records-primary projected links reconcile by root/diff polling until the
|
|
1611
|
+
// DWN has explicit path/context live subscription semantics.
|
|
1612
|
+
return scope.kind !== 'recordsProjection';
|
|
1613
|
+
}
|
|
1389
1614
|
|
|
1390
|
-
|
|
1391
|
-
|
|
1615
|
+
private async markLinkLive(target: SyncTarget, link: ReplicationLinkState, linkKey: string): Promise<void> {
|
|
1616
|
+
this.emitEvent({
|
|
1617
|
+
type : 'link:status-change',
|
|
1618
|
+
tenantDid : target.did,
|
|
1619
|
+
remoteEndpoint : target.dwnUrl,
|
|
1620
|
+
...syncEventScope(target.scope),
|
|
1621
|
+
from : 'initializing',
|
|
1622
|
+
to : 'live'
|
|
1623
|
+
});
|
|
1624
|
+
await this.ledger.setStatus(link, 'live');
|
|
1392
1625
|
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1626
|
+
if (link.needsReconcile) {
|
|
1627
|
+
this.scheduleReconcile(linkKey, 1000);
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
private async markLinkPolling(target: SyncTarget, link: ReplicationLinkState): Promise<void> {
|
|
1632
|
+
this.emitEvent({
|
|
1633
|
+
type : 'link:status-change',
|
|
1634
|
+
tenantDid : target.did,
|
|
1635
|
+
remoteEndpoint : target.dwnUrl,
|
|
1636
|
+
...syncEventScope(target.scope),
|
|
1637
|
+
from : 'initializing',
|
|
1638
|
+
to : 'polling'
|
|
1639
|
+
});
|
|
1640
|
+
await this.ledger.setStatus(link, 'polling');
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1643
|
+
private async handleInitializeLinkTargetError(
|
|
1644
|
+
target: SyncTarget,
|
|
1645
|
+
link: ReplicationLinkState | undefined,
|
|
1646
|
+
error: any,
|
|
1647
|
+
): Promise<LinkInitializationResult> {
|
|
1648
|
+
if (error.isProgressGap && link) {
|
|
1649
|
+
const linkKey = this.getReplicationLinkKey(target, link);
|
|
1650
|
+
console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
|
|
1651
|
+
this.emitEvent({
|
|
1652
|
+
type : 'gap:detected',
|
|
1653
|
+
tenantDid : target.did,
|
|
1654
|
+
remoteEndpoint : target.dwnUrl,
|
|
1655
|
+
...syncEventScope(target.scope),
|
|
1656
|
+
reason : 'ProgressGap'
|
|
1657
|
+
});
|
|
1658
|
+
await this.transitionToRepairing(linkKey, link, {
|
|
1659
|
+
resumeToken: error.gapInfo?.latestAvailable,
|
|
1660
|
+
});
|
|
1661
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
|
|
1665
|
+
if (link) {
|
|
1666
|
+
this.cleanupFailedLinkInitialization(this.getReplicationLinkKey(target, link));
|
|
1667
|
+
}
|
|
1668
|
+
if (this.isDidResolutionFailure(error)) {
|
|
1669
|
+
throw error;
|
|
1670
|
+
}
|
|
1671
|
+
return { status: LinkInitializationStatus.Failed };
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
private createActiveLinkInitializationResult(link: ReplicationLinkState): LinkInitializationResult {
|
|
1675
|
+
return {
|
|
1676
|
+
status : LinkInitializationStatus.Active,
|
|
1677
|
+
durableLinkIdentityKey : this.getDurableLinkIdentityKey(link),
|
|
1678
|
+
};
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
private cleanupFailedLinkInitialization(linkKey: string): void {
|
|
1682
|
+
this._activeLinks.delete(linkKey);
|
|
1683
|
+
this._linkRuntimes.delete(linkKey);
|
|
1684
|
+
|
|
1685
|
+
if (this._liveSubscriptions.length === 0) {
|
|
1686
|
+
this._connectivityState = 'unknown';
|
|
1396
1687
|
}
|
|
1397
1688
|
}
|
|
1398
1689
|
|
|
@@ -1404,31 +1695,31 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1404
1695
|
* causing a 401. Retrying with exponential backoff lets the
|
|
1405
1696
|
* propagation settle before giving up.
|
|
1406
1697
|
*/
|
|
1407
|
-
private async initializeLinkTargetWithRetry(target: {
|
|
1408
|
-
did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
|
|
1409
|
-
}): Promise<void> {
|
|
1698
|
+
private async initializeLinkTargetWithRetry(target: SyncTarget): Promise<LinkInitializationResult> {
|
|
1410
1699
|
try {
|
|
1411
|
-
await this.initializeLinkTarget(target);
|
|
1700
|
+
return await this.initializeLinkTarget(target);
|
|
1412
1701
|
} catch (error: any) {
|
|
1413
|
-
|
|
1414
|
-
const isDidResolutionFailure = msg.includes('GetPublicKeyNotFound') || msg.includes('notFound');
|
|
1415
|
-
if (!isDidResolutionFailure) { throw error; }
|
|
1702
|
+
if (!this.isDidResolutionFailure(error)) { throw error; }
|
|
1416
1703
|
|
|
1417
|
-
const
|
|
1418
|
-
for (const delay of delays) {
|
|
1704
|
+
for (const delay of SyncEngineLevel.DID_RESOLUTION_RETRY_BACKOFF_MS) {
|
|
1419
1705
|
await sleep(delay);
|
|
1420
1706
|
try {
|
|
1421
|
-
await this.initializeLinkTarget(target);
|
|
1422
|
-
return;
|
|
1707
|
+
return await this.initializeLinkTarget(target);
|
|
1423
1708
|
} catch {
|
|
1424
1709
|
// Continue to next attempt.
|
|
1425
1710
|
}
|
|
1426
1711
|
}
|
|
1427
1712
|
// All retries exhausted — the original error was already logged
|
|
1428
1713
|
// by initializeLinkTarget's catch block.
|
|
1714
|
+
return { status: LinkInitializationStatus.Failed };
|
|
1429
1715
|
}
|
|
1430
1716
|
}
|
|
1431
1717
|
|
|
1718
|
+
private isDidResolutionFailure(error: any): boolean {
|
|
1719
|
+
const message = error.message ?? '';
|
|
1720
|
+
return message.includes('GetPublicKeyNotFound');
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1432
1723
|
// ---------------------------------------------------------------------------
|
|
1433
1724
|
// Hot-add / hot-remove: per-identity live sync management
|
|
1434
1725
|
// ---------------------------------------------------------------------------
|
|
@@ -1447,23 +1738,23 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1447
1738
|
}
|
|
1448
1739
|
|
|
1449
1740
|
/** Hot-add a single identity to the active live sync session. */
|
|
1450
|
-
private async addIdentityToLiveSync(did: string, options: SyncIdentityOptions): Promise<
|
|
1451
|
-
const { protocols, delegateDid } = options;
|
|
1741
|
+
private async addIdentityToLiveSync(did: string, options: SyncIdentityOptions): Promise<Set<string>> {
|
|
1452
1742
|
const dwnEndpointUrls = await this.agent.dwn.getDwnEndpointUrlsForTarget(did);
|
|
1453
|
-
if (dwnEndpointUrls.length === 0) { return; }
|
|
1743
|
+
if (dwnEndpointUrls.length === 0) { return new Set(); }
|
|
1454
1744
|
|
|
1455
|
-
const targets:
|
|
1745
|
+
const targets: SyncTarget[] = [];
|
|
1456
1746
|
for (const dwnUrl of dwnEndpointUrls) {
|
|
1457
|
-
|
|
1458
|
-
targets.push({ did, delegateDid, dwnUrl });
|
|
1459
|
-
} else {
|
|
1460
|
-
for (const protocol of protocols) {
|
|
1461
|
-
targets.push({ did, delegateDid, dwnUrl, protocol });
|
|
1462
|
-
}
|
|
1463
|
-
}
|
|
1747
|
+
targets.push(...await this.buildSyncTargetsForEndpoint(did, dwnUrl, options));
|
|
1464
1748
|
}
|
|
1465
1749
|
|
|
1466
|
-
await Promise.allSettled(targets.map(t => this.initializeLinkTargetWithRetry(t)));
|
|
1750
|
+
const results = await Promise.allSettled(targets.map(t => this.initializeLinkTargetWithRetry(t)));
|
|
1751
|
+
const currentIdentityKeys = new Set<string>();
|
|
1752
|
+
for (const result of results) {
|
|
1753
|
+
if (result.status === 'fulfilled' && result.value.status === LinkInitializationStatus.Active) {
|
|
1754
|
+
currentIdentityKeys.add(result.value.durableLinkIdentityKey);
|
|
1755
|
+
}
|
|
1756
|
+
}
|
|
1757
|
+
return currentIdentityKeys;
|
|
1467
1758
|
}
|
|
1468
1759
|
|
|
1469
1760
|
/** Hot-remove a single identity from the active live sync session. */
|
|
@@ -1512,6 +1803,36 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1512
1803
|
this._closureContexts.delete(did);
|
|
1513
1804
|
}
|
|
1514
1805
|
|
|
1806
|
+
private async tryPruneSupersededDurableLinksForRegisteredIdentity(did: string, options: SyncIdentityOptions): Promise<void> {
|
|
1807
|
+
try {
|
|
1808
|
+
const currentIdentityKeys = await this.getDurableLinkIdentityKeysForRegisteredIdentity(did, options);
|
|
1809
|
+
await this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
1810
|
+
} catch (error: unknown) {
|
|
1811
|
+
console.warn(`SyncEngineLevel: Failed to prune superseded durable links for ${did}`, error);
|
|
1812
|
+
}
|
|
1813
|
+
}
|
|
1814
|
+
|
|
1815
|
+
private async getDurableLinkIdentityKeysForRegisteredIdentity(did: string, options: SyncIdentityOptions): Promise<Set<string>> {
|
|
1816
|
+
const scope = syncScopeFromProtocols(options.protocols);
|
|
1817
|
+
const resolutions = await this.buildSyncTargetResolutions(did, scope, options);
|
|
1818
|
+
const keys = new Set<string>();
|
|
1819
|
+
for (const resolution of resolutions) {
|
|
1820
|
+
const projectionId = await computeProjectionId(did, resolution.scope);
|
|
1821
|
+
keys.add(SyncEngineLevel.durableLinkIdentityKey(did, projectionId, resolution.authorizationEpoch));
|
|
1822
|
+
}
|
|
1823
|
+
return keys;
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
private async pruneSupersededDurableLinksForIdentity(did: string, currentIdentityKeys: Set<string>): Promise<void> {
|
|
1827
|
+
const links = await this.ledger.getLinksForTenant(did);
|
|
1828
|
+
await Promise.all(links.map(async link => {
|
|
1829
|
+
if (currentIdentityKeys.has(this.getDurableLinkIdentityKey(link))) {
|
|
1830
|
+
return;
|
|
1831
|
+
}
|
|
1832
|
+
await this.ledger.deleteLink(link.tenantDid, link.remoteEndpoint, link.projectionId, link.authorizationEpoch);
|
|
1833
|
+
}));
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1515
1836
|
// ---------------------------------------------------------------------------
|
|
1516
1837
|
// Live pull: MessagesSubscribe to remote DWN
|
|
1517
1838
|
// ---------------------------------------------------------------------------
|
|
@@ -1520,60 +1841,18 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1520
1841
|
* Opens a MessagesSubscribe WebSocket subscription to a remote DWN.
|
|
1521
1842
|
* Incoming events are processed locally as they arrive.
|
|
1522
1843
|
*/
|
|
1523
|
-
private async openLivePullSubscription(target: {
|
|
1524
|
-
did
|
|
1525
|
-
|
|
1526
|
-
}): Promise<void> {
|
|
1527
|
-
const { did, delegateDid, dwnUrl, protocol } = target;
|
|
1844
|
+
private async openLivePullSubscription(target: LinkSyncTarget): Promise<void> {
|
|
1845
|
+
const { did, delegateDid, dwnUrl } = target;
|
|
1846
|
+
const eventScope = syncEventScope(target.scope);
|
|
1528
1847
|
|
|
1529
|
-
// Resolve the cursor from the link's durable pull checkpoint.
|
|
1530
|
-
// Legacy syncCursors migration happens at link load time in startLiveSync().
|
|
1531
1848
|
const cursorKey = target.linkKey;
|
|
1532
1849
|
const link = this._activeLinks.get(cursorKey);
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
// Guard against corrupted tokens with empty fields — these would fail
|
|
1536
|
-
// MessagesSubscribe JSON schema validation (minLength: 1). Discard and
|
|
1537
|
-
// start from the beginning rather than crash the subscription.
|
|
1538
|
-
if (cursor && (!cursor.streamId || !cursor.messageCid || !cursor.epoch || !cursor.position)) {
|
|
1539
|
-
console.warn(`SyncEngineLevel: Discarding stored cursor with empty field(s) for ${did} -> ${dwnUrl}`);
|
|
1540
|
-
cursor = undefined;
|
|
1541
|
-
if (link) {
|
|
1542
|
-
ReplicationLedger.resetCheckpoint(link.pull);
|
|
1543
|
-
await this.ledger.saveLink(link);
|
|
1544
|
-
}
|
|
1545
|
-
}
|
|
1850
|
+
const cursor = await this.getInitialPullCursor({ did, dwnUrl, link });
|
|
1546
1851
|
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
// EventLog delivers only matching events (server-side filtering). This replaces
|
|
1550
|
-
// the less efficient agent-side isEventInScope filtering for the pull path.
|
|
1551
|
-
// Note: only the first prefix is used as the MessagesFilter field because
|
|
1552
|
-
// MessagesFilter.protocolPathPrefix is a single string. Multiple prefixes
|
|
1553
|
-
// would need multiple filters (OR semantics) — for now we use the first one.
|
|
1554
|
-
const protocolPathPrefix = link?.scope.kind === 'protocol'
|
|
1555
|
-
? link.scope.protocolPathPrefixes?.[0]
|
|
1556
|
-
: undefined;
|
|
1557
|
-
const filters = protocol
|
|
1558
|
-
? [{ protocol, ...(protocolPathPrefix ? { protocolPathPrefix } : {}) }]
|
|
1852
|
+
const filters = target.scope.kind === 'protocolSet'
|
|
1853
|
+
? target.scope.protocols.map(protocol => ({ protocol }))
|
|
1559
1854
|
: [];
|
|
1560
1855
|
|
|
1561
|
-
// Look up permission grant for MessagesSubscribe if using a delegate.
|
|
1562
|
-
// The unified scope matching in AgentPermissionsApi accepts a
|
|
1563
|
-
// Messages.Read grant for MessagesSubscribe requests, so a single
|
|
1564
|
-
// lookup is sufficient.
|
|
1565
|
-
let permissionGrantId: string | undefined;
|
|
1566
|
-
if (delegateDid) {
|
|
1567
|
-
const grant = await this._permissionsApi.getPermissionForRequest({
|
|
1568
|
-
connectedDid : did,
|
|
1569
|
-
messageType : DwnInterface.MessagesSubscribe,
|
|
1570
|
-
delegateDid,
|
|
1571
|
-
protocol,
|
|
1572
|
-
cached : true
|
|
1573
|
-
});
|
|
1574
|
-
permissionGrantId = grant.grant.id;
|
|
1575
|
-
}
|
|
1576
|
-
|
|
1577
1856
|
const handlerGeneration = this._engineGeneration;
|
|
1578
1857
|
|
|
1579
1858
|
// Define the subscription handler that processes incoming events.
|
|
@@ -1582,249 +1861,20 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1582
1861
|
// ensures the checkpoint advances only when all earlier deliveries are committed.
|
|
1583
1862
|
// Capture the link reference at subscription-open time so we can
|
|
1584
1863
|
// detect remove+re-add via object identity, not just key existence.
|
|
1585
|
-
const
|
|
1586
|
-
const
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1864
|
+
const isStale = this.createLinkStalePredicate(cursorKey, link, handlerGeneration);
|
|
1865
|
+
const pullContext: LivePullContext = {
|
|
1866
|
+
did,
|
|
1867
|
+
dwnUrl,
|
|
1868
|
+
delegateDid,
|
|
1869
|
+
eventScope,
|
|
1870
|
+
linkKey : cursorKey,
|
|
1871
|
+
link,
|
|
1872
|
+
permissionGrantIds : target.permissionGrantIds,
|
|
1873
|
+
isStale,
|
|
1874
|
+
};
|
|
1590
1875
|
|
|
1591
1876
|
const subscriptionHandler = async (subMessage: SubscriptionMessage): Promise<void> => {
|
|
1592
|
-
|
|
1593
|
-
return;
|
|
1594
|
-
}
|
|
1595
|
-
|
|
1596
|
-
if (subMessage.type === 'eose') {
|
|
1597
|
-
// End-of-stored-events — catch-up complete.
|
|
1598
|
-
if (link) {
|
|
1599
|
-
// Guard: if the link transitioned to repairing while catch-up events
|
|
1600
|
-
// were being processed, skip all mutations — repair owns the state now.
|
|
1601
|
-
if (link.status !== 'live' && link.status !== 'initializing') {
|
|
1602
|
-
return;
|
|
1603
|
-
}
|
|
1604
|
-
|
|
1605
|
-
if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1606
|
-
console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1607
|
-
if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
|
|
1608
|
-
return;
|
|
1609
|
-
}
|
|
1610
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1611
|
-
this.drainCommittedPull(cursorKey);
|
|
1612
|
-
if (isStale()) { return; }
|
|
1613
|
-
await this.ledger.saveLink(link);
|
|
1614
|
-
}
|
|
1615
|
-
// Transport is reachable — set connectivity to online.
|
|
1616
|
-
if (link) {
|
|
1617
|
-
const prevEoseConnectivity = link.connectivity;
|
|
1618
|
-
link.connectivity = 'online';
|
|
1619
|
-
if (prevEoseConnectivity !== 'online') {
|
|
1620
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevEoseConnectivity, to: 'online' });
|
|
1621
|
-
}
|
|
1622
|
-
// If the link was marked dirty, schedule reconciliation now that it's healthy.
|
|
1623
|
-
if (link.needsReconcile) {
|
|
1624
|
-
this.scheduleReconcile(cursorKey, 500);
|
|
1625
|
-
}
|
|
1626
|
-
} else {
|
|
1627
|
-
this._connectivityState = 'online';
|
|
1628
|
-
}
|
|
1629
|
-
return;
|
|
1630
|
-
}
|
|
1631
|
-
|
|
1632
|
-
if (subMessage.type === 'event') {
|
|
1633
|
-
const event: MessageEvent = subMessage.event;
|
|
1634
|
-
|
|
1635
|
-
// Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
|
|
1636
|
-
// skip all processing. Old subscription handlers may still fire after the
|
|
1637
|
-
// link transitions — these events should be ignored entirely, not just
|
|
1638
|
-
// skipped at the checkpoint level.
|
|
1639
|
-
if (link && link.status !== 'live' && link.status !== 'initializing') {
|
|
1640
|
-
return;
|
|
1641
|
-
}
|
|
1642
|
-
|
|
1643
|
-
// Domain validation: reject tokens from a different stream/epoch.
|
|
1644
|
-
if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1645
|
-
console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1646
|
-
if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
|
|
1647
|
-
return;
|
|
1648
|
-
}
|
|
1649
|
-
|
|
1650
|
-
// Subset scope filtering: if the link has protocolPath/contextId prefixes,
|
|
1651
|
-
// skip events that don't match. This is agent-side filtering because
|
|
1652
|
-
// MessagesSubscribe only supports protocol-level filtering today.
|
|
1653
|
-
//
|
|
1654
|
-
// Skipped events MUST advance contiguousAppliedToken — otherwise the
|
|
1655
|
-
// link would replay the same filtered-out events indefinitely after
|
|
1656
|
-
// reconnect/repair. This is safe because the event is intentionally
|
|
1657
|
-
// excluded from this scope and doesn't need processing.
|
|
1658
|
-
if (link && !isEventInScope(event.message, link.scope)) {
|
|
1659
|
-
if (!isStale()) {
|
|
1660
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1661
|
-
ReplicationLedger.commitContiguousToken(link.pull, subMessage.cursor);
|
|
1662
|
-
await this.ledger.saveLink(link);
|
|
1663
|
-
}
|
|
1664
|
-
return;
|
|
1665
|
-
}
|
|
1666
|
-
|
|
1667
|
-
// Assign a delivery ordinal BEFORE async processing begins.
|
|
1668
|
-
// This captures the delivery order even if processing completes out of order.
|
|
1669
|
-
const rt = link ? this.getOrCreateRuntime(cursorKey) : undefined;
|
|
1670
|
-
const ordinal = rt ? rt.nextDeliveryOrdinal++ : -1;
|
|
1671
|
-
if (rt) {
|
|
1672
|
-
rt.inflight.set(ordinal, { ordinal, token: subMessage.cursor, committed: false });
|
|
1673
|
-
}
|
|
1674
|
-
|
|
1675
|
-
try {
|
|
1676
|
-
// Extract inline data from the event (available for records <= 30 KB).
|
|
1677
|
-
let dataStream = this.extractDataStream(event);
|
|
1678
|
-
|
|
1679
|
-
// For large RecordsWrite messages (no inline data), fetch the data
|
|
1680
|
-
// from the remote DWN via MessagesRead before storing locally.
|
|
1681
|
-
if (!dataStream && isRecordsWrite(event) && (event.message.descriptor as any).dataCid) {
|
|
1682
|
-
const messageCid = await Message.getCid(event.message);
|
|
1683
|
-
const fetched = await fetchRemoteMessages({
|
|
1684
|
-
did, dwnUrl, delegateDid, protocol,
|
|
1685
|
-
messageCids : [messageCid],
|
|
1686
|
-
agent : this.agent,
|
|
1687
|
-
permissionsApi : this._permissionsApi,
|
|
1688
|
-
});
|
|
1689
|
-
if (fetched.length > 0 && fetched[0].dataStream) {
|
|
1690
|
-
dataStream = fetched[0].dataStream;
|
|
1691
|
-
}
|
|
1692
|
-
}
|
|
1693
|
-
|
|
1694
|
-
await this.agent.dwn.processRawMessage(did, event.message, { dataStream });
|
|
1695
|
-
if (isStale()) { return; }
|
|
1696
|
-
|
|
1697
|
-
// Invalidate closure cache entries that may be affected by this message.
|
|
1698
|
-
// Must run before closure validation so subsequent evaluations in the
|
|
1699
|
-
// same session see the updated local state.
|
|
1700
|
-
const closureCtxForInvalidation = this._closureContexts.get(did);
|
|
1701
|
-
if (closureCtxForInvalidation) {
|
|
1702
|
-
invalidateClosureCache(closureCtxForInvalidation, event.message);
|
|
1703
|
-
}
|
|
1704
|
-
|
|
1705
|
-
// Closure validation for scoped subset sync (Phase 3).
|
|
1706
|
-
// For protocol-scoped links, verify that all hard dependencies for
|
|
1707
|
-
// this operation are locally present before considering it committed.
|
|
1708
|
-
// Full-tenant scope bypasses this entirely (returns complete with 0 queries).
|
|
1709
|
-
if (link?.scope.kind === 'protocol') {
|
|
1710
|
-
const messageStore = this.agent.dwn.node.storage.messageStore;
|
|
1711
|
-
let closureCtx = this._closureContexts.get(did);
|
|
1712
|
-
if (!closureCtx) {
|
|
1713
|
-
closureCtx = createClosureContext(did, undefined, {
|
|
1714
|
-
isDelegateSession: !!delegateDid,
|
|
1715
|
-
});
|
|
1716
|
-
this._closureContexts.set(did, closureCtx);
|
|
1717
|
-
}
|
|
1718
|
-
|
|
1719
|
-
const closureResult = await evaluateClosure(
|
|
1720
|
-
event.message, messageStore, link.scope, closureCtx
|
|
1721
|
-
);
|
|
1722
|
-
|
|
1723
|
-
if (isStale()) { return; }
|
|
1724
|
-
|
|
1725
|
-
if (!closureResult.complete) {
|
|
1726
|
-
const failureCode = closureResult.failure!.code;
|
|
1727
|
-
const failureDetail = closureResult.failure!.detail;
|
|
1728
|
-
console.warn(
|
|
1729
|
-
`SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
|
|
1730
|
-
`${failureCode} — ${failureDetail}`
|
|
1731
|
-
);
|
|
1732
|
-
|
|
1733
|
-
// Record the message that triggered the closure failure.
|
|
1734
|
-
const closureCid = await Message.getCid(event.message);
|
|
1735
|
-
void this.recordDeadLetter({
|
|
1736
|
-
messageCid : closureCid,
|
|
1737
|
-
tenantDid : did,
|
|
1738
|
-
remoteEndpoint : dwnUrl,
|
|
1739
|
-
protocol,
|
|
1740
|
-
category : 'closure',
|
|
1741
|
-
errorCode : failureCode,
|
|
1742
|
-
errorDetail : failureDetail,
|
|
1743
|
-
});
|
|
1744
|
-
|
|
1745
|
-
if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
|
|
1746
|
-
return;
|
|
1747
|
-
}
|
|
1748
|
-
}
|
|
1749
|
-
|
|
1750
|
-
// Squash convergence: processRawMessage triggers the DWN's built-in
|
|
1751
|
-
// squash resumable task (performRecordsSquash) which runs inline and
|
|
1752
|
-
// handles subset consumers correctly:
|
|
1753
|
-
// - If older siblings are locally present → purges them
|
|
1754
|
-
// - If squash arrives before older siblings → backstop rejects them (409)
|
|
1755
|
-
// - If no older siblings are local → no-op (correct)
|
|
1756
|
-
// Both sync orderings (squash-first or siblings-first) converge to
|
|
1757
|
-
// the same final state. No additional sync-engine side-effect is needed.
|
|
1758
|
-
|
|
1759
|
-
// Track this CID for echo-loop suppression, scoped to the source endpoint.
|
|
1760
|
-
const pulledCid = await Message.getCid(event.message);
|
|
1761
|
-
this._recentlyPulledCids.set(`${pulledCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
|
|
1762
|
-
this.evictExpiredEchoEntries();
|
|
1763
|
-
|
|
1764
|
-
// Auto-clear any dead letter for this CID — it was processed
|
|
1765
|
-
// successfully, so a previous failure has been self-healed.
|
|
1766
|
-
this.clearFailedMessage(pulledCid, dwnUrl).catch(() => { /* teardown race */ });
|
|
1767
|
-
|
|
1768
|
-
// Mark this ordinal as committed and drain the checkpoint.
|
|
1769
|
-
// Guard: if the link transitioned to repairing while this handler was
|
|
1770
|
-
// in-flight (e.g., an earlier ordinal's handler failed concurrently),
|
|
1771
|
-
// skip all state mutations — the repair process owns progression now.
|
|
1772
|
-
if (link && rt && link.status === 'live' && !isStale()) {
|
|
1773
|
-
const entry = rt.inflight.get(ordinal);
|
|
1774
|
-
if (entry) { entry.committed = true; }
|
|
1775
|
-
|
|
1776
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1777
|
-
const drained = this.drainCommittedPull(cursorKey);
|
|
1778
|
-
if (drained > 0) {
|
|
1779
|
-
await this.ledger.saveLink(link);
|
|
1780
|
-
// Emit after durable save — "advanced" means persisted.
|
|
1781
|
-
if (link.pull.contiguousAppliedToken) {
|
|
1782
|
-
this.emitEvent({
|
|
1783
|
-
type : 'checkpoint:pull-advance',
|
|
1784
|
-
tenantDid : link.tenantDid,
|
|
1785
|
-
remoteEndpoint : link.remoteEndpoint,
|
|
1786
|
-
protocol : link.protocol,
|
|
1787
|
-
position : link.pull.contiguousAppliedToken.position,
|
|
1788
|
-
messageCid : link.pull.contiguousAppliedToken.messageCid,
|
|
1789
|
-
});
|
|
1790
|
-
}
|
|
1791
|
-
}
|
|
1792
|
-
|
|
1793
|
-
// Overflow: too many in-flight ordinals without draining.
|
|
1794
|
-
if (rt.inflight.size > MAX_PENDING_TOKENS) {
|
|
1795
|
-
console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1796
|
-
await this.transitionToRepairing(cursorKey, link);
|
|
1797
|
-
}
|
|
1798
|
-
}
|
|
1799
|
-
} catch (error: any) {
|
|
1800
|
-
console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
|
|
1801
|
-
|
|
1802
|
-
// Record the failing message in the dead letter store before
|
|
1803
|
-
// transitioning to repair. The CID identifies which specific
|
|
1804
|
-
// message caused the transition.
|
|
1805
|
-
try {
|
|
1806
|
-
const failedCid = await Message.getCid(event.message);
|
|
1807
|
-
void this.recordDeadLetter({
|
|
1808
|
-
messageCid : failedCid,
|
|
1809
|
-
tenantDid : did,
|
|
1810
|
-
remoteEndpoint : dwnUrl,
|
|
1811
|
-
protocol,
|
|
1812
|
-
category : 'pull-processing',
|
|
1813
|
-
errorDetail : error.message ?? String(error),
|
|
1814
|
-
});
|
|
1815
|
-
} catch {
|
|
1816
|
-
// Best effort — don't let dead letter recording block repair.
|
|
1817
|
-
}
|
|
1818
|
-
|
|
1819
|
-
// A failed processRawMessage means local state is incomplete.
|
|
1820
|
-
// Transition to repairing immediately — do NOT advance the checkpoint
|
|
1821
|
-
// past this failure or let later ordinals commit past it. SMT
|
|
1822
|
-
// reconciliation will discover and fill the gap.
|
|
1823
|
-
if (link && !isStale()) {
|
|
1824
|
-
await this.transitionToRepairing(cursorKey, link);
|
|
1825
|
-
}
|
|
1826
|
-
}
|
|
1827
|
-
}
|
|
1877
|
+
await this.handleLivePullMessage(pullContext, subMessage);
|
|
1828
1878
|
};
|
|
1829
1879
|
|
|
1830
1880
|
// Construct the subscribe message and send it directly to the specific
|
|
@@ -1837,7 +1887,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1837
1887
|
target : did,
|
|
1838
1888
|
messageType : DwnInterface.MessagesSubscribe as const,
|
|
1839
1889
|
granteeDid : delegateDid,
|
|
1840
|
-
messageParams : { filters, cursor,
|
|
1890
|
+
messageParams : { filters, cursor, permissionGrantIds: toMessagesPermissionGrantIds(target.permissionGrantIds) },
|
|
1841
1891
|
};
|
|
1842
1892
|
|
|
1843
1893
|
const { message } = await this.agent.dwn.processRequest(subscribeRequest);
|
|
@@ -1890,13 +1940,14 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1890
1940
|
throw new Error(`SyncEngineLevel: MessagesSubscribe failed for ${did} -> ${dwnUrl}: ${reply.status.code} ${reply.status.detail}`);
|
|
1891
1941
|
}
|
|
1892
1942
|
|
|
1943
|
+
const linkKey = cursorKey;
|
|
1944
|
+
const close = async (): Promise<void> => { await reply.subscription!.close(); };
|
|
1893
1945
|
this._liveSubscriptions.push({
|
|
1894
|
-
linkKey
|
|
1946
|
+
linkKey,
|
|
1895
1947
|
did,
|
|
1896
1948
|
dwnUrl,
|
|
1897
1949
|
delegateDid,
|
|
1898
|
-
|
|
1899
|
-
close : async (): Promise<void> => { await reply.subscription!.close(); },
|
|
1950
|
+
close,
|
|
1900
1951
|
});
|
|
1901
1952
|
|
|
1902
1953
|
// Set per-link connectivity to online after successful subscription setup.
|
|
@@ -1905,9 +1956,399 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1905
1956
|
const prevPullConnectivity = pullLink.connectivity;
|
|
1906
1957
|
pullLink.connectivity = 'online';
|
|
1907
1958
|
if (prevPullConnectivity !== 'online') {
|
|
1908
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1959
|
+
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: prevPullConnectivity, to: 'online' });
|
|
1960
|
+
}
|
|
1961
|
+
}
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1964
|
+
private async getInitialPullCursor({ did, dwnUrl, link }: {
|
|
1965
|
+
did: string;
|
|
1966
|
+
dwnUrl: string;
|
|
1967
|
+
link?: ReplicationLinkState;
|
|
1968
|
+
}): Promise<ProgressToken | undefined> {
|
|
1969
|
+
// Resolve the cursor from the link's durable pull checkpoint.
|
|
1970
|
+
if (!link) {
|
|
1971
|
+
return undefined;
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
const cursor = link.pull.contiguousAppliedToken;
|
|
1975
|
+
if (!cursor || this.isValidProgressToken(cursor)) {
|
|
1976
|
+
return cursor;
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
// Guard against corrupted tokens with empty fields — these would fail
|
|
1980
|
+
// MessagesSubscribe JSON schema validation (minLength: 1). Discard and
|
|
1981
|
+
// start from the beginning rather than crash the subscription.
|
|
1982
|
+
console.warn(`SyncEngineLevel: Discarding stored cursor with empty field(s) for ${did} -> ${dwnUrl}`);
|
|
1983
|
+
ReplicationLedger.resetCheckpoint(link.pull);
|
|
1984
|
+
await this.ledger.saveLink(link);
|
|
1985
|
+
return undefined;
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
private isValidProgressToken(token: ProgressToken): boolean {
|
|
1989
|
+
return !!(token.streamId && token.messageCid && token.epoch && token.position);
|
|
1990
|
+
}
|
|
1991
|
+
|
|
1992
|
+
private createLinkStalePredicate(
|
|
1993
|
+
linkKey: string,
|
|
1994
|
+
capturedLink: ReplicationLinkState | undefined,
|
|
1995
|
+
generation: number,
|
|
1996
|
+
): () => boolean {
|
|
1997
|
+
return (): boolean =>
|
|
1998
|
+
this._engineGeneration !== generation ||
|
|
1999
|
+
!this._activeLinks.has(linkKey) ||
|
|
2000
|
+
(capturedLink !== undefined && this._activeLinks.get(linkKey) !== capturedLink);
|
|
2001
|
+
}
|
|
2002
|
+
|
|
2003
|
+
private async handleLivePullMessage(context: LivePullContext, subMessage: SubscriptionMessage): Promise<void> {
|
|
2004
|
+
if (context.isStale()) {
|
|
2005
|
+
return;
|
|
2006
|
+
}
|
|
2007
|
+
|
|
2008
|
+
if (subMessage.type === 'eose') {
|
|
2009
|
+
await this.handleLivePullEose(context, subMessage);
|
|
2010
|
+
return;
|
|
2011
|
+
}
|
|
2012
|
+
|
|
2013
|
+
if (subMessage.type === 'error') {
|
|
2014
|
+
await this.handleLivePullSubscriptionError(context, subMessage);
|
|
2015
|
+
return;
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
if (subMessage.type === 'event') {
|
|
2019
|
+
await this.handleLivePullEvent(context, subMessage);
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
|
|
2023
|
+
private async handleLivePullEose(
|
|
2024
|
+
{ did, dwnUrl, eventScope, linkKey, link, isStale }: LivePullContext,
|
|
2025
|
+
subMessage: Extract<SubscriptionMessage, { type: 'eose' }>,
|
|
2026
|
+
): Promise<void> {
|
|
2027
|
+
if (link) {
|
|
2028
|
+
// Guard: if the link transitioned to repairing while catch-up events
|
|
2029
|
+
// were being processed, skip all mutations — repair owns the state now.
|
|
2030
|
+
if (link.status !== 'live' && link.status !== 'initializing') {
|
|
2031
|
+
return;
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
2035
|
+
console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
2036
|
+
if (!isStale()) { await this.transitionToRepairing(linkKey, link); }
|
|
2037
|
+
return;
|
|
2038
|
+
}
|
|
2039
|
+
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
2040
|
+
this.drainCommittedPull(linkKey);
|
|
2041
|
+
if (isStale()) { return; }
|
|
2042
|
+
await this.ledger.saveLink(link);
|
|
2043
|
+
}
|
|
2044
|
+
|
|
2045
|
+
this.markPullLinkOnline({ did, dwnUrl, eventScope, linkKey, link });
|
|
2046
|
+
}
|
|
2047
|
+
|
|
2048
|
+
private markPullLinkOnline({ did, dwnUrl, eventScope, linkKey, link }: {
|
|
2049
|
+
did: string;
|
|
2050
|
+
dwnUrl: string;
|
|
2051
|
+
eventScope: SyncEventScope;
|
|
2052
|
+
linkKey: string;
|
|
2053
|
+
link?: ReplicationLinkState;
|
|
2054
|
+
}): void {
|
|
2055
|
+
if (!link) {
|
|
2056
|
+
this._connectivityState = 'online';
|
|
2057
|
+
return;
|
|
2058
|
+
}
|
|
2059
|
+
|
|
2060
|
+
const previous = link.connectivity;
|
|
2061
|
+
link.connectivity = 'online';
|
|
2062
|
+
if (previous !== 'online') {
|
|
2063
|
+
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: previous, to: 'online' });
|
|
2064
|
+
}
|
|
2065
|
+
if (link.needsReconcile) {
|
|
2066
|
+
this.scheduleReconcile(linkKey, 500);
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
private async handleLivePullSubscriptionError(
|
|
2071
|
+
{ did, dwnUrl, linkKey, link, isStale }: LivePullContext,
|
|
2072
|
+
subMessage: Extract<SubscriptionMessage, { type: 'error' }>,
|
|
2073
|
+
): Promise<void> {
|
|
2074
|
+
console.warn(`SyncEngineLevel: subscription error for ${did} -> ${dwnUrl}: ${subMessage.error.code}`);
|
|
2075
|
+
|
|
2076
|
+
if (link && !isStale()) {
|
|
2077
|
+
await this.transitionToRepairing(linkKey, link);
|
|
2078
|
+
}
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
private async handleLivePullEvent(
|
|
2082
|
+
context: LivePullContext,
|
|
2083
|
+
subMessage: Extract<SubscriptionMessage, { type: 'event' }>,
|
|
2084
|
+
): Promise<void> {
|
|
2085
|
+
const event = subMessage.event;
|
|
2086
|
+
if (await this.shouldSkipLivePullEvent(context, subMessage)) {
|
|
2087
|
+
return;
|
|
2088
|
+
}
|
|
2089
|
+
|
|
2090
|
+
const delivery = this.startPullDelivery(context, subMessage.cursor);
|
|
2091
|
+
try {
|
|
2092
|
+
const pulledCid = await this.processLivePullEvent(context, event);
|
|
2093
|
+
if (!pulledCid) { return; }
|
|
2094
|
+
|
|
2095
|
+
this.trackRecentlyPulledMessage(pulledCid, context.dwnUrl);
|
|
2096
|
+
this.clearFailedMessage(pulledCid, context.dwnUrl).catch(() => { /* teardown race */ });
|
|
2097
|
+
await this.commitPullDelivery(context, subMessage.cursor, delivery);
|
|
2098
|
+
} catch (error: any) {
|
|
2099
|
+
await this.handleLivePullProcessingError(context, event, error);
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
2102
|
+
|
|
2103
|
+
private async shouldSkipLivePullEvent(
|
|
2104
|
+
{ did, dwnUrl, linkKey, link, isStale }: LivePullContext,
|
|
2105
|
+
subMessage: Extract<SubscriptionMessage, { type: 'event' }>,
|
|
2106
|
+
): Promise<boolean> {
|
|
2107
|
+
// Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
|
|
2108
|
+
// skip all processing. Old subscription handlers may still fire after the
|
|
2109
|
+
// link transitions — these events should be ignored entirely, not just
|
|
2110
|
+
// skipped at the checkpoint level.
|
|
2111
|
+
if (link && link.status !== 'live' && link.status !== 'initializing') {
|
|
2112
|
+
return true;
|
|
2113
|
+
}
|
|
2114
|
+
|
|
2115
|
+
// Domain validation: reject tokens from a different stream/epoch.
|
|
2116
|
+
if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
2117
|
+
console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
2118
|
+
if (!isStale()) { await this.transitionToRepairing(linkKey, link); }
|
|
2119
|
+
return true;
|
|
2120
|
+
}
|
|
2121
|
+
|
|
2122
|
+
if (link) {
|
|
2123
|
+
const scopeClassification = classifySyncEventScope(subMessage.event, link.scope);
|
|
2124
|
+
if (scopeClassification === 'out-of-scope') {
|
|
2125
|
+
await this.skipOutOfScopePullEvent({ link, cursor: subMessage.cursor, isStale });
|
|
2126
|
+
return true;
|
|
2127
|
+
}
|
|
2128
|
+
if (scopeClassification === 'unknown') {
|
|
2129
|
+
console.warn(`SyncEngineLevel: Unable to classify scoped pull event for ${did} -> ${dwnUrl}, transitioning to repair`);
|
|
2130
|
+
if (!isStale()) { await this.transitionToRepairing(linkKey, link); }
|
|
2131
|
+
return true;
|
|
1909
2132
|
}
|
|
1910
2133
|
}
|
|
2134
|
+
|
|
2135
|
+
return false;
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
private async skipOutOfScopePullEvent({ link, cursor, isStale }: {
|
|
2139
|
+
link: ReplicationLinkState;
|
|
2140
|
+
cursor: ProgressToken;
|
|
2141
|
+
isStale: () => boolean;
|
|
2142
|
+
}): Promise<void> {
|
|
2143
|
+
// Skipped events MUST advance contiguousAppliedToken — otherwise the link
|
|
2144
|
+
// would replay the same filtered-out events indefinitely after reconnect or
|
|
2145
|
+
// repair. This is safe because the event is intentionally excluded from
|
|
2146
|
+
// this scope and doesn't need processing.
|
|
2147
|
+
if (isStale()) { return; }
|
|
2148
|
+
|
|
2149
|
+
ReplicationLedger.setReceivedToken(link.pull, cursor);
|
|
2150
|
+
ReplicationLedger.commitContiguousToken(link.pull, cursor);
|
|
2151
|
+
await this.ledger.saveLink(link);
|
|
2152
|
+
}
|
|
2153
|
+
|
|
2154
|
+
private startPullDelivery({ linkKey, link }: LivePullContext, cursor: ProgressToken): PullDelivery {
|
|
2155
|
+
// Assign a delivery ordinal BEFORE async processing begins. This captures
|
|
2156
|
+
// delivery order even if processing completes out of order.
|
|
2157
|
+
const runtime = link ? this.getOrCreateRuntime(linkKey) : undefined;
|
|
2158
|
+
const ordinal = runtime ? runtime.nextDeliveryOrdinal++ : -1;
|
|
2159
|
+
if (runtime) {
|
|
2160
|
+
runtime.inflight.set(ordinal, { ordinal, token: cursor, committed: false });
|
|
2161
|
+
}
|
|
2162
|
+
return { runtime, ordinal };
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
private async processLivePullEvent(context: LivePullContext, event: MessageEvent): Promise<string | undefined> {
|
|
2166
|
+
const dataStream = await this.getLivePullDataStream(context, event);
|
|
2167
|
+
await this.agent.dwn.processRawMessage(context.did, event.message, { dataStream });
|
|
2168
|
+
if (context.isStale()) { return undefined; }
|
|
2169
|
+
|
|
2170
|
+
this.invalidateClosureCacheForMessage(context.did, event.message);
|
|
2171
|
+
if (!await this.ensureClosureComplete(context, event)) {
|
|
2172
|
+
return undefined;
|
|
2173
|
+
}
|
|
2174
|
+
|
|
2175
|
+
// Squash convergence: processRawMessage triggers the DWN's built-in
|
|
2176
|
+
// squash resumable task (performRecordsSquash), so no additional
|
|
2177
|
+
// sync-engine side effect is needed here.
|
|
2178
|
+
return Message.getCid(event.message);
|
|
2179
|
+
}
|
|
2180
|
+
|
|
2181
|
+
private async getLivePullDataStream(
|
|
2182
|
+
{ did, dwnUrl, delegateDid, permissionGrantIds }: LivePullContext,
|
|
2183
|
+
event: MessageEvent,
|
|
2184
|
+
): Promise<ReadableStream<Uint8Array> | undefined> {
|
|
2185
|
+
const inlineData = this.extractDataStream(event);
|
|
2186
|
+
if (inlineData || !isRecordsWrite(event) || !(event.message.descriptor as any).dataCid) {
|
|
2187
|
+
return inlineData;
|
|
2188
|
+
}
|
|
2189
|
+
|
|
2190
|
+
// For large RecordsWrite messages (no inline data), fetch the data from
|
|
2191
|
+
// the remote DWN via MessagesRead before storing locally.
|
|
2192
|
+
const messageCid = await Message.getCid(event.message);
|
|
2193
|
+
const fetched = await fetchRemoteMessages({
|
|
2194
|
+
did,
|
|
2195
|
+
dwnUrl,
|
|
2196
|
+
delegateDid,
|
|
2197
|
+
permissionGrantIds,
|
|
2198
|
+
messageCids : [messageCid],
|
|
2199
|
+
agent : this.agent,
|
|
2200
|
+
});
|
|
2201
|
+
return fetched[0]?.dataStream;
|
|
2202
|
+
}
|
|
2203
|
+
|
|
2204
|
+
private invalidateClosureCacheForMessage(did: string, message: GenericMessage): void {
|
|
2205
|
+
// Must run before closure validation so subsequent evaluations in the same
|
|
2206
|
+
// session see the updated local state.
|
|
2207
|
+
const closureCtx = this._closureContexts.get(did);
|
|
2208
|
+
if (closureCtx) {
|
|
2209
|
+
invalidateClosureCache(closureCtx, message);
|
|
2210
|
+
}
|
|
2211
|
+
}
|
|
2212
|
+
|
|
2213
|
+
private async ensureClosureComplete(context: LivePullContext, event: MessageEvent): Promise<boolean> {
|
|
2214
|
+
const { did, delegateDid, link, isStale } = context;
|
|
2215
|
+
if (!link || link.scope.kind === 'full') {
|
|
2216
|
+
return true;
|
|
2217
|
+
}
|
|
2218
|
+
|
|
2219
|
+
let closureCtx = this._closureContexts.get(did);
|
|
2220
|
+
if (!closureCtx) {
|
|
2221
|
+
closureCtx = createClosureContext(did, undefined, {
|
|
2222
|
+
isDelegateSession: !!delegateDid,
|
|
2223
|
+
});
|
|
2224
|
+
this._closureContexts.set(did, closureCtx);
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
const messageStore = this.agent.dwn.node.storage.messageStore;
|
|
2228
|
+
const closureResult = await evaluateClosure(event.message, messageStore, link.scope, closureCtx);
|
|
2229
|
+
if (isStale()) { return false; }
|
|
2230
|
+
if (closureResult.complete) { return true; }
|
|
2231
|
+
|
|
2232
|
+
await this.recordClosureFailure(context, event, closureResult.failure!.code, closureResult.failure!.detail);
|
|
2233
|
+
return false;
|
|
2234
|
+
}
|
|
2235
|
+
|
|
2236
|
+
private async recordClosureFailure(
|
|
2237
|
+
{ did, dwnUrl, linkKey, link, isStale }: LivePullContext,
|
|
2238
|
+
event: MessageEvent,
|
|
2239
|
+
failureCode: string,
|
|
2240
|
+
failureDetail: string,
|
|
2241
|
+
): Promise<void> {
|
|
2242
|
+
console.warn(
|
|
2243
|
+
`SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
|
|
2244
|
+
`${failureCode} — ${failureDetail}`
|
|
2245
|
+
);
|
|
2246
|
+
|
|
2247
|
+
const closureCid = await Message.getCid(event.message);
|
|
2248
|
+
void this.recordDeadLetter({
|
|
2249
|
+
messageCid : closureCid,
|
|
2250
|
+
tenantDid : did,
|
|
2251
|
+
remoteEndpoint : dwnUrl,
|
|
2252
|
+
protocol : (event.message.descriptor as Record<string, unknown>).protocol as string | undefined,
|
|
2253
|
+
category : 'closure',
|
|
2254
|
+
errorCode : failureCode,
|
|
2255
|
+
errorDetail : failureDetail,
|
|
2256
|
+
});
|
|
2257
|
+
|
|
2258
|
+
if (link && !isStale() && isTerminalClosureFailureCode(failureCode)) {
|
|
2259
|
+
await this.transitionToTerminalIncomplete(linkKey, link);
|
|
2260
|
+
return;
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
if (link && !isStale()) {
|
|
2264
|
+
await this.transitionToRepairing(linkKey, link);
|
|
2265
|
+
}
|
|
2266
|
+
}
|
|
2267
|
+
|
|
2268
|
+
private trackRecentlyPulledMessage(messageCid: string, dwnUrl: string): void {
|
|
2269
|
+
this._recentlyPulledCids.set(`${messageCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
|
|
2270
|
+
this.evictExpiredEchoEntries();
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2273
|
+
private async commitPullDelivery(
|
|
2274
|
+
{ did, dwnUrl, linkKey, link, isStale }: LivePullContext,
|
|
2275
|
+
cursor: ProgressToken,
|
|
2276
|
+
delivery: PullDelivery,
|
|
2277
|
+
): Promise<void> {
|
|
2278
|
+
// Guard: if the link transitioned to repairing while this handler was
|
|
2279
|
+
// in-flight, skip all state mutations — the repair process owns progression.
|
|
2280
|
+
if (!link || !delivery.runtime || link.status !== 'live' || isStale()) {
|
|
2281
|
+
return;
|
|
2282
|
+
}
|
|
2283
|
+
|
|
2284
|
+
const entry = delivery.runtime.inflight.get(delivery.ordinal);
|
|
2285
|
+
if (entry) { entry.committed = true; }
|
|
2286
|
+
|
|
2287
|
+
ReplicationLedger.setReceivedToken(link.pull, cursor);
|
|
2288
|
+
const drained = this.drainCommittedPull(linkKey);
|
|
2289
|
+
if (drained > 0) {
|
|
2290
|
+
await this.ledger.saveLink(link);
|
|
2291
|
+
this.emitPullCheckpointAdvance(link);
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
if (delivery.runtime.inflight.size > MAX_PENDING_TOKENS) {
|
|
2295
|
+
console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
2296
|
+
await this.transitionToRepairing(linkKey, link);
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
|
|
2300
|
+
private emitPullCheckpointAdvance(link: ReplicationLinkState): void {
|
|
2301
|
+
if (!link.pull.contiguousAppliedToken) {
|
|
2302
|
+
return;
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
// Emit after durable save — "advanced" means persisted.
|
|
2306
|
+
this.emitEvent({
|
|
2307
|
+
type : 'checkpoint:pull-advance',
|
|
2308
|
+
tenantDid : link.tenantDid,
|
|
2309
|
+
remoteEndpoint : link.remoteEndpoint,
|
|
2310
|
+
...syncEventScope(link.scope),
|
|
2311
|
+
position : link.pull.contiguousAppliedToken.position,
|
|
2312
|
+
messageCid : link.pull.contiguousAppliedToken.messageCid,
|
|
2313
|
+
});
|
|
2314
|
+
}
|
|
2315
|
+
|
|
2316
|
+
private async handleLivePullProcessingError(
|
|
2317
|
+
{ did, dwnUrl, linkKey, link, isStale }: LivePullContext,
|
|
2318
|
+
event: MessageEvent,
|
|
2319
|
+
error: any,
|
|
2320
|
+
): Promise<void> {
|
|
2321
|
+
console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
|
|
2322
|
+
await this.recordPullProcessingFailure({ did, dwnUrl, event, error });
|
|
2323
|
+
|
|
2324
|
+
// A failed processRawMessage means local state is incomplete. Transition
|
|
2325
|
+
// to repairing immediately — do NOT advance the checkpoint past this
|
|
2326
|
+
// failure or let later ordinals commit past it. SMT reconciliation will
|
|
2327
|
+
// discover and fill the gap.
|
|
2328
|
+
if (link && !isStale()) {
|
|
2329
|
+
await this.transitionToRepairing(linkKey, link);
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
|
|
2333
|
+
private async recordPullProcessingFailure({ did, dwnUrl, event, error }: {
|
|
2334
|
+
did: string;
|
|
2335
|
+
dwnUrl: string;
|
|
2336
|
+
event: MessageEvent;
|
|
2337
|
+
error: any;
|
|
2338
|
+
}): Promise<void> {
|
|
2339
|
+
try {
|
|
2340
|
+
const failedCid = await Message.getCid(event.message);
|
|
2341
|
+
void this.recordDeadLetter({
|
|
2342
|
+
messageCid : failedCid,
|
|
2343
|
+
tenantDid : did,
|
|
2344
|
+
remoteEndpoint : dwnUrl,
|
|
2345
|
+
protocol : (event.message.descriptor as Record<string, unknown>).protocol as string | undefined,
|
|
2346
|
+
category : 'pull-processing',
|
|
2347
|
+
errorDetail : error.message ?? String(error),
|
|
2348
|
+
});
|
|
2349
|
+
} catch {
|
|
2350
|
+
// Best effort — don't let dead letter recording block repair.
|
|
2351
|
+
}
|
|
1911
2352
|
}
|
|
1912
2353
|
|
|
1913
2354
|
// ---------------------------------------------------------------------------
|
|
@@ -1918,27 +2359,13 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1918
2359
|
* Subscribes to the local DWN's EventLog so that writes by the user are
|
|
1919
2360
|
* immediately pushed to the remote DWN instead of waiting for the next poll.
|
|
1920
2361
|
*/
|
|
1921
|
-
private async openLocalPushSubscription(target: {
|
|
1922
|
-
did
|
|
1923
|
-
|
|
1924
|
-
}): Promise<void> {
|
|
1925
|
-
const { did, delegateDid, dwnUrl, protocol } = target;
|
|
1926
|
-
|
|
1927
|
-
// Build filters scoped to the protocol (if any).
|
|
1928
|
-
const filters = protocol ? [{ protocol }] : [];
|
|
2362
|
+
private async openLocalPushSubscription(target: LinkSyncTarget): Promise<void> {
|
|
2363
|
+
const { did, delegateDid, dwnUrl } = target;
|
|
2364
|
+
const protocol = singleProtocolForSyncScope(target.scope);
|
|
1929
2365
|
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
const grant = await this._permissionsApi.getPermissionForRequest({
|
|
1934
|
-
connectedDid : did,
|
|
1935
|
-
messageType : DwnInterface.MessagesSubscribe,
|
|
1936
|
-
delegateDid,
|
|
1937
|
-
protocol,
|
|
1938
|
-
cached : true,
|
|
1939
|
-
});
|
|
1940
|
-
permissionGrantId = grant.grant.id;
|
|
1941
|
-
}
|
|
2366
|
+
const filters = target.scope.kind === 'protocolSet'
|
|
2367
|
+
? target.scope.protocols.map(protocol => ({ protocol }))
|
|
2368
|
+
: [];
|
|
1942
2369
|
|
|
1943
2370
|
const handlerGeneration = this._engineGeneration;
|
|
1944
2371
|
|
|
@@ -1959,12 +2386,19 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1959
2386
|
return;
|
|
1960
2387
|
}
|
|
1961
2388
|
|
|
1962
|
-
// Subset scope filtering: only push events that match the link
|
|
1963
|
-
//
|
|
2389
|
+
// Subset scope filtering: only push events that match the link scope.
|
|
2390
|
+
// Events outside the scope are not this link's responsibility.
|
|
1964
2391
|
const pushLinkKey = target.linkKey;
|
|
1965
2392
|
const pushLink = this._activeLinks.get(pushLinkKey);
|
|
1966
|
-
if (pushLink
|
|
1967
|
-
|
|
2393
|
+
if (pushLink) {
|
|
2394
|
+
const scopeClassification = classifySyncEventScope(subMessage.event, pushLink.scope);
|
|
2395
|
+
if (scopeClassification === 'out-of-scope') {
|
|
2396
|
+
return;
|
|
2397
|
+
}
|
|
2398
|
+
if (scopeClassification === 'unknown') {
|
|
2399
|
+
this.markLinkNeedsReconcile(pushLinkKey, pushLink, 'push-scope-unclassified');
|
|
2400
|
+
return;
|
|
2401
|
+
}
|
|
1968
2402
|
}
|
|
1969
2403
|
|
|
1970
2404
|
// Accumulate the message CID for a debounced push.
|
|
@@ -1982,7 +2416,11 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
1982
2416
|
}
|
|
1983
2417
|
|
|
1984
2418
|
const pushRuntime = this.getOrCreatePushRuntime(targetKey, {
|
|
1985
|
-
did,
|
|
2419
|
+
did,
|
|
2420
|
+
dwnUrl,
|
|
2421
|
+
delegateDid,
|
|
2422
|
+
protocol,
|
|
2423
|
+
permissionGrantIds: target.permissionGrantIds,
|
|
1986
2424
|
});
|
|
1987
2425
|
pushRuntime.entries.push({ cid });
|
|
1988
2426
|
|
|
@@ -2002,7 +2440,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2002
2440
|
target : did,
|
|
2003
2441
|
messageType : DwnInterface.MessagesSubscribe,
|
|
2004
2442
|
granteeDid : delegateDid,
|
|
2005
|
-
messageParams : { filters,
|
|
2443
|
+
messageParams : { filters, permissionGrantIds: toMessagesPermissionGrantIds(target.permissionGrantIds) },
|
|
2006
2444
|
subscriptionHandler : subscriptionHandler as any,
|
|
2007
2445
|
});
|
|
2008
2446
|
|
|
@@ -2011,13 +2449,13 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2011
2449
|
throw new Error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
|
|
2012
2450
|
}
|
|
2013
2451
|
|
|
2452
|
+
const close = async (): Promise<void> => { await reply.subscription!.close(); };
|
|
2014
2453
|
this._localSubscriptions.push({
|
|
2015
|
-
linkKey
|
|
2454
|
+
linkKey: target.linkKey,
|
|
2016
2455
|
did,
|
|
2017
2456
|
dwnUrl,
|
|
2018
2457
|
delegateDid,
|
|
2019
|
-
|
|
2020
|
-
close : async (): Promise<void> => { await reply.subscription!.close(); },
|
|
2458
|
+
close,
|
|
2021
2459
|
});
|
|
2022
2460
|
}
|
|
2023
2461
|
|
|
@@ -2031,112 +2469,165 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2031
2469
|
}
|
|
2032
2470
|
|
|
2033
2471
|
private async flushPendingPushesForLink(linkKey: string): Promise<void> {
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
// was removed.
|
|
2037
|
-
if (!this._activeLinks.has(linkKey)) {
|
|
2038
|
-
return;
|
|
2039
|
-
}
|
|
2040
|
-
|
|
2041
|
-
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
2042
|
-
if (!pushRuntime) {
|
|
2043
|
-
return;
|
|
2044
|
-
}
|
|
2472
|
+
const batch = this.takePushFlushBatch(linkKey);
|
|
2473
|
+
if (!batch) { return; }
|
|
2045
2474
|
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
const flushLink = this._activeLinks.get(linkKey);
|
|
2049
|
-
const isFlushStale = (): boolean =>
|
|
2050
|
-
!this._activeLinks.has(linkKey) ||
|
|
2051
|
-
(flushLink !== undefined && this._activeLinks.get(linkKey) !== flushLink);
|
|
2052
|
-
|
|
2053
|
-
const { did, dwnUrl, delegateDid, protocol, entries: pushEntries, retryCount } = pushRuntime;
|
|
2054
|
-
pushRuntime.entries = [];
|
|
2055
|
-
|
|
2056
|
-
if (pushEntries.length === 0) {
|
|
2057
|
-
if (!pushRuntime.timer && !pushRuntime.flushing && retryCount === 0) {
|
|
2058
|
-
this._pushRuntimes.delete(linkKey);
|
|
2059
|
-
}
|
|
2060
|
-
return;
|
|
2061
|
-
}
|
|
2062
|
-
|
|
2063
|
-
const cids = pushEntries.map((entry) => entry.cid);
|
|
2064
|
-
pushRuntime.flushing = true;
|
|
2475
|
+
const { pushRuntime, pushEntries, isStale } = batch;
|
|
2476
|
+
const { did, dwnUrl, delegateDid, protocol, permissionGrantIds, retryCount } = pushRuntime;
|
|
2065
2477
|
|
|
2066
2478
|
try {
|
|
2067
2479
|
const result = await pushMessages({
|
|
2068
|
-
did,
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2480
|
+
did,
|
|
2481
|
+
dwnUrl,
|
|
2482
|
+
delegateDid,
|
|
2483
|
+
permissionGrantIds,
|
|
2484
|
+
messageCids : pushEntries.map((entry) => entry.cid),
|
|
2485
|
+
agent : this.agent,
|
|
2072
2486
|
});
|
|
2073
2487
|
|
|
2074
|
-
|
|
2075
|
-
// post-push state mutations — the replacement session owns this key.
|
|
2076
|
-
if (isFlushStale()) { return; }
|
|
2077
|
-
|
|
2078
|
-
// Auto-clear dead letters for CIDs that succeeded — a previously
|
|
2079
|
-
// failed message may have been repaired by reconciliation.
|
|
2080
|
-
for (const cid of result.succeeded) {
|
|
2081
|
-
this.clearFailedMessage(cid, dwnUrl).catch(() => { /* teardown race */ });
|
|
2082
|
-
}
|
|
2083
|
-
|
|
2084
|
-
// Record permanently failed messages in the dead letter store.
|
|
2085
|
-
for (const entry of result.permanentlyFailed) {
|
|
2086
|
-
await this.recordDeadLetter({
|
|
2087
|
-
messageCid : entry.cid,
|
|
2088
|
-
tenantDid : did,
|
|
2089
|
-
remoteEndpoint : dwnUrl,
|
|
2090
|
-
protocol,
|
|
2091
|
-
category : 'push-permanent',
|
|
2092
|
-
errorCode : String(entry.statusCode ?? ''),
|
|
2093
|
-
errorDetail : entry.detail ?? 'permanent push failure',
|
|
2094
|
-
});
|
|
2095
|
-
}
|
|
2096
|
-
|
|
2097
|
-
if (result.failed.length > 0) {
|
|
2098
|
-
if (isFlushStale()) { return; }
|
|
2099
|
-
const failedSet = new Set(result.failed);
|
|
2100
|
-
const failedEntries = pushEntries.filter((entry) => failedSet.has(entry.cid));
|
|
2101
|
-
this.requeueOrReconcile(linkKey, {
|
|
2102
|
-
did, dwnUrl, delegateDid, protocol,
|
|
2103
|
-
entries : failedEntries,
|
|
2104
|
-
retryCount : retryCount + 1,
|
|
2105
|
-
});
|
|
2106
|
-
} else {
|
|
2107
|
-
// Successful push — reset retry count so subsequent unrelated
|
|
2108
|
-
// batches on this link start with a fresh budget.
|
|
2109
|
-
pushRuntime.retryCount = 0;
|
|
2110
|
-
if (!pushRuntime.timer && pushRuntime.entries.length === 0) {
|
|
2111
|
-
this._pushRuntimes.delete(linkKey);
|
|
2112
|
-
}
|
|
2113
|
-
}
|
|
2488
|
+
await this.handlePushBatchResult(linkKey, batch, result);
|
|
2114
2489
|
} catch (error: any) {
|
|
2115
|
-
if (
|
|
2490
|
+
if (isStale()) { return; }
|
|
2116
2491
|
console.error(`SyncEngineLevel: Push batch failed for ${did} -> ${dwnUrl}`, error);
|
|
2117
2492
|
this.requeueOrReconcile(linkKey, {
|
|
2118
|
-
did,
|
|
2493
|
+
did,
|
|
2494
|
+
dwnUrl,
|
|
2495
|
+
delegateDid,
|
|
2496
|
+
protocol,
|
|
2497
|
+
permissionGrantIds,
|
|
2119
2498
|
entries : pushEntries,
|
|
2120
2499
|
retryCount : retryCount + 1,
|
|
2121
2500
|
});
|
|
2122
2501
|
} finally {
|
|
2123
|
-
|
|
2502
|
+
this.finishPushFlush(linkKey, pushRuntime);
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2124
2505
|
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2506
|
+
private takePushFlushBatch(linkKey: string): PushFlushBatch | undefined {
|
|
2507
|
+
// Guard: bail if this link was hot-removed or is no longer live. Without
|
|
2508
|
+
// this, a stale debounce timer or retry callback could send pushes after
|
|
2509
|
+
// the DID was removed or the link entered repair/terminal state.
|
|
2510
|
+
const flushLink = this._activeLinks.get(linkKey);
|
|
2511
|
+
if (flushLink?.status !== 'live') {
|
|
2512
|
+
const staleRuntime = this._pushRuntimes.get(linkKey);
|
|
2513
|
+
if (staleRuntime?.timer) {
|
|
2514
|
+
clearTimeout(staleRuntime.timer);
|
|
2134
2515
|
}
|
|
2516
|
+
this._pushRuntimes.delete(linkKey);
|
|
2517
|
+
return undefined;
|
|
2518
|
+
}
|
|
2519
|
+
|
|
2520
|
+
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
2521
|
+
if (!pushRuntime) {
|
|
2522
|
+
return undefined;
|
|
2523
|
+
}
|
|
2524
|
+
|
|
2525
|
+
const { entries: pushEntries, retryCount } = pushRuntime;
|
|
2526
|
+
pushRuntime.entries = [];
|
|
2527
|
+
|
|
2528
|
+
if (pushEntries.length === 0) {
|
|
2529
|
+
if (!pushRuntime.timer && !pushRuntime.flushing && retryCount === 0) {
|
|
2530
|
+
this._pushRuntimes.delete(linkKey);
|
|
2531
|
+
}
|
|
2532
|
+
return undefined;
|
|
2533
|
+
}
|
|
2534
|
+
|
|
2535
|
+
// Capture the current active link identity so we can detect
|
|
2536
|
+
// remove+re-add during the await pushMessages() call.
|
|
2537
|
+
const isStale = (): boolean =>
|
|
2538
|
+
!this._activeLinks.has(linkKey) ||
|
|
2539
|
+
(flushLink !== undefined && this._activeLinks.get(linkKey) !== flushLink);
|
|
2540
|
+
|
|
2541
|
+
pushRuntime.flushing = true;
|
|
2542
|
+
return { pushRuntime, pushEntries, isStale };
|
|
2543
|
+
}
|
|
2544
|
+
|
|
2545
|
+
private async handlePushBatchResult(
|
|
2546
|
+
linkKey: string,
|
|
2547
|
+
batch: PushFlushBatch,
|
|
2548
|
+
result: PushResult,
|
|
2549
|
+
): Promise<void> {
|
|
2550
|
+
if (batch.isStale()) { return; }
|
|
2551
|
+
|
|
2552
|
+
this.clearSucceededPushFailures(result.succeeded, batch.pushRuntime.dwnUrl);
|
|
2553
|
+
await this.recordPermanentPushFailures(batch.pushRuntime, result.permanentlyFailed);
|
|
2554
|
+
|
|
2555
|
+
if (result.failed.length > 0) {
|
|
2556
|
+
this.requeueFailedPushes(linkKey, batch, result.failed);
|
|
2557
|
+
return;
|
|
2558
|
+
}
|
|
2559
|
+
|
|
2560
|
+
this.cleanupSuccessfulPushRuntime(linkKey, batch.pushRuntime);
|
|
2561
|
+
}
|
|
2562
|
+
|
|
2563
|
+
private clearSucceededPushFailures(cids: string[], dwnUrl: string): void {
|
|
2564
|
+
for (const cid of cids) {
|
|
2565
|
+
this.clearFailedMessage(cid, dwnUrl).catch(() => { /* teardown race */ });
|
|
2566
|
+
}
|
|
2567
|
+
}
|
|
2568
|
+
|
|
2569
|
+
private async recordPermanentPushFailures(
|
|
2570
|
+
pushRuntime: PushRuntimeState,
|
|
2571
|
+
permanentlyFailed: PushResult['permanentlyFailed'],
|
|
2572
|
+
): Promise<void> {
|
|
2573
|
+
for (const entry of permanentlyFailed) {
|
|
2574
|
+
await this.recordDeadLetter({
|
|
2575
|
+
messageCid : entry.cid,
|
|
2576
|
+
tenantDid : pushRuntime.did,
|
|
2577
|
+
remoteEndpoint : pushRuntime.dwnUrl,
|
|
2578
|
+
protocol : pushRuntime.protocol,
|
|
2579
|
+
category : 'push-permanent',
|
|
2580
|
+
errorCode : String(entry.statusCode ?? ''),
|
|
2581
|
+
errorDetail : entry.detail ?? 'permanent push failure',
|
|
2582
|
+
});
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
|
|
2586
|
+
private requeueFailedPushes(linkKey: string, batch: PushFlushBatch, failedCids: string[]): void {
|
|
2587
|
+
if (batch.isStale()) { return; }
|
|
2588
|
+
|
|
2589
|
+
const { did, dwnUrl, delegateDid, protocol, permissionGrantIds, retryCount } = batch.pushRuntime;
|
|
2590
|
+
const failedSet = new Set(failedCids);
|
|
2591
|
+
const failedEntries = batch.pushEntries.filter((entry) => failedSet.has(entry.cid));
|
|
2592
|
+
this.requeueOrReconcile(linkKey, {
|
|
2593
|
+
did,
|
|
2594
|
+
dwnUrl,
|
|
2595
|
+
delegateDid,
|
|
2596
|
+
protocol,
|
|
2597
|
+
permissionGrantIds,
|
|
2598
|
+
entries : failedEntries,
|
|
2599
|
+
retryCount : retryCount + 1,
|
|
2600
|
+
});
|
|
2601
|
+
}
|
|
2602
|
+
|
|
2603
|
+
private cleanupSuccessfulPushRuntime(linkKey: string, pushRuntime: PushRuntimeState): void {
|
|
2604
|
+
// Successful push — reset retry count so subsequent unrelated batches on
|
|
2605
|
+
// this link start with a fresh budget.
|
|
2606
|
+
pushRuntime.retryCount = 0;
|
|
2607
|
+
if (!pushRuntime.timer && pushRuntime.entries.length === 0) {
|
|
2608
|
+
this._pushRuntimes.delete(linkKey);
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2612
|
+
private finishPushFlush(linkKey: string, pushRuntime: PushRuntimeState): void {
|
|
2613
|
+
pushRuntime.flushing = false;
|
|
2614
|
+
|
|
2615
|
+
// If new entries accumulated while this push was in flight, schedule a
|
|
2616
|
+
// short drain to flush them. This gives a brief batching window for burst
|
|
2617
|
+
// writes while keeping single-write latency low.
|
|
2618
|
+
const rt = this._pushRuntimes.get(linkKey);
|
|
2619
|
+
if (rt && rt.entries.length > 0 && !rt.timer) {
|
|
2620
|
+
rt.timer = setTimeout((): void => {
|
|
2621
|
+
rt.timer = undefined;
|
|
2622
|
+
void this.flushPendingPushesForLink(linkKey);
|
|
2623
|
+
}, PUSH_DEBOUNCE_MS);
|
|
2135
2624
|
}
|
|
2136
2625
|
}
|
|
2137
2626
|
|
|
2138
2627
|
/** Push retry backoff schedule: immediate, 250ms, 1s, 2s, then give up. */
|
|
2139
2628
|
private static readonly PUSH_RETRY_BACKOFF_MS = [0, 250, 1000, 2000];
|
|
2629
|
+
private static readonly ROOT_CONVERGENCE_CLEARABLE_DEAD_LETTER_CATEGORIES: ReadonlySet<DeadLetterCategory> =
|
|
2630
|
+
new Set(['push-permanent', 'push-exhausted', 'pull-processing', 'pull-scope-rejected']);
|
|
2140
2631
|
|
|
2141
2632
|
/**
|
|
2142
2633
|
* Re-queues a failed push batch for retry, or marks the link
|
|
@@ -2145,7 +2636,8 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2145
2636
|
*/
|
|
2146
2637
|
private requeueOrReconcile(targetKey: string, pending: {
|
|
2147
2638
|
did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
|
|
2148
|
-
|
|
2639
|
+
permissionGrantIds?: NonEmptyStringArray;
|
|
2640
|
+
entries: PushRuntimeEntry[];
|
|
2149
2641
|
retryCount: number;
|
|
2150
2642
|
}): void {
|
|
2151
2643
|
const maxRetries = SyncEngineLevel.PUSH_RETRY_BACKOFF_MS.length;
|
|
@@ -2169,12 +2661,8 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2169
2661
|
}
|
|
2170
2662
|
this._pushRuntimes.delete(targetKey);
|
|
2171
2663
|
const link = this._activeLinks.get(targetKey);
|
|
2172
|
-
if (link
|
|
2173
|
-
|
|
2174
|
-
void this.ledger.saveLink(link).then(() => {
|
|
2175
|
-
this.emitEvent({ type: 'reconcile:needed', tenantDid: pending.did, remoteEndpoint: pending.dwnUrl, protocol: pending.protocol, reason: 'push-retry-exhausted' });
|
|
2176
|
-
this.scheduleReconcile(targetKey);
|
|
2177
|
-
});
|
|
2664
|
+
if (link) {
|
|
2665
|
+
this.markLinkNeedsReconcile(targetKey, link, 'push-retry-exhausted');
|
|
2178
2666
|
}
|
|
2179
2667
|
return;
|
|
2180
2668
|
}
|
|
@@ -2185,21 +2673,956 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2185
2673
|
if (pushRuntime.timer) {
|
|
2186
2674
|
clearTimeout(pushRuntime.timer);
|
|
2187
2675
|
}
|
|
2188
|
-
pushRuntime.timer = setTimeout((): void => {
|
|
2189
|
-
pushRuntime.timer = undefined;
|
|
2190
|
-
void this.flushPendingPushesForLink(targetKey);
|
|
2191
|
-
}, delayMs);
|
|
2676
|
+
pushRuntime.timer = setTimeout((): void => {
|
|
2677
|
+
pushRuntime.timer = undefined;
|
|
2678
|
+
void this.flushPendingPushesForLink(targetKey);
|
|
2679
|
+
}, delayMs);
|
|
2680
|
+
}
|
|
2681
|
+
|
|
2682
|
+
private markLinkNeedsReconcile(linkKey: string, link: ReplicationLinkState, reason: string): void {
|
|
2683
|
+
if (link.needsReconcile) {
|
|
2684
|
+
this.scheduleReconcile(linkKey);
|
|
2685
|
+
return;
|
|
2686
|
+
}
|
|
2687
|
+
|
|
2688
|
+
link.needsReconcile = true;
|
|
2689
|
+
void this.ledger.saveLink(link).then(() => {
|
|
2690
|
+
this.emitEvent({
|
|
2691
|
+
type : 'reconcile:needed',
|
|
2692
|
+
tenantDid : link.tenantDid,
|
|
2693
|
+
remoteEndpoint : link.remoteEndpoint,
|
|
2694
|
+
...syncEventScope(link.scope),
|
|
2695
|
+
reason,
|
|
2696
|
+
});
|
|
2697
|
+
this.scheduleReconcile(linkKey);
|
|
2698
|
+
}).catch((error: unknown) => {
|
|
2699
|
+
console.error(`SyncEngineLevel: Failed to mark link for reconciliation ${link.tenantDid} -> ${link.remoteEndpoint}`, error);
|
|
2700
|
+
});
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2703
|
+
private createLinkReconciler(shouldContinue?: () => boolean): SyncLinkReconciler {
|
|
2704
|
+
return new SyncLinkReconciler({
|
|
2705
|
+
getLocalRoot : async (did, delegateDid, protocol, permissionGrantIds) => this.getLocalRoot(did, delegateDid, protocol, permissionGrantIds),
|
|
2706
|
+
getRemoteRoot : async (did, dwnUrl, delegateDid, protocol, permissionGrantIds) =>
|
|
2707
|
+
this.getRemoteRoot(did, dwnUrl, delegateDid, protocol, permissionGrantIds),
|
|
2708
|
+
diffWithRemote : async (target) => this.diffWithRemote(target),
|
|
2709
|
+
pullMessages : async (params) => this.pullMessages(params),
|
|
2710
|
+
pushMessages : async (params) => this.pushMessages(params),
|
|
2711
|
+
shouldContinue,
|
|
2712
|
+
});
|
|
2713
|
+
}
|
|
2714
|
+
|
|
2715
|
+
private getReconcileProtocols(scope: SyncScope): (string | undefined)[] {
|
|
2716
|
+
return protocolsForSyncScope(scope) ?? [undefined];
|
|
2717
|
+
}
|
|
2718
|
+
|
|
2719
|
+
private getAuthorizationGrantIds(authorization: SyncAuthorization): NonEmptyStringArray | undefined {
|
|
2720
|
+
return authorization.kind === 'delegate' ? authorization.permissionGrantIds : undefined;
|
|
2721
|
+
}
|
|
2722
|
+
|
|
2723
|
+
private async reconcileProjectionTarget(
|
|
2724
|
+
target: ProjectionReconcileTarget,
|
|
2725
|
+
options?: ProjectionReconcileOptions,
|
|
2726
|
+
shouldContinue?: () => boolean,
|
|
2727
|
+
): Promise<ProjectionReconcileResult> {
|
|
2728
|
+
if (target.scope.kind === 'recordsProjection') {
|
|
2729
|
+
return this.reconcileRecordsProjectionTarget(target, target.scope, options, shouldContinue);
|
|
2730
|
+
}
|
|
2731
|
+
|
|
2732
|
+
if (target.scope.kind === 'protocolSet' && target.scope.protocols.length > 1) {
|
|
2733
|
+
return this.reconcileProtocolSetProjectionTarget(target, options, shouldContinue);
|
|
2734
|
+
}
|
|
2735
|
+
|
|
2736
|
+
let converged = true;
|
|
2737
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
2738
|
+
const reconciler = this.createLinkReconciler(shouldContinue);
|
|
2739
|
+
|
|
2740
|
+
for (const protocol of this.getReconcileProtocols(target.scope)) {
|
|
2741
|
+
const outcome = await reconciler.reconcile({
|
|
2742
|
+
did : target.did,
|
|
2743
|
+
dwnUrl : target.dwnUrl,
|
|
2744
|
+
delegateDid : target.delegateDid,
|
|
2745
|
+
protocol,
|
|
2746
|
+
permissionGrantIds,
|
|
2747
|
+
}, options);
|
|
2748
|
+
if (outcome.aborted) {
|
|
2749
|
+
return { aborted: true };
|
|
2750
|
+
}
|
|
2751
|
+
if (options?.verifyConvergence === true && outcome.converged !== true) {
|
|
2752
|
+
converged = false;
|
|
2753
|
+
}
|
|
2754
|
+
}
|
|
2755
|
+
|
|
2756
|
+
return options?.verifyConvergence === true ? { converged } : {};
|
|
2757
|
+
}
|
|
2758
|
+
|
|
2759
|
+
private async reconcileRecordsProjectionTarget(
|
|
2760
|
+
target: ProjectionReconcileTarget,
|
|
2761
|
+
scope: RecordsProjectionSyncScope,
|
|
2762
|
+
options?: ProjectionReconcileOptions,
|
|
2763
|
+
shouldContinue?: () => boolean,
|
|
2764
|
+
): Promise<ProjectionReconcileResult> {
|
|
2765
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
2766
|
+
const localRoot = await this.getLocalProjectedRoot(target.did, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2767
|
+
if (shouldContinue?.() === false) { return { aborted: true }; }
|
|
2768
|
+
|
|
2769
|
+
const remoteRoot = await this.getRemoteProjectedRoot(target.did, target.dwnUrl, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2770
|
+
if (shouldContinue?.() === false) { return { aborted: true }; }
|
|
2771
|
+
|
|
2772
|
+
if (localRoot !== remoteRoot) {
|
|
2773
|
+
const diff = await this.diffProjectedWithRemote({
|
|
2774
|
+
did : target.did,
|
|
2775
|
+
dwnUrl : target.dwnUrl,
|
|
2776
|
+
delegateDid : target.delegateDid,
|
|
2777
|
+
scopes : scope.scopes,
|
|
2778
|
+
permissionGrantIds,
|
|
2779
|
+
});
|
|
2780
|
+
if (shouldContinue?.() === false) { return { aborted: true }; }
|
|
2781
|
+
|
|
2782
|
+
const aborted = await this.applyProjectedDiff(target, scope, diff, permissionGrantIds, options, shouldContinue);
|
|
2783
|
+
if (aborted) { return { aborted: true }; }
|
|
2784
|
+
}
|
|
2785
|
+
|
|
2786
|
+
if (options?.verifyConvergence !== true) {
|
|
2787
|
+
return {};
|
|
2788
|
+
}
|
|
2789
|
+
|
|
2790
|
+
const postLocalRoot = await this.getLocalProjectedRoot(target.did, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2791
|
+
if (shouldContinue?.() === false) { return { aborted: true }; }
|
|
2792
|
+
|
|
2793
|
+
const postRemoteRoot = await this.getRemoteProjectedRoot(target.did, target.dwnUrl, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2794
|
+
if (shouldContinue?.() === false) { return { aborted: true }; }
|
|
2795
|
+
|
|
2796
|
+
return { converged: postLocalRoot === postRemoteRoot };
|
|
2797
|
+
}
|
|
2798
|
+
|
|
2799
|
+
private async reconcileProtocolSetProjectionTarget(
|
|
2800
|
+
target: ProjectionReconcileTarget,
|
|
2801
|
+
options?: ProjectionReconcileOptions,
|
|
2802
|
+
shouldContinue?: () => boolean,
|
|
2803
|
+
): Promise<ProjectionReconcileResult> {
|
|
2804
|
+
if (target.scope.kind !== 'protocolSet') {
|
|
2805
|
+
return {};
|
|
2806
|
+
}
|
|
2807
|
+
|
|
2808
|
+
const scope = target.scope;
|
|
2809
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
2810
|
+
const diffPlan = await this.collectProtocolSetDiffPlan(target, scope, permissionGrantIds, shouldContinue);
|
|
2811
|
+
if (!diffPlan) {
|
|
2812
|
+
return { aborted: true };
|
|
2813
|
+
}
|
|
2814
|
+
|
|
2815
|
+
if (diffPlan.changedProtocols.length === 0) {
|
|
2816
|
+
return options?.verifyConvergence === true ? { converged: true } : {};
|
|
2817
|
+
}
|
|
2818
|
+
|
|
2819
|
+
const aborted = await this.applyProtocolSetDiffPlan(target, scope, diffPlan, permissionGrantIds, options, shouldContinue);
|
|
2820
|
+
if (aborted) {
|
|
2821
|
+
return { aborted: true };
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2824
|
+
if (options?.verifyConvergence !== true) {
|
|
2825
|
+
return {};
|
|
2826
|
+
}
|
|
2827
|
+
|
|
2828
|
+
return this.verifyProtocolSetConvergence(target, diffPlan.changedProtocols, permissionGrantIds, shouldContinue);
|
|
2829
|
+
}
|
|
2830
|
+
|
|
2831
|
+
private async collectProtocolSetDiffPlan(
|
|
2832
|
+
target: ProjectionReconcileTarget,
|
|
2833
|
+
scope: ProtocolSetScope,
|
|
2834
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
2835
|
+
shouldContinue?: () => boolean,
|
|
2836
|
+
): Promise<ProtocolSetDiffPlan | undefined> {
|
|
2837
|
+
const plan: ProtocolSetDiffPlan = { changedProtocols: [], onlyRemote: [], onlyLocal: [] };
|
|
2838
|
+
|
|
2839
|
+
for (const protocol of scope.protocols) {
|
|
2840
|
+
const roots = await this.getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue);
|
|
2841
|
+
if (!roots) { return undefined; }
|
|
2842
|
+
|
|
2843
|
+
if (roots.localRoot === roots.remoteRoot) {
|
|
2844
|
+
continue;
|
|
2845
|
+
}
|
|
2846
|
+
|
|
2847
|
+
plan.changedProtocols.push(protocol);
|
|
2848
|
+
const diff = await this.diffWithRemote({
|
|
2849
|
+
did : target.did,
|
|
2850
|
+
dwnUrl : target.dwnUrl,
|
|
2851
|
+
delegateDid : target.delegateDid,
|
|
2852
|
+
protocol,
|
|
2853
|
+
permissionGrantIds,
|
|
2854
|
+
});
|
|
2855
|
+
if (shouldContinue?.() === false) { return undefined; }
|
|
2856
|
+
|
|
2857
|
+
plan.onlyRemote.push(...diff.onlyRemote);
|
|
2858
|
+
plan.onlyLocal.push(...diff.onlyLocal);
|
|
2859
|
+
}
|
|
2860
|
+
|
|
2861
|
+
return plan;
|
|
2862
|
+
}
|
|
2863
|
+
|
|
2864
|
+
private async getProtocolRoots(
|
|
2865
|
+
target: ProjectionReconcileTarget,
|
|
2866
|
+
protocol: string,
|
|
2867
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
2868
|
+
shouldContinue?: () => boolean,
|
|
2869
|
+
): Promise<{ localRoot: string; remoteRoot: string } | undefined> {
|
|
2870
|
+
const localRoot = await this.getLocalRoot(target.did, target.delegateDid, protocol, permissionGrantIds);
|
|
2871
|
+
if (shouldContinue?.() === false) { return undefined; }
|
|
2872
|
+
|
|
2873
|
+
const remoteRoot = await this.getRemoteRoot(target.did, target.dwnUrl, target.delegateDid, protocol, permissionGrantIds);
|
|
2874
|
+
if (shouldContinue?.() === false) { return undefined; }
|
|
2875
|
+
|
|
2876
|
+
return { localRoot, remoteRoot };
|
|
2877
|
+
}
|
|
2878
|
+
|
|
2879
|
+
private async applyProtocolSetDiffPlan(
|
|
2880
|
+
target: ProjectionReconcileTarget,
|
|
2881
|
+
scope: ProtocolSetScope,
|
|
2882
|
+
diffPlan: ProtocolSetDiffPlan,
|
|
2883
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
2884
|
+
options?: ProjectionReconcileOptions,
|
|
2885
|
+
shouldContinue?: () => boolean,
|
|
2886
|
+
): Promise<boolean> {
|
|
2887
|
+
// Keep the remote diff combined across protocols so topologicalSort can
|
|
2888
|
+
// order composed protocol configs before records that use them. Any future
|
|
2889
|
+
// chunking for large protocol sets must preserve this global dependency
|
|
2890
|
+
// order instead of reverting to independent per-protocol chunks.
|
|
2891
|
+
if (
|
|
2892
|
+
options?.direction !== 'push' &&
|
|
2893
|
+
diffPlan.onlyRemote.length > 0 &&
|
|
2894
|
+
await this.pullRemoteDiffEntries(target, scope, diffPlan.onlyRemote, permissionGrantIds, shouldContinue)
|
|
2895
|
+
) {
|
|
2896
|
+
return true;
|
|
2897
|
+
}
|
|
2898
|
+
|
|
2899
|
+
if (options?.direction === 'pull' || diffPlan.onlyLocal.length === 0) {
|
|
2900
|
+
return false;
|
|
2901
|
+
}
|
|
2902
|
+
|
|
2903
|
+
return this.pushLocalDiffEntries(target, diffPlan.onlyLocal, permissionGrantIds, shouldContinue);
|
|
2904
|
+
}
|
|
2905
|
+
|
|
2906
|
+
private async applyProjectedDiff(
|
|
2907
|
+
target: ProjectionReconcileTarget,
|
|
2908
|
+
scope: RecordsProjectionSyncScope,
|
|
2909
|
+
diff: ProjectionDiffResult,
|
|
2910
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
2911
|
+
options?: ProjectionReconcileOptions,
|
|
2912
|
+
shouldContinue?: () => boolean,
|
|
2913
|
+
): Promise<boolean> {
|
|
2914
|
+
if (await this.pullProjectedRemoteDiff(target, scope, diff, permissionGrantIds, options, shouldContinue)) {
|
|
2915
|
+
return true;
|
|
2916
|
+
}
|
|
2917
|
+
|
|
2918
|
+
return this.pushProjectedLocalDiff(target, diff.onlyLocal, permissionGrantIds, options, shouldContinue);
|
|
2919
|
+
}
|
|
2920
|
+
|
|
2921
|
+
private async pullProjectedRemoteDiff(
|
|
2922
|
+
target: ProjectionReconcileTarget,
|
|
2923
|
+
scope: RecordsProjectionSyncScope,
|
|
2924
|
+
diff: ProjectionDiffResult,
|
|
2925
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
2926
|
+
options?: ProjectionReconcileOptions,
|
|
2927
|
+
shouldContinue?: () => boolean,
|
|
2928
|
+
): Promise<boolean> {
|
|
2929
|
+
if (options?.direction === 'push' || diff.onlyRemote.length === 0) {
|
|
2930
|
+
return false;
|
|
2931
|
+
}
|
|
2932
|
+
|
|
2933
|
+
return this.pullRemoteDiffEntries(target, scope, diff.onlyRemote, permissionGrantIds, shouldContinue, diff.dependencies ?? []);
|
|
2934
|
+
}
|
|
2935
|
+
|
|
2936
|
+
private async pushProjectedLocalDiff(
|
|
2937
|
+
target: ProjectionReconcileTarget,
|
|
2938
|
+
onlyLocal: string[],
|
|
2939
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
2940
|
+
options?: ProjectionReconcileOptions,
|
|
2941
|
+
shouldContinue?: () => boolean,
|
|
2942
|
+
): Promise<boolean> {
|
|
2943
|
+
if (options?.direction === 'pull' || onlyLocal.length === 0) {
|
|
2944
|
+
return false;
|
|
2945
|
+
}
|
|
2946
|
+
|
|
2947
|
+
return this.pushLocalDiffEntries(target, onlyLocal, permissionGrantIds, shouldContinue);
|
|
2948
|
+
}
|
|
2949
|
+
|
|
2950
|
+
private async pullRemoteDiffEntries(
|
|
2951
|
+
target: ProjectionReconcileTarget,
|
|
2952
|
+
scope: SyncScope,
|
|
2953
|
+
onlyRemote: MessagesSyncDiffEntry[],
|
|
2954
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
2955
|
+
shouldContinue?: () => boolean,
|
|
2956
|
+
dependencies: MessagesSyncDependencyEntry[] = [],
|
|
2957
|
+
): Promise<boolean> {
|
|
2958
|
+
const primaryEntries = SyncEngineLevel.dedupeRemoteEntries(onlyRemote);
|
|
2959
|
+
try {
|
|
2960
|
+
let verifiedInitialWrites: RecordsWriteMessage[] = [];
|
|
2961
|
+
if (scope.kind === 'recordsProjection') {
|
|
2962
|
+
verifiedInitialWrites = await this.pullProjectedDependencyHints(
|
|
2963
|
+
target,
|
|
2964
|
+
scope,
|
|
2965
|
+
primaryEntries,
|
|
2966
|
+
dependencies,
|
|
2967
|
+
permissionGrantIds,
|
|
2968
|
+
shouldContinue,
|
|
2969
|
+
);
|
|
2970
|
+
}
|
|
2971
|
+
|
|
2972
|
+
const { prefetched, needsFetchCids } = partitionRemoteEntries(primaryEntries);
|
|
2973
|
+
await this.pullMessages({
|
|
2974
|
+
did : target.did,
|
|
2975
|
+
dwnUrl : target.dwnUrl,
|
|
2976
|
+
delegateDid : target.delegateDid,
|
|
2977
|
+
scope,
|
|
2978
|
+
permissionGrantIds,
|
|
2979
|
+
messageCids : needsFetchCids,
|
|
2980
|
+
prefetched,
|
|
2981
|
+
verifiedInitialWrites,
|
|
2982
|
+
shouldContinue,
|
|
2983
|
+
});
|
|
2984
|
+
} catch (error) {
|
|
2985
|
+
if (error instanceof SyncPullAbortedError) {
|
|
2986
|
+
return true;
|
|
2987
|
+
}
|
|
2988
|
+
throw error;
|
|
2989
|
+
}
|
|
2990
|
+
return shouldContinue?.() === false;
|
|
2991
|
+
}
|
|
2992
|
+
|
|
2993
|
+
private async pullProjectedDependencyHints(
|
|
2994
|
+
target: ProjectionReconcileTarget,
|
|
2995
|
+
scope: RecordsProjectionSyncScope,
|
|
2996
|
+
primaryEntries: MessagesSyncDiffEntry[],
|
|
2997
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
2998
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
2999
|
+
shouldContinue?: () => boolean,
|
|
3000
|
+
): Promise<RecordsWriteMessage[]> {
|
|
3001
|
+
const verified = await this.verifyProjectedDependencies(target.did, scope, primaryEntries, dependencies);
|
|
3002
|
+
if (verified.length === 0) {
|
|
3003
|
+
return [];
|
|
3004
|
+
}
|
|
3005
|
+
|
|
3006
|
+
await this.pullMessages({
|
|
3007
|
+
did : target.did,
|
|
3008
|
+
dwnUrl : target.dwnUrl,
|
|
3009
|
+
delegateDid : target.delegateDid,
|
|
3010
|
+
scope : SyncEngineLevel.protocolSetScopeForProjectedDependencies(verified),
|
|
3011
|
+
permissionGrantIds,
|
|
3012
|
+
messageCids : [],
|
|
3013
|
+
prefetched : verified,
|
|
3014
|
+
shouldContinue,
|
|
3015
|
+
});
|
|
3016
|
+
|
|
3017
|
+
return SyncEngineLevel.recordsInitialWritesFromVerifiedDependencies(verified);
|
|
3018
|
+
}
|
|
3019
|
+
|
|
3020
|
+
private async verifyProjectedDependencies(
|
|
3021
|
+
tenantDid: string,
|
|
3022
|
+
scope: RecordsProjectionSyncScope,
|
|
3023
|
+
primaryEntries: MessagesSyncDiffEntry[],
|
|
3024
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3025
|
+
): Promise<MessagesSyncDependencyEntry[]> {
|
|
3026
|
+
const primaryByCid = SyncEngineLevel.indexEntriesWithMessage(primaryEntries);
|
|
3027
|
+
const initialWritesByRoot = await this.collectProjectedRecordsInitialWriteDependencies(
|
|
3028
|
+
scope,
|
|
3029
|
+
primaryByCid,
|
|
3030
|
+
dependencies,
|
|
3031
|
+
);
|
|
3032
|
+
const protocolConfigs = await this.verifyProjectedProtocolConfigDependencies(
|
|
3033
|
+
tenantDid,
|
|
3034
|
+
scope,
|
|
3035
|
+
primaryEntries,
|
|
3036
|
+
dependencies,
|
|
3037
|
+
initialWritesByRoot,
|
|
3038
|
+
);
|
|
3039
|
+
|
|
3040
|
+
return SyncEngineLevel.dedupeDependencyEntries([
|
|
3041
|
+
...protocolConfigs,
|
|
3042
|
+
...initialWritesByRoot.values(),
|
|
3043
|
+
]);
|
|
3044
|
+
}
|
|
3045
|
+
|
|
3046
|
+
private async verifyProjectedProtocolConfigDependencies(
|
|
3047
|
+
tenantDid: string,
|
|
3048
|
+
scope: RecordsProjectionSyncScope,
|
|
3049
|
+
primaryEntries: MessagesSyncDiffEntry[],
|
|
3050
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3051
|
+
initialWritesByRoot: Map<string, AuthenticatedRecordsInitialWriteDependency> = new Map(),
|
|
3052
|
+
): Promise<MessagesSyncDependencyEntry[]> {
|
|
3053
|
+
// Projected sync dependency entries are untrusted server hints. Before any
|
|
3054
|
+
// config is applied, bind it to an accepted primary record by CID,
|
|
3055
|
+
// tenant authorship, signature, timestamp, scope, and protocol closure;
|
|
3056
|
+
// malformed or unrelated hints are ignored.
|
|
3057
|
+
const primaryByCid = SyncEngineLevel.indexEntriesWithMessage(primaryEntries);
|
|
3058
|
+
const candidatesByRoot = await this.collectProjectedProtocolConfigCandidates(
|
|
3059
|
+
tenantDid,
|
|
3060
|
+
scope,
|
|
3061
|
+
primaryByCid,
|
|
3062
|
+
dependencies,
|
|
3063
|
+
initialWritesByRoot,
|
|
3064
|
+
);
|
|
3065
|
+
const verified = new Map<string, MessagesSyncDependencyEntry>();
|
|
3066
|
+
|
|
3067
|
+
for (const [rootMessageCid, rootCandidates] of candidatesByRoot) {
|
|
3068
|
+
const primary = primaryByCid.get(rootMessageCid);
|
|
3069
|
+
const rootRecordsWrite = primary === undefined
|
|
3070
|
+
? undefined
|
|
3071
|
+
: SyncEngineLevel.protocolConfigRootRecordsWrite(primary.message, initialWritesByRoot.get(rootMessageCid)?.message);
|
|
3072
|
+
const rootProtocol = SyncEngineLevel.recordsWriteProtocol(rootRecordsWrite);
|
|
3073
|
+
if (rootProtocol === undefined) {
|
|
3074
|
+
continue;
|
|
3075
|
+
}
|
|
3076
|
+
|
|
3077
|
+
for (const dependency of SyncEngineLevel.filterProtocolConfigClosure(rootProtocol, rootCandidates)) {
|
|
3078
|
+
verified.set(dependency.messageCid, dependency);
|
|
3079
|
+
}
|
|
3080
|
+
}
|
|
3081
|
+
|
|
3082
|
+
return [...verified.values()];
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
private static indexEntriesWithMessage(
|
|
3086
|
+
entries: MessagesSyncDiffEntry[],
|
|
3087
|
+
): Map<string, SyncDiffEntryWithMessage> {
|
|
3088
|
+
const entriesByCid = new Map<string, SyncDiffEntryWithMessage>();
|
|
3089
|
+
for (const entry of entries) {
|
|
3090
|
+
if (SyncEngineLevel.hasMessage(entry)) {
|
|
3091
|
+
entriesByCid.set(entry.messageCid, entry);
|
|
3092
|
+
}
|
|
3093
|
+
}
|
|
3094
|
+
return entriesByCid;
|
|
3095
|
+
}
|
|
3096
|
+
|
|
3097
|
+
private static recordsInitialWritesFromVerifiedDependencies(
|
|
3098
|
+
entries: MessagesSyncDependencyEntry[],
|
|
3099
|
+
): RecordsWriteMessage[] {
|
|
3100
|
+
const initialWrites: RecordsWriteMessage[] = [];
|
|
3101
|
+
for (const entry of entries) {
|
|
3102
|
+
if (SyncEngineLevel.hasMessage(entry) && SyncEngineLevel.isRecordsWriteMessage(entry.message)) {
|
|
3103
|
+
initialWrites.push(entry.message);
|
|
3104
|
+
}
|
|
3105
|
+
}
|
|
3106
|
+
return initialWrites;
|
|
3107
|
+
}
|
|
3108
|
+
|
|
3109
|
+
private async collectProjectedRecordsInitialWriteDependencies(
|
|
3110
|
+
scope: RecordsProjectionSyncScope,
|
|
3111
|
+
primaryByCid: Map<string, SyncDiffEntryWithMessage>,
|
|
3112
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3113
|
+
): Promise<Map<string, AuthenticatedRecordsInitialWriteDependency>> {
|
|
3114
|
+
const dependenciesByRoot = new Map<string, AuthenticatedRecordsInitialWriteDependency>();
|
|
3115
|
+
for (const dependency of dependencies) {
|
|
3116
|
+
const verified = await this.verifyRecordsInitialWriteCandidate(scope, primaryByCid, dependency);
|
|
3117
|
+
if (verified === undefined) {
|
|
3118
|
+
continue;
|
|
3119
|
+
}
|
|
3120
|
+
|
|
3121
|
+
dependenciesByRoot.set(verified.rootMessageCid, verified.dependency);
|
|
3122
|
+
}
|
|
3123
|
+
return dependenciesByRoot;
|
|
3124
|
+
}
|
|
3125
|
+
|
|
3126
|
+
private async verifyRecordsInitialWriteCandidate(
|
|
3127
|
+
scope: RecordsProjectionSyncScope,
|
|
3128
|
+
primaryByCid: Map<string, SyncDiffEntryWithMessage>,
|
|
3129
|
+
dependency: MessagesSyncDependencyEntry,
|
|
3130
|
+
): Promise<VerifiedRecordsInitialWriteCandidate | undefined> {
|
|
3131
|
+
if (dependency.dependencyClass !== 'recordsInitialWrite' ||
|
|
3132
|
+
!SyncEngineLevel.hasMessage(dependency) ||
|
|
3133
|
+
SyncEngineLevel.hasDependencyPayloadBytes(dependency)) {
|
|
3134
|
+
return undefined;
|
|
3135
|
+
}
|
|
3136
|
+
|
|
3137
|
+
const primary = primaryByCid.get(dependency.rootMessageCid);
|
|
3138
|
+
if (primary === undefined ||
|
|
3139
|
+
!SyncEngineLevel.isRecordsDeleteMessage(primary.message) ||
|
|
3140
|
+
!await SyncEngineLevel.projectedDependencyCidsMatch({
|
|
3141
|
+
dependencyCid : dependency.messageCid,
|
|
3142
|
+
dependencyMessage : dependency.message,
|
|
3143
|
+
primaryCid : primary.messageCid,
|
|
3144
|
+
primaryMessage : primary.message,
|
|
3145
|
+
})) {
|
|
3146
|
+
return undefined;
|
|
3147
|
+
}
|
|
3148
|
+
|
|
3149
|
+
const initialWrite = await this.toAuthenticatedRecordsInitialWriteDependency(dependency);
|
|
3150
|
+
if (initialWrite === undefined ||
|
|
3151
|
+
initialWrite.message.recordId !== SyncEngineLevel.recordsDeleteRecordId(primary.message) ||
|
|
3152
|
+
classifySyncMessageScope({ message: primary.message, initialWrite: initialWrite.message, scope }) !== 'in-scope') {
|
|
3153
|
+
return undefined;
|
|
3154
|
+
}
|
|
3155
|
+
|
|
3156
|
+
return { dependency: initialWrite, rootMessageCid: dependency.rootMessageCid };
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
private async toAuthenticatedRecordsInitialWriteDependency(
|
|
3160
|
+
dependency: SyncDependencyEntryWithMessage,
|
|
3161
|
+
): Promise<AuthenticatedRecordsInitialWriteDependency | undefined> {
|
|
3162
|
+
if (!SyncEngineLevel.isRecordsWriteMessage(dependency.message)) {
|
|
3163
|
+
return undefined;
|
|
3164
|
+
}
|
|
3165
|
+
|
|
3166
|
+
try {
|
|
3167
|
+
const recordsWrite = await RecordsWrite.parse(dependency.message);
|
|
3168
|
+
await authenticate(recordsWrite.message.authorization, this.agent.did, recordsWrite.message.attestation);
|
|
3169
|
+
return await recordsWrite.isInitialWrite()
|
|
3170
|
+
? { ...dependency, message: recordsWrite.message }
|
|
3171
|
+
: undefined;
|
|
3172
|
+
} catch {
|
|
3173
|
+
return undefined;
|
|
3174
|
+
}
|
|
3175
|
+
}
|
|
3176
|
+
|
|
3177
|
+
private async collectProjectedProtocolConfigCandidates(
|
|
3178
|
+
tenantDid: string,
|
|
3179
|
+
scope: RecordsProjectionSyncScope,
|
|
3180
|
+
primaryByCid: Map<string, SyncDiffEntryWithMessage>,
|
|
3181
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3182
|
+
initialWritesByRoot: Map<string, AuthenticatedRecordsInitialWriteDependency>,
|
|
3183
|
+
): Promise<Map<string, AuthenticatedProtocolConfigDependency[]>> {
|
|
3184
|
+
const candidatesByRoot = new Map<string, AuthenticatedProtocolConfigDependency[]>();
|
|
3185
|
+
for (const dependency of dependencies) {
|
|
3186
|
+
const verified = await this.verifyProtocolConfigCandidate(tenantDid, scope, primaryByCid, dependency, initialWritesByRoot);
|
|
3187
|
+
if (verified === undefined) {
|
|
3188
|
+
continue;
|
|
3189
|
+
}
|
|
3190
|
+
|
|
3191
|
+
const rootCandidates = candidatesByRoot.get(verified.rootMessageCid) ?? [];
|
|
3192
|
+
rootCandidates.push(verified.dependency);
|
|
3193
|
+
candidatesByRoot.set(verified.rootMessageCid, rootCandidates);
|
|
3194
|
+
}
|
|
3195
|
+
return candidatesByRoot;
|
|
3196
|
+
}
|
|
3197
|
+
|
|
3198
|
+
private async verifyProtocolConfigCandidate(
|
|
3199
|
+
tenantDid: string,
|
|
3200
|
+
scope: RecordsProjectionSyncScope,
|
|
3201
|
+
primaryByCid: Map<string, SyncDiffEntryWithMessage>,
|
|
3202
|
+
dependency: MessagesSyncDependencyEntry,
|
|
3203
|
+
initialWritesByRoot: Map<string, AuthenticatedRecordsInitialWriteDependency>,
|
|
3204
|
+
): Promise<VerifiedProtocolConfigCandidate | undefined> {
|
|
3205
|
+
if (dependency.dependencyClass !== 'protocolsConfigure' || !SyncEngineLevel.hasMessage(dependency)) {
|
|
3206
|
+
return undefined;
|
|
3207
|
+
}
|
|
3208
|
+
|
|
3209
|
+
const primary = primaryByCid.get(dependency.rootMessageCid);
|
|
3210
|
+
if (primary === undefined) {
|
|
3211
|
+
return undefined;
|
|
3212
|
+
}
|
|
3213
|
+
|
|
3214
|
+
const verifiedDependency = await this.verifyProtocolConfigCandidateMessage(
|
|
3215
|
+
tenantDid,
|
|
3216
|
+
scope,
|
|
3217
|
+
primary,
|
|
3218
|
+
dependency,
|
|
3219
|
+
initialWritesByRoot.get(dependency.rootMessageCid)?.message,
|
|
3220
|
+
);
|
|
3221
|
+
return verifiedDependency === undefined
|
|
3222
|
+
? undefined
|
|
3223
|
+
: { dependency: verifiedDependency, rootMessageCid: dependency.rootMessageCid };
|
|
3224
|
+
}
|
|
3225
|
+
|
|
3226
|
+
private async verifyProtocolConfigCandidateMessage(
|
|
3227
|
+
tenantDid: string,
|
|
3228
|
+
scope: RecordsProjectionSyncScope,
|
|
3229
|
+
primary: SyncDiffEntryWithMessage,
|
|
3230
|
+
dependency: SyncDependencyEntryWithMessage,
|
|
3231
|
+
initialWrite: RecordsWriteMessage | undefined,
|
|
3232
|
+
): Promise<AuthenticatedProtocolConfigDependency | undefined> {
|
|
3233
|
+
// Protocol authorization is temporal: a record is governed by the protocol
|
|
3234
|
+
// definition active at its creation timestamp. Future configs may add
|
|
3235
|
+
// unrelated `uses` dependencies, so they must not widen this primary's
|
|
3236
|
+
// dependency closure.
|
|
3237
|
+
if (!await SyncEngineLevel.projectedDependencyCidsMatch({
|
|
3238
|
+
dependencyCid : dependency.messageCid,
|
|
3239
|
+
dependencyMessage : dependency.message,
|
|
3240
|
+
primaryCid : primary.messageCid,
|
|
3241
|
+
primaryMessage : primary.message,
|
|
3242
|
+
})) {
|
|
3243
|
+
return undefined;
|
|
3244
|
+
}
|
|
3245
|
+
|
|
3246
|
+
const authenticatedDependency = await this.toAuthenticatedProtocolConfigDependency(tenantDid, dependency);
|
|
3247
|
+
if (authenticatedDependency === undefined) {
|
|
3248
|
+
return undefined;
|
|
3249
|
+
}
|
|
3250
|
+
|
|
3251
|
+
const rootRecordsWrite = SyncEngineLevel.protocolConfigRootRecordsWrite(primary.message, initialWrite);
|
|
3252
|
+
const primaryIsInScope = rootRecordsWrite !== undefined &&
|
|
3253
|
+
classifySyncMessageScope({ message: primary.message, initialWrite, scope }) === 'in-scope';
|
|
3254
|
+
if (!primaryIsInScope ||
|
|
3255
|
+
!SyncEngineLevel.protocolsConfigureIsNotNewerThanRecordsWrite(authenticatedDependency.message, rootRecordsWrite)) {
|
|
3256
|
+
return undefined;
|
|
3257
|
+
}
|
|
3258
|
+
|
|
3259
|
+
return authenticatedDependency;
|
|
3260
|
+
}
|
|
3261
|
+
|
|
3262
|
+
private async toAuthenticatedProtocolConfigDependency(
|
|
3263
|
+
tenantDid: string,
|
|
3264
|
+
dependency: SyncDependencyEntryWithMessage,
|
|
3265
|
+
): Promise<AuthenticatedProtocolConfigDependency | undefined> {
|
|
3266
|
+
if (!SyncEngineLevel.isProtocolsConfigureDefinitionMessage(dependency.message)) {
|
|
3267
|
+
return undefined;
|
|
3268
|
+
}
|
|
3269
|
+
|
|
3270
|
+
try {
|
|
3271
|
+
await ProtocolsConfigure.parse(dependency.message);
|
|
3272
|
+
if (Message.getAuthor(dependency.message) !== tenantDid) {
|
|
3273
|
+
return undefined;
|
|
3274
|
+
}
|
|
3275
|
+
await authenticate(dependency.message.authorization, this.agent.did);
|
|
3276
|
+
return { ...dependency, message: dependency.message };
|
|
3277
|
+
} catch {
|
|
3278
|
+
return undefined;
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3281
|
+
|
|
3282
|
+
private static async projectedDependencyCidsMatch({
|
|
3283
|
+
dependencyCid,
|
|
3284
|
+
dependencyMessage,
|
|
3285
|
+
primaryCid,
|
|
3286
|
+
primaryMessage,
|
|
3287
|
+
}: {
|
|
3288
|
+
dependencyCid: string;
|
|
3289
|
+
dependencyMessage: GenericMessage;
|
|
3290
|
+
primaryCid: string;
|
|
3291
|
+
primaryMessage: GenericMessage;
|
|
3292
|
+
}): Promise<boolean> {
|
|
3293
|
+
return await Message.getCid(primaryMessage) === primaryCid &&
|
|
3294
|
+
await Message.getCid(dependencyMessage) === dependencyCid;
|
|
3295
|
+
}
|
|
3296
|
+
|
|
3297
|
+
private static recordsWriteProtocol(message: GenericMessage | undefined): string | undefined {
|
|
3298
|
+
if (!SyncEngineLevel.isRecordsWriteProtocolMessage(message)) {
|
|
3299
|
+
return undefined;
|
|
3300
|
+
}
|
|
3301
|
+
|
|
3302
|
+
const { protocol } = message.descriptor;
|
|
3303
|
+
return typeof protocol === 'string' ? protocol : undefined;
|
|
3304
|
+
}
|
|
3305
|
+
|
|
3306
|
+
private static recordsDeleteRecordId(message: GenericMessage): string | undefined {
|
|
3307
|
+
if (!SyncEngineLevel.isRecordsDeleteMessage(message)) {
|
|
3308
|
+
return undefined;
|
|
3309
|
+
}
|
|
3310
|
+
|
|
3311
|
+
const recordId = (message.descriptor as Record<string, unknown>).recordId;
|
|
3312
|
+
return typeof recordId === 'string' ? recordId : undefined;
|
|
3313
|
+
}
|
|
3314
|
+
|
|
3315
|
+
private static protocolConfigRootRecordsWrite(
|
|
3316
|
+
primary: GenericMessage,
|
|
3317
|
+
initialWrite: RecordsWriteMessage | undefined,
|
|
3318
|
+
): RecordsWriteMessage | undefined {
|
|
3319
|
+
if (SyncEngineLevel.isRecordsWriteMessage(primary)) {
|
|
3320
|
+
return primary;
|
|
3321
|
+
}
|
|
3322
|
+
|
|
3323
|
+
return SyncEngineLevel.isRecordsDeleteMessage(primary) ? initialWrite : undefined;
|
|
3324
|
+
}
|
|
3325
|
+
|
|
3326
|
+
private static protocolsConfigureProtocol(message: ProtocolsConfigureMessage): string {
|
|
3327
|
+
return message.descriptor.definition.protocol;
|
|
3328
|
+
}
|
|
3329
|
+
|
|
3330
|
+
private static protocolsConfigureProtocolFromGenericMessage(message: GenericMessage): string | undefined {
|
|
3331
|
+
if (!SyncEngineLevel.isProtocolsConfigureDefinitionMessage(message)) {
|
|
3332
|
+
return undefined;
|
|
3333
|
+
}
|
|
3334
|
+
|
|
3335
|
+
return message.descriptor.definition.protocol;
|
|
3336
|
+
}
|
|
3337
|
+
|
|
3338
|
+
private static protocolsConfigureIsNotNewerThanRecordsWrite(
|
|
3339
|
+
protocolsConfigureMessage: GenericMessage,
|
|
3340
|
+
recordsWriteMessage: GenericMessage,
|
|
3341
|
+
): boolean {
|
|
3342
|
+
return protocolsConfigureMessage.descriptor.messageTimestamp <= recordsWriteMessage.descriptor.messageTimestamp;
|
|
3343
|
+
}
|
|
3344
|
+
|
|
3345
|
+
private static protocolsConfigureUses(message: ProtocolsConfigureMessage): string[] {
|
|
3346
|
+
const uses = message.descriptor.definition?.uses;
|
|
3347
|
+
return uses === undefined
|
|
3348
|
+
? []
|
|
3349
|
+
: Object.values(uses).filter((protocol): protocol is string => typeof protocol === 'string');
|
|
3350
|
+
}
|
|
3351
|
+
|
|
3352
|
+
private static hasMessage<T extends MessagesSyncDiffEntry>(
|
|
3353
|
+
entry: T | undefined,
|
|
3354
|
+
): entry is T & { message: GenericMessage } {
|
|
3355
|
+
return entry?.message !== undefined;
|
|
3356
|
+
}
|
|
3357
|
+
|
|
3358
|
+
private static isRecordsWriteProtocolMessage(message: GenericMessage | undefined): message is RecordsWriteProtocolMessage {
|
|
3359
|
+
return message?.descriptor.interface === DwnInterfaceName.Records &&
|
|
3360
|
+
message.descriptor.method === DwnMethodName.Write;
|
|
3361
|
+
}
|
|
3362
|
+
|
|
3363
|
+
private static isRecordsWriteMessage(message: GenericMessage | undefined): message is RecordsWriteMessage {
|
|
3364
|
+
return SyncEngineLevel.isRecordsWriteProtocolMessage(message) &&
|
|
3365
|
+
'recordId' in message &&
|
|
3366
|
+
typeof message.recordId === 'string' &&
|
|
3367
|
+
'contextId' in message &&
|
|
3368
|
+
typeof message.contextId === 'string';
|
|
3369
|
+
}
|
|
3370
|
+
|
|
3371
|
+
private static isRecordsDeleteMessage(message: GenericMessage): boolean {
|
|
3372
|
+
return message.descriptor.interface === DwnInterfaceName.Records &&
|
|
3373
|
+
message.descriptor.method === DwnMethodName.Delete;
|
|
3374
|
+
}
|
|
3375
|
+
|
|
3376
|
+
private static isProtocolsConfigureDefinitionMessage(message: GenericMessage): message is ProtocolsConfigureMessage {
|
|
3377
|
+
return message.descriptor.interface === DwnInterfaceName.Protocols &&
|
|
3378
|
+
message.descriptor.method === DwnMethodName.Configure &&
|
|
3379
|
+
message.authorization !== undefined &&
|
|
3380
|
+
SyncEngineLevel.hasProtocolsConfigureDefinition(message.descriptor);
|
|
3381
|
+
}
|
|
3382
|
+
|
|
3383
|
+
private static hasProtocolsConfigureDefinition(
|
|
3384
|
+
descriptor: MaybeProtocolsConfigureDefinitionDescriptor,
|
|
3385
|
+
): descriptor is ProtocolsConfigureDefinitionDescriptor {
|
|
3386
|
+
return SyncEngineLevel.isProtocolsConfigureDefinition(descriptor.definition);
|
|
3387
|
+
}
|
|
3388
|
+
|
|
3389
|
+
private static isProtocolsConfigureDefinition(
|
|
3390
|
+
definition: unknown,
|
|
3391
|
+
): definition is ProtocolsConfigureDefinition {
|
|
3392
|
+
return typeof definition === 'object' &&
|
|
3393
|
+
definition !== null &&
|
|
3394
|
+
'protocol' in definition &&
|
|
3395
|
+
typeof definition.protocol === 'string';
|
|
3396
|
+
}
|
|
3397
|
+
|
|
3398
|
+
private static filterProtocolConfigClosure(
|
|
3399
|
+
primaryProtocol: string,
|
|
3400
|
+
candidates: AuthenticatedProtocolConfigDependency[],
|
|
3401
|
+
): MessagesSyncDependencyEntry[] {
|
|
3402
|
+
// Start from the primary record's protocol and walk only protocols named by
|
|
3403
|
+
// accepted, signed config definitions. This keeps composed-protocol support
|
|
3404
|
+
// narrow: the governing config can admit its `uses` targets, but arbitrary
|
|
3405
|
+
// protocol config hints cannot enter the apply set.
|
|
3406
|
+
const candidatesByProtocol = SyncEngineLevel.groupGoverningProtocolConfigCandidatesByProtocol(candidates);
|
|
3407
|
+
const visitedProtocols = new Set<string>();
|
|
3408
|
+
const pendingProtocols = [primaryProtocol];
|
|
3409
|
+
const accepted = new Map<string, MessagesSyncDependencyEntry>();
|
|
3410
|
+
|
|
3411
|
+
for (
|
|
3412
|
+
let protocol = SyncEngineLevel.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols);
|
|
3413
|
+
protocol !== undefined;
|
|
3414
|
+
protocol = SyncEngineLevel.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols)
|
|
3415
|
+
) {
|
|
3416
|
+
SyncEngineLevel.acceptProtocolConfigCandidates({
|
|
3417
|
+
protocol,
|
|
3418
|
+
candidatesByProtocol,
|
|
3419
|
+
visitedProtocols,
|
|
3420
|
+
pendingProtocols,
|
|
3421
|
+
accepted,
|
|
3422
|
+
});
|
|
3423
|
+
}
|
|
3424
|
+
|
|
3425
|
+
return [...accepted.values()];
|
|
3426
|
+
}
|
|
3427
|
+
|
|
3428
|
+
private static groupGoverningProtocolConfigCandidatesByProtocol(
|
|
3429
|
+
candidates: AuthenticatedProtocolConfigDependency[],
|
|
3430
|
+
): Map<string, AuthenticatedProtocolConfigDependency> {
|
|
3431
|
+
const candidatesByProtocol = new Map<string, AuthenticatedProtocolConfigDependency>();
|
|
3432
|
+
for (const candidate of candidates) {
|
|
3433
|
+
const protocol = SyncEngineLevel.protocolsConfigureProtocol(candidate.message);
|
|
3434
|
+
const existing = candidatesByProtocol.get(protocol);
|
|
3435
|
+
if (existing !== undefined && SyncEngineLevel.isProtocolConfigCandidateAtLeastAsNew(existing, candidate)) {
|
|
3436
|
+
continue;
|
|
3437
|
+
}
|
|
3438
|
+
|
|
3439
|
+
candidatesByProtocol.set(protocol, candidate);
|
|
3440
|
+
}
|
|
3441
|
+
return candidatesByProtocol;
|
|
3442
|
+
}
|
|
3443
|
+
|
|
3444
|
+
private static isProtocolConfigCandidateAtLeastAsNew(
|
|
3445
|
+
existing: AuthenticatedProtocolConfigDependency,
|
|
3446
|
+
candidate: AuthenticatedProtocolConfigDependency,
|
|
3447
|
+
): boolean {
|
|
3448
|
+
const existingTimestamp = existing.message.descriptor.messageTimestamp;
|
|
3449
|
+
const candidateTimestamp = candidate.message.descriptor.messageTimestamp;
|
|
3450
|
+
if (existingTimestamp !== candidateTimestamp) {
|
|
3451
|
+
return existingTimestamp > candidateTimestamp;
|
|
3452
|
+
}
|
|
3453
|
+
return lexicographicalCompare(existing.messageCid, candidate.messageCid) >= 0;
|
|
3454
|
+
}
|
|
3455
|
+
|
|
3456
|
+
private static takeNextUnvisitedProtocol(
|
|
3457
|
+
pendingProtocols: string[],
|
|
3458
|
+
visitedProtocols: Set<string>,
|
|
3459
|
+
): string | undefined {
|
|
3460
|
+
while (pendingProtocols.length > 0) {
|
|
3461
|
+
const protocol = pendingProtocols.shift()!;
|
|
3462
|
+
if (visitedProtocols.has(protocol)) {
|
|
3463
|
+
continue;
|
|
3464
|
+
}
|
|
3465
|
+
visitedProtocols.add(protocol);
|
|
3466
|
+
return protocol;
|
|
3467
|
+
}
|
|
3468
|
+
return undefined;
|
|
3469
|
+
}
|
|
3470
|
+
|
|
3471
|
+
private static acceptProtocolConfigCandidates({
|
|
3472
|
+
protocol,
|
|
3473
|
+
candidatesByProtocol,
|
|
3474
|
+
visitedProtocols,
|
|
3475
|
+
pendingProtocols,
|
|
3476
|
+
accepted,
|
|
3477
|
+
}: {
|
|
3478
|
+
protocol: string;
|
|
3479
|
+
candidatesByProtocol: Map<string, AuthenticatedProtocolConfigDependency>;
|
|
3480
|
+
visitedProtocols: Set<string>;
|
|
3481
|
+
pendingProtocols: string[];
|
|
3482
|
+
accepted: Map<string, MessagesSyncDependencyEntry>;
|
|
3483
|
+
}): void {
|
|
3484
|
+
const candidate = candidatesByProtocol.get(protocol);
|
|
3485
|
+
if (candidate === undefined) {
|
|
3486
|
+
return;
|
|
3487
|
+
}
|
|
3488
|
+
|
|
3489
|
+
accepted.set(candidate.messageCid, candidate);
|
|
3490
|
+
SyncEngineLevel.queueUnvisitedProtocols(
|
|
3491
|
+
SyncEngineLevel.protocolsConfigureUses(candidate.message),
|
|
3492
|
+
visitedProtocols,
|
|
3493
|
+
pendingProtocols,
|
|
3494
|
+
);
|
|
3495
|
+
}
|
|
3496
|
+
|
|
3497
|
+
private static queueUnvisitedProtocols(
|
|
3498
|
+
protocols: string[],
|
|
3499
|
+
visitedProtocols: Set<string>,
|
|
3500
|
+
pendingProtocols: string[],
|
|
3501
|
+
): void {
|
|
3502
|
+
for (const protocol of protocols) {
|
|
3503
|
+
if (!visitedProtocols.has(protocol)) {
|
|
3504
|
+
pendingProtocols.push(protocol);
|
|
3505
|
+
}
|
|
3506
|
+
}
|
|
3507
|
+
}
|
|
3508
|
+
|
|
3509
|
+
private static protocolSetScopeForProjectedDependencies(
|
|
3510
|
+
dependencies: MessagesSyncDependencyEntry[],
|
|
3511
|
+
): Extract<SyncScope, { kind: 'protocolSet' }> {
|
|
3512
|
+
// Verification above is the security boundary. This protocolSet scope only
|
|
3513
|
+
// routes already-verified config dependencies through the existing
|
|
3514
|
+
// pull/apply path, which expects every prefetched message to be accepted by
|
|
3515
|
+
// the supplied sync scope before it reaches processRawMessage().
|
|
3516
|
+
const protocols = SyncEngineLevel.dedupeStrings(
|
|
3517
|
+
dependencies
|
|
3518
|
+
.flatMap(dependency => SyncEngineLevel.projectedDependencyProtocols(dependency))
|
|
3519
|
+
).sort(lexicographicalCompare);
|
|
3520
|
+
if (protocols.length === 0) {
|
|
3521
|
+
throw new Error('SyncEngineLevel: projected dependency hints contained no protocols.');
|
|
3522
|
+
}
|
|
3523
|
+
|
|
3524
|
+
return {
|
|
3525
|
+
kind : 'protocolSet',
|
|
3526
|
+
protocols : protocols as NonEmptyStringArray,
|
|
3527
|
+
};
|
|
3528
|
+
}
|
|
3529
|
+
|
|
3530
|
+
private static projectedDependencyProtocols(dependency: MessagesSyncDependencyEntry): string[] {
|
|
3531
|
+
if (dependency.message === undefined) {
|
|
3532
|
+
return [];
|
|
3533
|
+
}
|
|
3534
|
+
|
|
3535
|
+
const protocol = dependency.dependencyClass === 'protocolsConfigure'
|
|
3536
|
+
? SyncEngineLevel.protocolsConfigureProtocolFromGenericMessage(dependency.message)
|
|
3537
|
+
: SyncEngineLevel.recordsWriteProtocol(dependency.message);
|
|
3538
|
+
return protocol === undefined ? [] : [protocol];
|
|
3539
|
+
}
|
|
3540
|
+
|
|
3541
|
+
private static hasDependencyPayloadBytes(dependency: MessagesSyncDependencyEntry): boolean {
|
|
3542
|
+
if (dependency.encodedData !== undefined) {
|
|
3543
|
+
return true;
|
|
3544
|
+
}
|
|
3545
|
+
|
|
3546
|
+
const message = dependency.message as Record<string, unknown> | undefined;
|
|
3547
|
+
return message !== undefined && 'encodedData' in message;
|
|
3548
|
+
}
|
|
3549
|
+
|
|
3550
|
+
private static dedupeDependencyEntries(
|
|
3551
|
+
dependencies: Iterable<MessagesSyncDependencyEntry>,
|
|
3552
|
+
): MessagesSyncDependencyEntry[] {
|
|
3553
|
+
const deduped = new Map<string, MessagesSyncDependencyEntry>();
|
|
3554
|
+
for (const dependency of dependencies) {
|
|
3555
|
+
deduped.set(dependency.messageCid, dependency);
|
|
3556
|
+
}
|
|
3557
|
+
return [...deduped.values()];
|
|
3558
|
+
}
|
|
3559
|
+
|
|
3560
|
+
private async pushLocalDiffEntries(
|
|
3561
|
+
target: ProjectionReconcileTarget,
|
|
3562
|
+
onlyLocal: string[],
|
|
3563
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3564
|
+
shouldContinue?: () => boolean,
|
|
3565
|
+
): Promise<boolean> {
|
|
3566
|
+
await this.pushMessages({
|
|
3567
|
+
did : target.did,
|
|
3568
|
+
dwnUrl : target.dwnUrl,
|
|
3569
|
+
delegateDid : target.delegateDid,
|
|
3570
|
+
permissionGrantIds,
|
|
3571
|
+
messageCids : SyncEngineLevel.dedupeStrings(onlyLocal),
|
|
3572
|
+
});
|
|
3573
|
+
return shouldContinue?.() === false;
|
|
3574
|
+
}
|
|
3575
|
+
|
|
3576
|
+
private async verifyProtocolSetConvergence(
|
|
3577
|
+
target: ProjectionReconcileTarget,
|
|
3578
|
+
changedProtocols: string[],
|
|
3579
|
+
permissionGrantIds: NonEmptyStringArray | undefined,
|
|
3580
|
+
shouldContinue?: () => boolean,
|
|
3581
|
+
): Promise<ProjectionReconcileResult> {
|
|
3582
|
+
for (const protocol of changedProtocols) {
|
|
3583
|
+
const roots = await this.getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue);
|
|
3584
|
+
if (!roots) { return { aborted: true }; }
|
|
3585
|
+
|
|
3586
|
+
if (roots.localRoot !== roots.remoteRoot) {
|
|
3587
|
+
return { converged: false };
|
|
3588
|
+
}
|
|
3589
|
+
}
|
|
3590
|
+
|
|
3591
|
+
return { converged: true };
|
|
3592
|
+
}
|
|
3593
|
+
|
|
3594
|
+
private static dedupeRemoteEntries(entries: MessagesSyncDiffEntry[]): MessagesSyncDiffEntry[] {
|
|
3595
|
+
const seen = new Set<string>();
|
|
3596
|
+
const unique: MessagesSyncDiffEntry[] = [];
|
|
3597
|
+
for (const entry of entries) {
|
|
3598
|
+
if (seen.has(entry.messageCid)) {
|
|
3599
|
+
continue;
|
|
3600
|
+
}
|
|
3601
|
+
seen.add(entry.messageCid);
|
|
3602
|
+
unique.push(entry);
|
|
3603
|
+
}
|
|
3604
|
+
return unique;
|
|
2192
3605
|
}
|
|
2193
3606
|
|
|
2194
|
-
private
|
|
2195
|
-
return new
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
3607
|
+
private static dedupeStrings(values: string[]): string[] {
|
|
3608
|
+
return [...new Set(values)];
|
|
3609
|
+
}
|
|
3610
|
+
|
|
3611
|
+
private async clearRootConvergenceDeadLettersForScope(
|
|
3612
|
+
tenantDid: string,
|
|
3613
|
+
remoteEndpoint: string,
|
|
3614
|
+
scope: SyncScope,
|
|
3615
|
+
): Promise<void> {
|
|
3616
|
+
if (scope.kind === 'recordsProjection' || (scope.kind === 'protocolSet' && scope.protocols.length > 1)) {
|
|
3617
|
+
// Batched multi-protocol and projected pulls pass the full scope to
|
|
3618
|
+
// pullMessages, so pull dead letters can be recorded without a single
|
|
3619
|
+
// protocol bucket.
|
|
3620
|
+
await this.clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint);
|
|
3621
|
+
}
|
|
3622
|
+
|
|
3623
|
+
for (const protocol of this.getReconcileProtocols(scope)) {
|
|
3624
|
+
await this.clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint, protocol);
|
|
3625
|
+
}
|
|
2203
3626
|
}
|
|
2204
3627
|
|
|
2205
3628
|
// ---------------------------------------------------------------------------
|
|
@@ -2277,21 +3700,30 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2277
3700
|
// closure's captured `link` reference may no longer be the active
|
|
2278
3701
|
// link object. Bail before mutating the replacement's state.
|
|
2279
3702
|
const isStaleLink = (): boolean => this._activeLinks.get(linkKey) !== link;
|
|
3703
|
+
const shouldContinue = (): boolean =>
|
|
3704
|
+
this._engineGeneration === generation &&
|
|
3705
|
+
!isStaleLink() &&
|
|
3706
|
+
link.status === 'live';
|
|
2280
3707
|
|
|
2281
|
-
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid,
|
|
3708
|
+
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, scope, authorization } = link;
|
|
3709
|
+
const eventScope = syncEventScope(scope);
|
|
2282
3710
|
|
|
2283
3711
|
try {
|
|
2284
|
-
const reconcileOutcome = await this.
|
|
2285
|
-
|
|
2286
|
-
|
|
3712
|
+
const reconcileOutcome = await this.reconcileProjectionTarget({
|
|
3713
|
+
did,
|
|
3714
|
+
dwnUrl,
|
|
3715
|
+
delegateDid,
|
|
3716
|
+
scope,
|
|
3717
|
+
authorization,
|
|
3718
|
+
}, { verifyConvergence: true }, shouldContinue);
|
|
2287
3719
|
if (reconcileOutcome.aborted || isStaleLink()) { return; }
|
|
2288
3720
|
|
|
2289
3721
|
if (reconcileOutcome.converged) {
|
|
2290
3722
|
await this.ledger.clearNeedsReconcile(link);
|
|
2291
|
-
// SMT roots match
|
|
2292
|
-
//
|
|
2293
|
-
|
|
2294
|
-
this.emitEvent({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
3723
|
+
// SMT roots match, so transport/apply failures for this link may no
|
|
3724
|
+
// longer be current. Closure failures are not cleared by root equality.
|
|
3725
|
+
await this.clearRootConvergenceDeadLettersForScope(did, dwnUrl, scope);
|
|
3726
|
+
this.emitEvent({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope });
|
|
2295
3727
|
} else {
|
|
2296
3728
|
// Roots still differ — retry after a delay. This can happen when
|
|
2297
3729
|
// pushMessages() had permanent failures, pullMessages() partially
|
|
@@ -2311,6 +3743,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2311
3743
|
dwnUrl: string;
|
|
2312
3744
|
delegateDid?: string;
|
|
2313
3745
|
protocol?: string;
|
|
3746
|
+
permissionGrantIds?: NonEmptyStringArray;
|
|
2314
3747
|
}): PushRuntimeState {
|
|
2315
3748
|
let pushRuntime = this._pushRuntimes.get(linkKey);
|
|
2316
3749
|
if (!pushRuntime) {
|
|
@@ -2335,65 +3768,11 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2335
3768
|
* Live-mode subscription methods (`openLivePullSubscription`,
|
|
2336
3769
|
* `openLocalPushSubscription`) receive `linkKey` directly and never
|
|
2337
3770
|
* call this. The remaining callers are poll-mode `sync()` and the
|
|
2338
|
-
* live-mode startup/error paths that already have
|
|
2339
|
-
*
|
|
2340
|
-
* The `undefined` fallback (which produces a legacy cursor key) exists
|
|
2341
|
-
* only for the no-protocol full-tenant targets in poll mode.
|
|
2342
|
-
*/
|
|
2343
|
-
private buildLinkKey(did: string, dwnUrl: string, scopeIdOrProtocol?: string): string {
|
|
2344
|
-
return scopeIdOrProtocol ? buildLinkId(did, dwnUrl, scopeIdOrProtocol) : buildLegacyCursorKey(did, dwnUrl);
|
|
2345
|
-
}
|
|
2346
|
-
|
|
2347
|
-
/**
|
|
2348
|
-
* @deprecated Used by poll-mode sync and one-time migration only. Live mode
|
|
2349
|
-
* uses ReplicationLedger checkpoints. Handles migration from old string cursors:
|
|
2350
|
-
* if the stored value is a bare string (pre-ProgressToken format), it is treated
|
|
2351
|
-
* as absent — the sync engine will do a full SMT reconciliation on first startup
|
|
2352
|
-
* after upgrade, which is correct and safe.
|
|
2353
|
-
*/
|
|
2354
|
-
private async getCursor(key: string): Promise<ProgressToken | undefined> {
|
|
2355
|
-
const cursors = this._db.sublevel('syncCursors');
|
|
2356
|
-
try {
|
|
2357
|
-
const raw = await cursors.get(key);
|
|
2358
|
-
try {
|
|
2359
|
-
const parsed = JSON.parse(raw);
|
|
2360
|
-
if (parsed && typeof parsed === 'object' &&
|
|
2361
|
-
typeof parsed.streamId === 'string' && parsed.streamId.length > 0 &&
|
|
2362
|
-
typeof parsed.epoch === 'string' && parsed.epoch.length > 0 &&
|
|
2363
|
-
typeof parsed.position === 'string' && parsed.position.length > 0 &&
|
|
2364
|
-
typeof parsed.messageCid === 'string' && parsed.messageCid.length > 0) {
|
|
2365
|
-
return parsed as ProgressToken;
|
|
2366
|
-
}
|
|
2367
|
-
} catch {
|
|
2368
|
-
// Not valid JSON (old string cursor) — fall through to delete.
|
|
2369
|
-
}
|
|
2370
|
-
// Entry exists but is unparseable or has invalid/empty fields. Delete it
|
|
2371
|
-
// so subsequent startups don't re-check it on every launch.
|
|
2372
|
-
await this.deleteLegacyCursor(key);
|
|
2373
|
-
return undefined;
|
|
2374
|
-
} catch (error) {
|
|
2375
|
-
const e = error as { code: string };
|
|
2376
|
-
if (e.code === 'LEVEL_NOT_FOUND') {
|
|
2377
|
-
return undefined;
|
|
2378
|
-
}
|
|
2379
|
-
throw error;
|
|
2380
|
-
}
|
|
2381
|
-
}
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
/**
|
|
2385
|
-
* Delete a legacy cursor from the old syncCursors sublevel.
|
|
2386
|
-
* Called as part of one-time migration to ReplicationLedger.
|
|
3771
|
+
* live-mode startup/error paths that already have a projection ID and
|
|
3772
|
+
* authorization epoch.
|
|
2387
3773
|
*/
|
|
2388
|
-
private
|
|
2389
|
-
|
|
2390
|
-
try {
|
|
2391
|
-
await cursors.del(key);
|
|
2392
|
-
} catch {
|
|
2393
|
-
// Best-effort — ignore LEVEL_NOT_FOUND and transient I/O errors alike.
|
|
2394
|
-
// A failed delete leaves the bad entry for one more re-check on the
|
|
2395
|
-
// next startup, which is harmless.
|
|
2396
|
-
}
|
|
3774
|
+
private buildLinkKey(did: string, dwnUrl: string, projectionId: string, authorizationEpoch: string): string {
|
|
3775
|
+
return buildLinkId(did, dwnUrl, projectionId, authorizationEpoch);
|
|
2397
3776
|
}
|
|
2398
3777
|
|
|
2399
3778
|
// ---------------------------------------------------------------------------
|
|
@@ -2492,7 +3871,12 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2492
3871
|
*
|
|
2493
3872
|
* Returns a hex-encoded root hash string.
|
|
2494
3873
|
*/
|
|
2495
|
-
private async getLocalRoot(
|
|
3874
|
+
private async getLocalRoot(
|
|
3875
|
+
did: string,
|
|
3876
|
+
delegateDid?: string,
|
|
3877
|
+
protocol?: string,
|
|
3878
|
+
permissionGrantIds?: string[],
|
|
3879
|
+
): Promise<string> {
|
|
2496
3880
|
const si = this.stateIndex;
|
|
2497
3881
|
if (si) {
|
|
2498
3882
|
const rootHash = protocol === undefined
|
|
@@ -2502,16 +3886,15 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2502
3886
|
}
|
|
2503
3887
|
|
|
2504
3888
|
// Remote mode fallback: go through processRequest → RPC.
|
|
2505
|
-
const permissionGrantId = await this.getSyncPermissionGrantId(did, delegateDid, protocol);
|
|
2506
3889
|
const response = await this.agent.dwn.processRequest({
|
|
2507
3890
|
author : did,
|
|
2508
3891
|
target : did,
|
|
2509
3892
|
messageType : DwnInterface.MessagesSync,
|
|
2510
3893
|
granteeDid : delegateDid,
|
|
2511
3894
|
messageParams : {
|
|
2512
|
-
action: 'root',
|
|
3895
|
+
action : 'root',
|
|
2513
3896
|
protocol,
|
|
2514
|
-
|
|
3897
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
|
|
2515
3898
|
}
|
|
2516
3899
|
});
|
|
2517
3900
|
const reply = response.reply as MessagesSyncReply;
|
|
@@ -2522,9 +3905,13 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2522
3905
|
* Get the SMT root hash from a remote DWN via a MessagesSync 'root' action.
|
|
2523
3906
|
* Returns a hex-encoded root hash string.
|
|
2524
3907
|
*/
|
|
2525
|
-
private async getRemoteRoot(
|
|
2526
|
-
|
|
2527
|
-
|
|
3908
|
+
private async getRemoteRoot(
|
|
3909
|
+
did: string,
|
|
3910
|
+
dwnUrl: string,
|
|
3911
|
+
delegateDid?: string,
|
|
3912
|
+
protocol?: string,
|
|
3913
|
+
permissionGrantIds?: string[],
|
|
3914
|
+
): Promise<string> {
|
|
2528
3915
|
const syncMessage = await this.agent.dwn.processRequest({
|
|
2529
3916
|
store : false,
|
|
2530
3917
|
author : did,
|
|
@@ -2532,9 +3919,71 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2532
3919
|
messageType : DwnInterface.MessagesSync,
|
|
2533
3920
|
granteeDid : delegateDid,
|
|
2534
3921
|
messageParams : {
|
|
2535
|
-
action: 'root',
|
|
3922
|
+
action : 'root',
|
|
2536
3923
|
protocol,
|
|
2537
|
-
|
|
3924
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
|
|
3925
|
+
}
|
|
3926
|
+
});
|
|
3927
|
+
|
|
3928
|
+
const reply = await this.agent.rpc.sendDwnRequest({
|
|
3929
|
+
dwnUrl,
|
|
3930
|
+
targetDid : did,
|
|
3931
|
+
message : syncMessage.message,
|
|
3932
|
+
}) as MessagesSyncReply;
|
|
3933
|
+
|
|
3934
|
+
return reply.root ?? '';
|
|
3935
|
+
}
|
|
3936
|
+
|
|
3937
|
+
private async getLocalProjectedRoot(
|
|
3938
|
+
did: string,
|
|
3939
|
+
delegateDid: string | undefined,
|
|
3940
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
|
|
3941
|
+
permissionGrantIds?: string[],
|
|
3942
|
+
): Promise<string> {
|
|
3943
|
+
if (this.stateIndex) {
|
|
3944
|
+
// Local projected roots use the already-derived scope directly. The
|
|
3945
|
+
// remote root/diff request still re-authorizes the invoked grant set.
|
|
3946
|
+
return RecordsProjection.getRootHex({
|
|
3947
|
+
tenant : did,
|
|
3948
|
+
messageStore : this.agent.dwn.node.storage.messageStore,
|
|
3949
|
+
scopes,
|
|
3950
|
+
});
|
|
3951
|
+
}
|
|
3952
|
+
|
|
3953
|
+
const response = await this.agent.dwn.processRequest({
|
|
3954
|
+
author : did,
|
|
3955
|
+
target : did,
|
|
3956
|
+
messageType : DwnInterface.MessagesSync,
|
|
3957
|
+
granteeDid : delegateDid,
|
|
3958
|
+
messageParams : {
|
|
3959
|
+
action : 'root',
|
|
3960
|
+
projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
|
|
3961
|
+
projectionScopes : [...scopes],
|
|
3962
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
|
|
3963
|
+
}
|
|
3964
|
+
});
|
|
3965
|
+
const reply = response.reply as MessagesSyncReply;
|
|
3966
|
+
return reply.root ?? '';
|
|
3967
|
+
}
|
|
3968
|
+
|
|
3969
|
+
private async getRemoteProjectedRoot(
|
|
3970
|
+
did: string,
|
|
3971
|
+
dwnUrl: string,
|
|
3972
|
+
delegateDid: string | undefined,
|
|
3973
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
|
|
3974
|
+
permissionGrantIds?: string[],
|
|
3975
|
+
): Promise<string> {
|
|
3976
|
+
const syncMessage = await this.agent.dwn.processRequest({
|
|
3977
|
+
store : false,
|
|
3978
|
+
author : did,
|
|
3979
|
+
target : did,
|
|
3980
|
+
messageType : DwnInterface.MessagesSync,
|
|
3981
|
+
granteeDid : delegateDid,
|
|
3982
|
+
messageParams : {
|
|
3983
|
+
action : 'root',
|
|
3984
|
+
projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
|
|
3985
|
+
projectionScopes : [...scopes],
|
|
3986
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
|
|
2538
3987
|
}
|
|
2539
3988
|
});
|
|
2540
3989
|
|
|
@@ -2565,55 +4014,102 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2565
4014
|
*
|
|
2566
4015
|
* This replaces `walkTreeDiff()` which required one HTTP call per tree node.
|
|
2567
4016
|
*/
|
|
2568
|
-
private async diffWithRemote({ did, dwnUrl, delegateDid, protocol }: {
|
|
4017
|
+
private async diffWithRemote({ did, dwnUrl, delegateDid, protocol, permissionGrantIds }: {
|
|
2569
4018
|
did: string;
|
|
2570
4019
|
dwnUrl: string;
|
|
2571
4020
|
delegateDid?: string;
|
|
2572
4021
|
protocol?: string;
|
|
2573
|
-
|
|
4022
|
+
permissionGrantIds?: string[];
|
|
4023
|
+
}): Promise<ProjectionDiffResult> {
|
|
2574
4024
|
// Step 1: Collect local subtree hashes at BATCHED_DIFF_DEPTH directly from StateIndex.
|
|
2575
|
-
const localHashes = await this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH);
|
|
4025
|
+
const localHashes = await this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH, permissionGrantIds);
|
|
2576
4026
|
|
|
2577
4027
|
// Step 2: Send a single 'diff' request to the remote with our hashes.
|
|
2578
|
-
const
|
|
4028
|
+
const messageParams: DwnMessageParams[DwnInterface.MessagesSync] = {
|
|
4029
|
+
action : 'diff',
|
|
4030
|
+
protocol,
|
|
4031
|
+
hashes : localHashes,
|
|
4032
|
+
depth : BATCHED_DIFF_DEPTH,
|
|
4033
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
|
|
4034
|
+
};
|
|
4035
|
+
|
|
4036
|
+
// Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
|
|
4037
|
+
// Reuse the same grant set from step 2.
|
|
4038
|
+
return this.diffRemoteMessages(
|
|
4039
|
+
{ did, dwnUrl, delegateDid },
|
|
4040
|
+
messageParams,
|
|
4041
|
+
prefix => this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantIds),
|
|
4042
|
+
'diff',
|
|
4043
|
+
);
|
|
4044
|
+
}
|
|
4045
|
+
|
|
4046
|
+
private async diffProjectedWithRemote({ did, dwnUrl, delegateDid, scopes, permissionGrantIds }: {
|
|
4047
|
+
did: string;
|
|
4048
|
+
dwnUrl: string;
|
|
4049
|
+
delegateDid?: string;
|
|
4050
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]];
|
|
4051
|
+
permissionGrantIds?: string[];
|
|
4052
|
+
}): Promise<ProjectionDiffResult> {
|
|
4053
|
+
const localHashes = await this.collectLocalProjectedSubtreeHashes(
|
|
4054
|
+
did,
|
|
4055
|
+
scopes,
|
|
4056
|
+
BATCHED_DIFF_DEPTH,
|
|
4057
|
+
delegateDid,
|
|
4058
|
+
permissionGrantIds,
|
|
4059
|
+
);
|
|
4060
|
+
|
|
4061
|
+
const messageParams: DwnMessageParams[DwnInterface.MessagesSync] = {
|
|
4062
|
+
action : 'diff',
|
|
4063
|
+
projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
|
|
4064
|
+
projectionScopes : [...scopes],
|
|
4065
|
+
hashes : localHashes,
|
|
4066
|
+
depth : BATCHED_DIFF_DEPTH,
|
|
4067
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
|
|
4068
|
+
};
|
|
4069
|
+
|
|
4070
|
+
return this.diffRemoteMessages(
|
|
4071
|
+
{ did, dwnUrl, delegateDid },
|
|
4072
|
+
messageParams,
|
|
4073
|
+
prefix => this.getLocalProjectedLeaves(did, prefix, delegateDid, scopes, permissionGrantIds),
|
|
4074
|
+
'projected diff',
|
|
4075
|
+
);
|
|
4076
|
+
}
|
|
2579
4077
|
|
|
4078
|
+
private async diffRemoteMessages(
|
|
4079
|
+
target: Pick<ProjectionReconcileTarget, 'did' | 'dwnUrl' | 'delegateDid'>,
|
|
4080
|
+
messageParams: DwnMessageParams[DwnInterface.MessagesSync],
|
|
4081
|
+
getLocalLeavesForPrefix: (prefix: string) => Promise<string[]>,
|
|
4082
|
+
operationName: string,
|
|
4083
|
+
): Promise<ProjectionDiffResult> {
|
|
2580
4084
|
const syncMessage = await this.agent.dwn.processRequest({
|
|
2581
|
-
store
|
|
2582
|
-
author
|
|
2583
|
-
target
|
|
2584
|
-
messageType
|
|
2585
|
-
granteeDid
|
|
2586
|
-
messageParams
|
|
2587
|
-
action : 'diff',
|
|
2588
|
-
protocol,
|
|
2589
|
-
hashes : localHashes,
|
|
2590
|
-
depth : BATCHED_DIFF_DEPTH,
|
|
2591
|
-
permissionGrantId,
|
|
2592
|
-
}
|
|
4085
|
+
store : false,
|
|
4086
|
+
author : target.did,
|
|
4087
|
+
target : target.did,
|
|
4088
|
+
messageType : DwnInterface.MessagesSync,
|
|
4089
|
+
granteeDid : target.delegateDid,
|
|
4090
|
+
messageParams,
|
|
2593
4091
|
});
|
|
2594
4092
|
|
|
2595
4093
|
const reply = await this.agent.rpc.sendDwnRequest({
|
|
2596
|
-
dwnUrl,
|
|
2597
|
-
targetDid : did,
|
|
4094
|
+
dwnUrl : target.dwnUrl,
|
|
4095
|
+
targetDid : target.did,
|
|
2598
4096
|
message : syncMessage.message,
|
|
2599
4097
|
}) as MessagesSyncReply;
|
|
2600
4098
|
|
|
2601
4099
|
if (reply.status.code !== 200) {
|
|
2602
|
-
throw new Error(`SyncEngineLevel:
|
|
4100
|
+
throw new Error(`SyncEngineLevel: ${operationName} failed with ${reply.status.code}: ${reply.status.detail}`);
|
|
2603
4101
|
}
|
|
2604
4102
|
|
|
2605
|
-
// Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
|
|
2606
|
-
// Reuse the same grant ID from step 2 (avoids redundant lookup).
|
|
2607
|
-
const permissionGrantIdForLeaves = permissionGrantId;
|
|
2608
4103
|
const onlyLocalCids: string[] = [];
|
|
2609
4104
|
for (const prefix of reply.onlyLocal ?? []) {
|
|
2610
|
-
const leaves = await
|
|
4105
|
+
const leaves = await getLocalLeavesForPrefix(prefix);
|
|
2611
4106
|
onlyLocalCids.push(...leaves);
|
|
2612
4107
|
}
|
|
2613
4108
|
|
|
2614
4109
|
return {
|
|
2615
|
-
|
|
2616
|
-
|
|
4110
|
+
dependencies : reply.dependencies ?? [],
|
|
4111
|
+
onlyRemote : reply.onlyRemote ?? [],
|
|
4112
|
+
onlyLocal : onlyLocalCids,
|
|
2617
4113
|
};
|
|
2618
4114
|
}
|
|
2619
4115
|
|
|
@@ -2629,6 +4125,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2629
4125
|
did: string,
|
|
2630
4126
|
protocol: string | undefined,
|
|
2631
4127
|
depth: number,
|
|
4128
|
+
permissionGrantIds?: string[],
|
|
2632
4129
|
): Promise<Record<string, string>> {
|
|
2633
4130
|
const result: Record<string, string> = {};
|
|
2634
4131
|
const defaultHash = await this.getDefaultHashHex(depth);
|
|
@@ -2646,7 +4143,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2646
4143
|
hexHash = hashToHex(hash);
|
|
2647
4144
|
} else {
|
|
2648
4145
|
// Remote mode fallback.
|
|
2649
|
-
hexHash = await this.getLocalSubtreeHash(did, prefix, undefined, protocol);
|
|
4146
|
+
hexHash = await this.getLocalSubtreeHash(did, prefix, undefined, protocol, permissionGrantIds);
|
|
2650
4147
|
}
|
|
2651
4148
|
|
|
2652
4149
|
if (hexHash === defaultHash) {
|
|
@@ -2670,6 +4167,52 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2670
4167
|
return result;
|
|
2671
4168
|
}
|
|
2672
4169
|
|
|
4170
|
+
private async collectLocalProjectedSubtreeHashes(
|
|
4171
|
+
did: string,
|
|
4172
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
|
|
4173
|
+
depth: number,
|
|
4174
|
+
delegateDid?: string,
|
|
4175
|
+
permissionGrantIds?: string[],
|
|
4176
|
+
): Promise<Record<string, string>> {
|
|
4177
|
+
const result: Record<string, string> = {};
|
|
4178
|
+
const snapshot = this.stateIndex
|
|
4179
|
+
? await RecordsProjection.createSnapshot({
|
|
4180
|
+
tenant : did,
|
|
4181
|
+
messageStore : this.agent.dwn.node.storage.messageStore,
|
|
4182
|
+
scopes,
|
|
4183
|
+
})
|
|
4184
|
+
: undefined;
|
|
4185
|
+
|
|
4186
|
+
try {
|
|
4187
|
+
const walk = async (prefix: string, currentDepth: number): Promise<void> => {
|
|
4188
|
+
const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
|
|
4189
|
+
const hexHash = snapshot
|
|
4190
|
+
? hashToHex(await snapshot.getSubtreeHash(bitPath))
|
|
4191
|
+
: await this.getLocalProjectedSubtreeHash(did, prefix, delegateDid, scopes, permissionGrantIds);
|
|
4192
|
+
const defaultHash = await this.getDefaultHashHex(currentDepth);
|
|
4193
|
+
|
|
4194
|
+
if (hexHash === defaultHash) {
|
|
4195
|
+
return;
|
|
4196
|
+
}
|
|
4197
|
+
|
|
4198
|
+
if (currentDepth >= depth) {
|
|
4199
|
+
result[prefix] = hexHash;
|
|
4200
|
+
return;
|
|
4201
|
+
}
|
|
4202
|
+
|
|
4203
|
+
await Promise.all([
|
|
4204
|
+
walk(prefix + '0', currentDepth + 1),
|
|
4205
|
+
walk(prefix + '1', currentDepth + 1),
|
|
4206
|
+
]);
|
|
4207
|
+
};
|
|
4208
|
+
|
|
4209
|
+
await walk('', 0);
|
|
4210
|
+
return result;
|
|
4211
|
+
} finally {
|
|
4212
|
+
await snapshot?.close();
|
|
4213
|
+
}
|
|
4214
|
+
}
|
|
4215
|
+
|
|
2673
4216
|
/**
|
|
2674
4217
|
* Get the subtree hash at a given bit prefix from the local DWN.
|
|
2675
4218
|
*
|
|
@@ -2677,7 +4220,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2677
4220
|
* In remote mode: constructs a signed MessagesSync message and routes through RPC.
|
|
2678
4221
|
*/
|
|
2679
4222
|
private async getLocalSubtreeHash(
|
|
2680
|
-
did: string, prefix: string, delegateDid?: string, protocol?: string,
|
|
4223
|
+
did: string, prefix: string, delegateDid?: string, protocol?: string, permissionGrantIds?: string[]
|
|
2681
4224
|
): Promise<string> {
|
|
2682
4225
|
const si = this.stateIndex;
|
|
2683
4226
|
if (si) {
|
|
@@ -2695,10 +4238,34 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2695
4238
|
messageType : DwnInterface.MessagesSync,
|
|
2696
4239
|
granteeDid : delegateDid,
|
|
2697
4240
|
messageParams : {
|
|
2698
|
-
action: 'subtree',
|
|
4241
|
+
action : 'subtree',
|
|
2699
4242
|
prefix,
|
|
2700
4243
|
protocol,
|
|
2701
|
-
|
|
4244
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
|
|
4245
|
+
}
|
|
4246
|
+
});
|
|
4247
|
+
const reply = response.reply as MessagesSyncReply;
|
|
4248
|
+
return reply.hash ?? '';
|
|
4249
|
+
}
|
|
4250
|
+
|
|
4251
|
+
private async getLocalProjectedSubtreeHash(
|
|
4252
|
+
did: string,
|
|
4253
|
+
prefix: string,
|
|
4254
|
+
delegateDid: string | undefined,
|
|
4255
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
|
|
4256
|
+
permissionGrantIds?: string[],
|
|
4257
|
+
): Promise<string> {
|
|
4258
|
+
const response = await this.agent.dwn.processRequest({
|
|
4259
|
+
author : did,
|
|
4260
|
+
target : did,
|
|
4261
|
+
messageType : DwnInterface.MessagesSync,
|
|
4262
|
+
granteeDid : delegateDid,
|
|
4263
|
+
messageParams : {
|
|
4264
|
+
action : 'subtree',
|
|
4265
|
+
prefix,
|
|
4266
|
+
projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
|
|
4267
|
+
projectionScopes : [...scopes],
|
|
4268
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
|
|
2702
4269
|
}
|
|
2703
4270
|
});
|
|
2704
4271
|
const reply = response.reply as MessagesSyncReply;
|
|
@@ -2712,7 +4279,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2712
4279
|
* In remote mode: constructs a signed MessagesSync message and routes through RPC.
|
|
2713
4280
|
*/
|
|
2714
4281
|
private async getLocalLeaves(
|
|
2715
|
-
did: string, prefix: string, delegateDid?: string, protocol?: string,
|
|
4282
|
+
did: string, prefix: string, delegateDid?: string, protocol?: string, permissionGrantIds?: string[]
|
|
2716
4283
|
): Promise<string[]> {
|
|
2717
4284
|
const si = this.stateIndex;
|
|
2718
4285
|
if (si) {
|
|
@@ -2729,10 +4296,43 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2729
4296
|
messageType : DwnInterface.MessagesSync,
|
|
2730
4297
|
granteeDid : delegateDid,
|
|
2731
4298
|
messageParams : {
|
|
2732
|
-
action: 'leaves',
|
|
4299
|
+
action : 'leaves',
|
|
2733
4300
|
prefix,
|
|
2734
4301
|
protocol,
|
|
2735
|
-
|
|
4302
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
|
|
4303
|
+
}
|
|
4304
|
+
});
|
|
4305
|
+
const reply = response.reply as MessagesSyncReply;
|
|
4306
|
+
return reply.entries ?? [];
|
|
4307
|
+
}
|
|
4308
|
+
|
|
4309
|
+
private async getLocalProjectedLeaves(
|
|
4310
|
+
did: string,
|
|
4311
|
+
prefix: string,
|
|
4312
|
+
delegateDid: string | undefined,
|
|
4313
|
+
scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
|
|
4314
|
+
permissionGrantIds?: string[],
|
|
4315
|
+
): Promise<string[]> {
|
|
4316
|
+
if (this.stateIndex) {
|
|
4317
|
+
return RecordsProjection.getLeaves({
|
|
4318
|
+
tenant : did,
|
|
4319
|
+
messageStore : this.agent.dwn.node.storage.messageStore,
|
|
4320
|
+
scopes,
|
|
4321
|
+
prefix : SyncEngineLevel.parseBitPrefix(prefix),
|
|
4322
|
+
});
|
|
4323
|
+
}
|
|
4324
|
+
|
|
4325
|
+
const response = await this.agent.dwn.processRequest({
|
|
4326
|
+
author : did,
|
|
4327
|
+
target : did,
|
|
4328
|
+
messageType : DwnInterface.MessagesSync,
|
|
4329
|
+
granteeDid : delegateDid,
|
|
4330
|
+
messageParams : {
|
|
4331
|
+
action : 'leaves',
|
|
4332
|
+
prefix,
|
|
4333
|
+
projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
|
|
4334
|
+
projectionScopes : [...scopes],
|
|
4335
|
+
permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
|
|
2736
4336
|
}
|
|
2737
4337
|
});
|
|
2738
4338
|
const reply = response.reply as MessagesSyncReply;
|
|
@@ -2751,33 +4351,161 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2751
4351
|
* they are processed directly without additional HTTP round-trips.
|
|
2752
4352
|
* Only `messageCids` that were NOT prefetched are fetched individually.
|
|
2753
4353
|
*/
|
|
2754
|
-
private async pullMessages({
|
|
4354
|
+
private async pullMessages({
|
|
4355
|
+
did,
|
|
4356
|
+
dwnUrl,
|
|
4357
|
+
delegateDid,
|
|
4358
|
+
protocol,
|
|
4359
|
+
scope,
|
|
4360
|
+
permissionGrantIds,
|
|
4361
|
+
messageCids,
|
|
4362
|
+
prefetched,
|
|
4363
|
+
verifiedInitialWrites,
|
|
4364
|
+
shouldContinue,
|
|
4365
|
+
}: {
|
|
2755
4366
|
did: string;
|
|
2756
4367
|
dwnUrl: string;
|
|
2757
4368
|
delegateDid?: string;
|
|
2758
4369
|
protocol?: string;
|
|
4370
|
+
scope?: SyncScope;
|
|
4371
|
+
permissionGrantIds?: string[];
|
|
2759
4372
|
messageCids: string[];
|
|
2760
4373
|
prefetched?: MessagesSyncDiffEntry[];
|
|
4374
|
+
verifiedInitialWrites?: RecordsWriteMessage[];
|
|
4375
|
+
shouldContinue?: () => boolean;
|
|
2761
4376
|
}): Promise<void> {
|
|
4377
|
+
const acceptanceScope: SyncScope = scope ?? (protocol === undefined
|
|
4378
|
+
? { kind: 'full' }
|
|
4379
|
+
: { kind: 'protocolSet', protocols: [protocol] });
|
|
4380
|
+
const rejectedPullEntries = new Map<string, Extract<PullAcceptanceResult, { accepted: false }>>();
|
|
2762
4381
|
const failedCids = await pullMessages({
|
|
2763
|
-
did,
|
|
2764
|
-
|
|
2765
|
-
|
|
4382
|
+
did,
|
|
4383
|
+
dwnUrl,
|
|
4384
|
+
delegateDid,
|
|
4385
|
+
permissionGrantIds,
|
|
4386
|
+
messageCids,
|
|
4387
|
+
prefetched,
|
|
4388
|
+
shouldContinue,
|
|
4389
|
+
agent : this.agent,
|
|
4390
|
+
acceptEntry : async (entry, entries) => {
|
|
4391
|
+
const result = await this.acceptPulledSyncEntry(did, acceptanceScope, entry, entries, verifiedInitialWrites);
|
|
4392
|
+
if (!result.accepted) {
|
|
4393
|
+
rejectedPullEntries.set(await getMessageCid(entry.message), result);
|
|
4394
|
+
}
|
|
4395
|
+
return result.accepted;
|
|
4396
|
+
},
|
|
2766
4397
|
});
|
|
2767
4398
|
|
|
2768
4399
|
// Record permanently failed pull entries in the dead letter store.
|
|
2769
4400
|
for (const cid of failedCids) {
|
|
4401
|
+
const rejection = rejectedPullEntries.get(cid);
|
|
2770
4402
|
await this.recordDeadLetter({
|
|
2771
4403
|
messageCid : cid,
|
|
2772
4404
|
tenantDid : did,
|
|
2773
4405
|
remoteEndpoint : dwnUrl,
|
|
2774
4406
|
protocol,
|
|
2775
|
-
category : 'pull-processing',
|
|
2776
|
-
|
|
4407
|
+
category : rejection ? 'pull-scope-rejected' : 'pull-processing',
|
|
4408
|
+
errorCode : rejection?.classification,
|
|
4409
|
+
errorDetail : rejection
|
|
4410
|
+
? `pulled message rejected by ${rejection.classification} sync scope gate`
|
|
4411
|
+
: 'pull processing failed after retry passes exhausted',
|
|
2777
4412
|
});
|
|
2778
4413
|
}
|
|
2779
4414
|
}
|
|
2780
4415
|
|
|
4416
|
+
private async acceptPulledSyncEntry(
|
|
4417
|
+
did: string,
|
|
4418
|
+
scope: SyncScope,
|
|
4419
|
+
entry: SyncMessageEntry,
|
|
4420
|
+
entries: SyncMessageEntry[],
|
|
4421
|
+
verifiedInitialWrites: RecordsWriteMessage[] = [],
|
|
4422
|
+
): Promise<PullAcceptanceResult> {
|
|
4423
|
+
if (scope.kind === 'full') {
|
|
4424
|
+
return { accepted: true };
|
|
4425
|
+
}
|
|
4426
|
+
|
|
4427
|
+
const initialWrite = await this.resolvePulledDeleteInitialWrite(did, entry.message, entries, verifiedInitialWrites);
|
|
4428
|
+
const classification = classifySyncMessageScope({
|
|
4429
|
+
message: entry.message,
|
|
4430
|
+
initialWrite,
|
|
4431
|
+
scope,
|
|
4432
|
+
});
|
|
4433
|
+
|
|
4434
|
+
if (classification === 'in-scope') {
|
|
4435
|
+
return { accepted: true };
|
|
4436
|
+
}
|
|
4437
|
+
|
|
4438
|
+
const messageCid = await getMessageCid(entry.message);
|
|
4439
|
+
console.warn(`SyncEngineLevel: refusing to apply ${classification} pulled message ${messageCid}`);
|
|
4440
|
+
return { accepted: false, classification };
|
|
4441
|
+
}
|
|
4442
|
+
|
|
4443
|
+
private async resolvePulledDeleteInitialWrite(
|
|
4444
|
+
did: string,
|
|
4445
|
+
message: GenericMessage,
|
|
4446
|
+
entries: SyncMessageEntry[],
|
|
4447
|
+
verifiedInitialWrites: RecordsWriteMessage[] = [],
|
|
4448
|
+
): Promise<RecordsWriteMessage | undefined> {
|
|
4449
|
+
const descriptor = message.descriptor as Record<string, unknown>;
|
|
4450
|
+
if (
|
|
4451
|
+
descriptor.interface !== DwnInterfaceName.Records ||
|
|
4452
|
+
descriptor.method !== DwnMethodName.Delete ||
|
|
4453
|
+
typeof descriptor.recordId !== 'string'
|
|
4454
|
+
) {
|
|
4455
|
+
return undefined;
|
|
4456
|
+
}
|
|
4457
|
+
|
|
4458
|
+
if (!this.agent.dwn.isRemoteMode) {
|
|
4459
|
+
const localInitialWrite = await RecordsWrite.fetchInitialRecordsWriteMessage(
|
|
4460
|
+
this.agent.dwn.node.storage.messageStore,
|
|
4461
|
+
did,
|
|
4462
|
+
descriptor.recordId,
|
|
4463
|
+
);
|
|
4464
|
+
if (localInitialWrite) {
|
|
4465
|
+
return localInitialWrite;
|
|
4466
|
+
}
|
|
4467
|
+
}
|
|
4468
|
+
|
|
4469
|
+
const verifiedInitialWrite = SyncEngineLevel.findInitialWriteByRecordId(descriptor.recordId, verifiedInitialWrites);
|
|
4470
|
+
if (verifiedInitialWrite !== undefined) {
|
|
4471
|
+
return verifiedInitialWrite;
|
|
4472
|
+
}
|
|
4473
|
+
|
|
4474
|
+
// Batch entries are only used when the initial write has not been applied
|
|
4475
|
+
// locally yet. Verified dependency hints cover the projected remote-mode
|
|
4476
|
+
// path where the initial write was applied in the previous pull batch and
|
|
4477
|
+
// no embedded local message store is available. Batch entries are still
|
|
4478
|
+
// parsed as initial RecordsWrite messages, and processRawMessage
|
|
4479
|
+
// authenticates the delete before any local mutation occurs.
|
|
4480
|
+
return this.findInitialWriteInPullBatch(descriptor.recordId, entries);
|
|
4481
|
+
}
|
|
4482
|
+
|
|
4483
|
+
private static findInitialWriteByRecordId(
|
|
4484
|
+
recordId: string,
|
|
4485
|
+
initialWrites: RecordsWriteMessage[],
|
|
4486
|
+
): RecordsWriteMessage | undefined {
|
|
4487
|
+
return initialWrites.find(initialWrite => initialWrite.recordId === recordId);
|
|
4488
|
+
}
|
|
4489
|
+
|
|
4490
|
+
private async findInitialWriteInPullBatch(
|
|
4491
|
+
recordId: string,
|
|
4492
|
+
entries: SyncMessageEntry[],
|
|
4493
|
+
): Promise<RecordsWriteMessage | undefined> {
|
|
4494
|
+
for (const entry of entries) {
|
|
4495
|
+
if (entry.message.descriptor.interface !== DwnInterfaceName.Records ||
|
|
4496
|
+
entry.message.descriptor.method !== DwnMethodName.Write) {
|
|
4497
|
+
continue;
|
|
4498
|
+
}
|
|
4499
|
+
|
|
4500
|
+
const candidate = entry.message as RecordsWriteMessage;
|
|
4501
|
+
if (candidate.recordId === recordId && await RecordsWrite.isInitialWrite(candidate)) {
|
|
4502
|
+
return candidate;
|
|
4503
|
+
}
|
|
4504
|
+
}
|
|
4505
|
+
|
|
4506
|
+
return undefined;
|
|
4507
|
+
}
|
|
4508
|
+
|
|
2781
4509
|
// ---------------------------------------------------------------------------
|
|
2782
4510
|
// Echo-loop suppression
|
|
2783
4511
|
// ---------------------------------------------------------------------------
|
|
@@ -2828,17 +4556,16 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2828
4556
|
* Reads missing messages from the local DWN and pushes them to the remote DWN
|
|
2829
4557
|
* in dependency order (topological sort).
|
|
2830
4558
|
*/
|
|
2831
|
-
private async pushMessages({ did, dwnUrl, delegateDid,
|
|
4559
|
+
private async pushMessages({ did, dwnUrl, delegateDid, permissionGrantIds, messageCids }: {
|
|
2832
4560
|
did: string;
|
|
2833
4561
|
dwnUrl: string;
|
|
2834
4562
|
delegateDid?: string;
|
|
2835
|
-
|
|
4563
|
+
permissionGrantIds?: string[];
|
|
2836
4564
|
messageCids: string[];
|
|
2837
4565
|
}): Promise<PushResult> {
|
|
2838
4566
|
return pushMessages({
|
|
2839
|
-
did, dwnUrl, delegateDid,
|
|
2840
|
-
agent
|
|
2841
|
-
permissionsApi : this._permissionsApi,
|
|
4567
|
+
did, dwnUrl, delegateDid, permissionGrantIds, messageCids,
|
|
4568
|
+
agent: this.agent,
|
|
2842
4569
|
});
|
|
2843
4570
|
}
|
|
2844
4571
|
|
|
@@ -2867,14 +4594,20 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2867
4594
|
* When `protocol` is undefined (full-tenant link), clears entries that
|
|
2868
4595
|
* also have no protocol.
|
|
2869
4596
|
*/
|
|
2870
|
-
private async clearDeadLettersForLink(
|
|
4597
|
+
private async clearDeadLettersForLink(
|
|
4598
|
+
tenantDid: string,
|
|
4599
|
+
remoteEndpoint: string,
|
|
4600
|
+
protocol?: string,
|
|
4601
|
+
options: { categories?: ReadonlySet<DeadLetterCategory> } = {},
|
|
4602
|
+
): Promise<void> {
|
|
2871
4603
|
const batch: { type: 'del'; key: string }[] = [];
|
|
2872
4604
|
try {
|
|
2873
4605
|
for await (const [key, value] of this._deadLetters.iterator()) {
|
|
2874
4606
|
const entry = JSON.parse(value) as DeadLetterEntry;
|
|
2875
4607
|
if (entry.tenantDid === tenantDid &&
|
|
2876
4608
|
entry.remoteEndpoint === remoteEndpoint &&
|
|
2877
|
-
entry.protocol === protocol
|
|
4609
|
+
entry.protocol === protocol &&
|
|
4610
|
+
(options.categories === undefined || options.categories.has(entry.category))) {
|
|
2878
4611
|
batch.push({ type: 'del', key });
|
|
2879
4612
|
}
|
|
2880
4613
|
}
|
|
@@ -2887,6 +4620,20 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2887
4620
|
}
|
|
2888
4621
|
}
|
|
2889
4622
|
|
|
4623
|
+
private async clearRootConvergenceDeadLetters(
|
|
4624
|
+
tenantDid: string,
|
|
4625
|
+
remoteEndpoint: string,
|
|
4626
|
+
protocol?: string,
|
|
4627
|
+
): Promise<void> {
|
|
4628
|
+
try {
|
|
4629
|
+
await this.clearDeadLettersForLink(tenantDid, remoteEndpoint, protocol, {
|
|
4630
|
+
categories: SyncEngineLevel.ROOT_CONVERGENCE_CLEARABLE_DEAD_LETTER_CATEGORIES,
|
|
4631
|
+
});
|
|
4632
|
+
} catch (error) {
|
|
4633
|
+
console.warn(`SyncEngineLevel: Failed to clear root-convergence dead letters for ${tenantDid} -> ${remoteEndpoint}`, error);
|
|
4634
|
+
}
|
|
4635
|
+
}
|
|
4636
|
+
|
|
2890
4637
|
/**
|
|
2891
4638
|
* Build a compound dead letter key. Different remotes can fail the same CID
|
|
2892
4639
|
* for different reasons, so the key includes the remote endpoint.
|
|
@@ -2929,7 +4676,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2929
4676
|
}
|
|
2930
4677
|
}
|
|
2931
4678
|
// Deterministic ordering: newest first so apps see the most recent failures.
|
|
2932
|
-
entries.sort((a, b) => b.failedAt
|
|
4679
|
+
entries.sort((a, b) => lexicographicalCompare(b.failedAt, a.failedAt));
|
|
2933
4680
|
return entries;
|
|
2934
4681
|
}
|
|
2935
4682
|
|
|
@@ -2984,44 +4731,87 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
2984
4731
|
|
|
2985
4732
|
public async getSyncHealth(): Promise<SyncHealthSummary> {
|
|
2986
4733
|
let failedMessageCount = 0;
|
|
2987
|
-
|
|
4734
|
+
let closureFailureCount = 0;
|
|
4735
|
+
for await (const [, value] of this._deadLetters.iterator()) {
|
|
2988
4736
|
failedMessageCount++;
|
|
4737
|
+
const entry = JSON.parse(value) as DeadLetterEntry;
|
|
4738
|
+
if (entry.category === 'closure') {
|
|
4739
|
+
closureFailureCount++;
|
|
4740
|
+
}
|
|
2989
4741
|
}
|
|
2990
4742
|
|
|
2991
|
-
//
|
|
2992
|
-
//
|
|
2993
|
-
//
|
|
4743
|
+
// Superseded authorization epochs can leave durable link state behind. Only
|
|
4744
|
+
// links that still belong to the current registered projection/epoch should
|
|
4745
|
+
// affect health. Endpoint-level orphan cleanup is a separate GC concern.
|
|
4746
|
+
const currentLinkIdentityKeys = await this.getCurrentDurableLinkIdentityKeys();
|
|
2994
4747
|
let degradedLinkCount = 0;
|
|
2995
4748
|
const allLinks = await this.ledger.getAllLinks();
|
|
2996
4749
|
for (const link of allLinks) {
|
|
2997
|
-
|
|
4750
|
+
const isCurrentLink = currentLinkIdentityKeys === undefined || currentLinkIdentityKeys.has(this.getDurableLinkIdentityKey(link));
|
|
4751
|
+
if (isCurrentLink && SyncEngineLevel.isUnhealthyLinkStatus(link.status)) {
|
|
2998
4752
|
degradedLinkCount++;
|
|
2999
4753
|
}
|
|
3000
4754
|
}
|
|
3001
4755
|
|
|
3002
4756
|
return {
|
|
3003
|
-
connectivity: this.connectivityState,
|
|
3004
|
-
failedMessageCount,
|
|
3005
|
-
|
|
4757
|
+
connectivity : this.connectivityState,
|
|
4758
|
+
failedMessageCount : failedMessageCount,
|
|
4759
|
+
closureFailureCount : closureFailureCount,
|
|
4760
|
+
degradedLinkCount : degradedLinkCount,
|
|
4761
|
+
syncHealthy : failedMessageCount === 0 && degradedLinkCount === 0,
|
|
3006
4762
|
};
|
|
3007
4763
|
}
|
|
3008
4764
|
|
|
4765
|
+
private async getCurrentDurableLinkIdentityKeys(): Promise<Set<string> | undefined> {
|
|
4766
|
+
try {
|
|
4767
|
+
const identityKeys = new Set<string>();
|
|
4768
|
+
for await (const [did, options] of this._db.sublevel('registeredIdentities').iterator()) {
|
|
4769
|
+
let parsed: SyncIdentityOptions;
|
|
4770
|
+
try {
|
|
4771
|
+
parsed = JSON.parse(options) as SyncIdentityOptions;
|
|
4772
|
+
} catch (error: unknown) {
|
|
4773
|
+
console.warn(`SyncEngineLevel: Corrupt sync options for ${did}, skipping health target:`, error);
|
|
4774
|
+
continue;
|
|
4775
|
+
}
|
|
4776
|
+
|
|
4777
|
+
const scope = syncScopeFromProtocols(parsed.protocols);
|
|
4778
|
+
const resolutions = await this.buildSyncTargetResolutions(did, scope, parsed);
|
|
4779
|
+
for (const resolution of resolutions) {
|
|
4780
|
+
const projectionId = await computeProjectionId(did, resolution.scope);
|
|
4781
|
+
identityKeys.add(SyncEngineLevel.durableLinkIdentityKey(did, projectionId, resolution.authorizationEpoch));
|
|
4782
|
+
}
|
|
4783
|
+
}
|
|
4784
|
+
return identityKeys;
|
|
4785
|
+
} catch (error: unknown) {
|
|
4786
|
+
console.warn('SyncEngineLevel: Failed to resolve current durable link identity keys for health; falling back to all durable links', error);
|
|
4787
|
+
return undefined;
|
|
4788
|
+
}
|
|
4789
|
+
}
|
|
4790
|
+
|
|
4791
|
+
private getDurableLinkIdentityKey(link: ReplicationLinkState): string {
|
|
4792
|
+
return SyncEngineLevel.durableLinkIdentityKey(link.tenantDid, link.projectionId, link.authorizationEpoch);
|
|
4793
|
+
}
|
|
4794
|
+
|
|
4795
|
+
private static durableLinkIdentityKey(tenantDid: string, projectionId: string, authorizationEpoch: string): string {
|
|
4796
|
+
return `${tenantDid}^${projectionId}^${authorizationEpoch}`;
|
|
4797
|
+
}
|
|
4798
|
+
|
|
4799
|
+
private static isUnhealthyLinkStatus(status: ReplicationLinkState['status']): boolean {
|
|
4800
|
+
return status === 'repairing' || status === 'degraded_poll' || status === 'terminal_incomplete';
|
|
4801
|
+
}
|
|
4802
|
+
|
|
3009
4803
|
// ---------------------------------------------------------------------------
|
|
3010
4804
|
// Sync targets
|
|
3011
4805
|
// ---------------------------------------------------------------------------
|
|
3012
4806
|
|
|
3013
4807
|
/**
|
|
3014
|
-
* Returns the list of sync targets:
|
|
4808
|
+
* Returns the list of sync targets: one canonical projection target per
|
|
4809
|
+
* registered DID and resolved DWN endpoint.
|
|
3015
4810
|
* Results are cached for up to 30 seconds to avoid redundant DID resolution
|
|
3016
4811
|
* on every sync tick. The cache is invalidated when identities are registered,
|
|
3017
4812
|
* unregistered, or updated.
|
|
3018
4813
|
*/
|
|
3019
|
-
private async getSyncTargets(): Promise<{
|
|
3020
|
-
did: string;
|
|
3021
|
-
dwnUrl: string;
|
|
3022
|
-
delegateDid?: string;
|
|
3023
|
-
protocol?: string;
|
|
3024
|
-
}[]> {
|
|
4814
|
+
private async getSyncTargets(): Promise<SyncTarget[]> {
|
|
3025
4815
|
// Return cached targets if still valid.
|
|
3026
4816
|
if (this._syncTargetsCache
|
|
3027
4817
|
&& (Date.now() - this._syncTargetsCache.timestamp) < SyncEngineLevel.SYNC_TARGETS_CACHE_TTL_MS) {
|
|
@@ -3033,7 +4823,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
3033
4823
|
// make our result stale.
|
|
3034
4824
|
const generationAtStart = this._syncTargetsCacheGeneration;
|
|
3035
4825
|
|
|
3036
|
-
const targets:
|
|
4826
|
+
const targets: SyncTarget[] = [];
|
|
3037
4827
|
let hasRegisteredIdentities = false;
|
|
3038
4828
|
let anyEndpointMissing = false;
|
|
3039
4829
|
|
|
@@ -3047,8 +4837,6 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
3047
4837
|
continue;
|
|
3048
4838
|
}
|
|
3049
4839
|
|
|
3050
|
-
const { protocols, delegateDid } = parsed;
|
|
3051
|
-
|
|
3052
4840
|
const dwnEndpointUrls = await this.agent.dwn.getDwnEndpointUrlsForTarget(did);
|
|
3053
4841
|
if (dwnEndpointUrls.length === 0) {
|
|
3054
4842
|
anyEndpointMissing = true;
|
|
@@ -3056,14 +4844,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
3056
4844
|
}
|
|
3057
4845
|
|
|
3058
4846
|
for (const dwnUrl of dwnEndpointUrls) {
|
|
3059
|
-
|
|
3060
|
-
// Sync all protocols (global tree).
|
|
3061
|
-
targets.push({ did, delegateDid, dwnUrl });
|
|
3062
|
-
} else {
|
|
3063
|
-
for (const protocol of protocols) {
|
|
3064
|
-
targets.push({ did, delegateDid, dwnUrl, protocol });
|
|
3065
|
-
}
|
|
3066
|
-
}
|
|
4847
|
+
targets.push(...await this.buildSyncTargetsForEndpoint(did, dwnUrl, parsed));
|
|
3067
4848
|
}
|
|
3068
4849
|
}
|
|
3069
4850
|
|
|
@@ -3081,22 +4862,4 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
3081
4862
|
return targets;
|
|
3082
4863
|
}
|
|
3083
4864
|
|
|
3084
|
-
/**
|
|
3085
|
-
* Gets the permission grant ID for MessagesSync if a delegateDid is provided.
|
|
3086
|
-
* Returns undefined if no delegate is in use (owner access).
|
|
3087
|
-
*/
|
|
3088
|
-
private async getSyncPermissionGrantId(did: string, delegateDid?: string, protocol?: string): Promise<string | undefined> {
|
|
3089
|
-
if (!delegateDid) {
|
|
3090
|
-
return undefined;
|
|
3091
|
-
}
|
|
3092
|
-
|
|
3093
|
-
const messagesSyncGrant = await this._permissionsApi.getPermissionForRequest({
|
|
3094
|
-
connectedDid : did,
|
|
3095
|
-
messageType : DwnInterface.MessagesSync,
|
|
3096
|
-
delegateDid,
|
|
3097
|
-
protocol,
|
|
3098
|
-
cached : true
|
|
3099
|
-
});
|
|
3100
|
-
return messagesSyncGrant.grant.id;
|
|
3101
|
-
}
|
|
3102
4865
|
}
|