@enbox/agent 0.7.7 → 0.7.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/dist/browser.mjs +9 -9
  2. package/dist/browser.mjs.map +4 -4
  3. package/dist/esm/dwn-api.js +3 -2
  4. package/dist/esm/dwn-api.js.map +1 -1
  5. package/dist/esm/enbox-connect-protocol.js +5 -5
  6. package/dist/esm/enbox-connect-protocol.js.map +1 -1
  7. package/dist/esm/index.js +1 -1
  8. package/dist/esm/index.js.map +1 -1
  9. package/dist/esm/permissions-api.js +7 -34
  10. package/dist/esm/permissions-api.js.map +1 -1
  11. package/dist/esm/sync-closure-resolver.js +229 -110
  12. package/dist/esm/sync-closure-resolver.js.map +1 -1
  13. package/dist/esm/sync-closure-types.js +39 -7
  14. package/dist/esm/sync-closure-types.js.map +1 -1
  15. package/dist/esm/sync-engine-level.js +2242 -797
  16. package/dist/esm/sync-engine-level.js.map +1 -1
  17. package/dist/esm/sync-link-id.js +4 -13
  18. package/dist/esm/sync-link-id.js.map +1 -1
  19. package/dist/esm/sync-link-reconciler.js +26 -8
  20. package/dist/esm/sync-link-reconciler.js.map +1 -1
  21. package/dist/esm/sync-messages.js +218 -154
  22. package/dist/esm/sync-messages.js.map +1 -1
  23. package/dist/esm/sync-permission-grants.js +208 -0
  24. package/dist/esm/sync-permission-grants.js.map +1 -0
  25. package/dist/esm/sync-replication-ledger.js +23 -40
  26. package/dist/esm/sync-replication-ledger.js.map +1 -1
  27. package/dist/esm/sync-scope-acceptance.js +126 -0
  28. package/dist/esm/sync-scope-acceptance.js.map +1 -0
  29. package/dist/esm/sync-topological-sort.js +57 -15
  30. package/dist/esm/sync-topological-sort.js.map +1 -1
  31. package/dist/esm/types/sync.js +130 -22
  32. package/dist/esm/types/sync.js.map +1 -1
  33. package/dist/types/dwn-api.d.ts.map +1 -1
  34. package/dist/types/index.d.ts +1 -1
  35. package/dist/types/index.d.ts.map +1 -1
  36. package/dist/types/permissions-api.d.ts +1 -2
  37. package/dist/types/permissions-api.d.ts.map +1 -1
  38. package/dist/types/sync-closure-resolver.d.ts.map +1 -1
  39. package/dist/types/sync-closure-types.d.ts +14 -3
  40. package/dist/types/sync-closure-types.d.ts.map +1 -1
  41. package/dist/types/sync-engine-level.d.ts +144 -31
  42. package/dist/types/sync-engine-level.d.ts.map +1 -1
  43. package/dist/types/sync-link-id.d.ts +3 -9
  44. package/dist/types/sync-link-id.d.ts.map +1 -1
  45. package/dist/types/sync-link-reconciler.d.ts +12 -2
  46. package/dist/types/sync-link-reconciler.d.ts.map +1 -1
  47. package/dist/types/sync-messages.d.ts +16 -13
  48. package/dist/types/sync-messages.d.ts.map +1 -1
  49. package/dist/types/sync-permission-grants.d.ts +52 -0
  50. package/dist/types/sync-permission-grants.d.ts.map +1 -0
  51. package/dist/types/sync-replication-ledger.d.ts +5 -13
  52. package/dist/types/sync-replication-ledger.d.ts.map +1 -1
  53. package/dist/types/sync-scope-acceptance.d.ts +28 -0
  54. package/dist/types/sync-scope-acceptance.d.ts.map +1 -0
  55. package/dist/types/sync-topological-sort.d.ts +2 -1
  56. package/dist/types/sync-topological-sort.d.ts.map +1 -1
  57. package/dist/types/types/permissions.d.ts +2 -0
  58. package/dist/types/types/permissions.d.ts.map +1 -1
  59. package/dist/types/types/sync.d.ts +137 -75
  60. package/dist/types/types/sync.d.ts.map +1 -1
  61. package/package.json +3 -3
  62. package/src/dwn-api.ts +3 -2
  63. package/src/enbox-connect-protocol.ts +5 -5
  64. package/src/index.ts +10 -1
  65. package/src/permissions-api.ts +11 -42
  66. package/src/sync-closure-resolver.ts +306 -126
  67. package/src/sync-closure-types.ts +54 -9
  68. package/src/sync-engine-level.ts +3051 -967
  69. package/src/sync-link-id.ts +9 -14
  70. package/src/sync-link-reconciler.ts +43 -10
  71. package/src/sync-messages.ts +263 -159
  72. package/src/sync-permission-grants.ts +297 -0
  73. package/src/sync-replication-ledger.ts +55 -50
  74. package/src/sync-scope-acceptance.ts +186 -0
  75. package/src/sync-topological-sort.ts +89 -21
  76. package/src/types/permissions.ts +2 -0
  77. package/src/types/sync.ts +235 -62
@@ -1,31 +1,36 @@
1
1
  import type { AbstractLevel } from 'abstract-level';
2
2
 
3
3
  import type { DwnSubscriptionHandler, ResubscribeFactory } from '@enbox/dwn-clients';
4
- import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncDiffEntry, MessagesSyncReply, ProgressToken, StateIndex, SubscriptionMessage } from '@enbox/dwn-sdk-js';
4
+ import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncDependencyEntry, MessagesSyncDiffEntry, MessagesSyncReply, ProgressToken, ProtocolsConfigureMessage, ProtocolsQueryReply, RecordsProjectionScope, RecordsWriteMessage, StateIndex, SubscriptionMessage } from '@enbox/dwn-sdk-js';
5
5
 
6
6
  import ms from 'ms';
7
7
 
8
8
  import { Level } from 'level';
9
9
  import { sleep } from '@enbox/common';
10
- import { Encoder, hashToHex, initDefaultHashes, Message } from '@enbox/dwn-sdk-js';
10
+ import { authenticate, DwnInterfaceName, DwnMethodName, Encoder, hashToHex, initDefaultHashes, Message, ProtocolsConfigure, RECORDS_PROJECTION_ROOT_VERSION, RecordsProjection, RecordsWrite } from '@enbox/dwn-sdk-js';
11
11
 
12
- import type { ClosureEvaluationContext } from './sync-closure-types.js';
12
+ import type { DwnMessageParams } from './types/dwn.js';
13
13
  import type { EnboxPlatformAgent } from './types/agent.js';
14
14
  import type { PermissionsApi } from './types/permissions.js';
15
- import type { DeadLetterCategory, DeadLetterEntry, PushResult, ReplicationLinkState, StartSyncParams, SyncConnectivityState, SyncEngine, SyncEvent, SyncEventListener, SyncHealthSummary, SyncIdentityOptions, SyncMode, SyncScope } from './types/sync.js';
16
-
17
- import { evaluateClosure } from './sync-closure-resolver.js';
18
- import { MAX_PENDING_TOKENS } from './types/sync.js';
19
- import { ReplicationLedger } from './sync-replication-ledger.js';
20
- import { createClosureContext, invalidateClosureCache } from './sync-closure-types.js';
15
+ import type { SyncMessageEntry } from './sync-messages.js';
16
+ import type { SyncScopeClassification } from './sync-scope-acceptance.js';
17
+ import type { ClosureEvaluationContext, ClosureResult } from './sync-closure-types.js';
18
+ import type { DeadLetterCategory, DeadLetterEntry, NonEmptyStringArray, PushResult, ReplicationLinkState, StartSyncParams, SyncAuthorization, SyncConnectivityState, SyncEngine, SyncEvent, SyncEventListener, SyncEventScope, SyncHealthSummary, SyncIdentityOptions, SyncMode, SyncScope } from './types/sync.js';
21
19
 
22
20
  import { AgentPermissionsApi } from './permissions-api.js';
21
+
22
+ import { buildLinkId } from './sync-link-id.js';
23
23
  import { DwnInterface } from './types/dwn.js';
24
+ import { evaluateClosure } from './sync-closure-resolver.js';
24
25
  import { isRecordsWrite } from './utils.js';
25
- import { SyncLinkReconciler } from './sync-link-reconciler.js';
26
+ import { ReplicationLedger } from './sync-replication-ledger.js';
26
27
  import { topologicalSort } from './sync-topological-sort.js';
27
- import { buildLegacyCursorKey, buildLinkId } from './sync-link-id.js';
28
- import { fetchRemoteMessages, pullMessages, pushMessages } from './sync-messages.js';
28
+ import { classifySyncEventScope, classifySyncMessageScope } from './sync-scope-acceptance.js';
29
+ import { ClosureFailureCode, createClosureContext, invalidateClosureCache, isTerminalClosureFailureCode } from './sync-closure-types.js';
30
+ import { computeAuthorizationEpoch, computeProjectionId, lexicographicalCompare, MAX_PENDING_TOKENS, protocolsForSyncScope, singleProtocolForSyncScope, syncScopeFromProtocols } from './types/sync.js';
31
+ import { fetchRemoteMessages, getMessageCid, pullMessages, pushMessages, SyncPullAbortedError } from './sync-messages.js';
32
+ import { partitionRemoteEntries, SyncLinkReconciler } from './sync-link-reconciler.js';
33
+ import { permissionGrantIdsFromEntries, resolveMessagesSyncScopes, toMessagesPermissionGrantIds, toSyncAuthorizationGrants } from './sync-permission-grants.js';
29
34
 
30
35
  export type SyncEngineLevelParams = {
31
36
  agent?: EnboxPlatformAgent;
@@ -34,7 +39,7 @@ export type SyncEngineLevelParams = {
34
39
  };
35
40
 
36
41
  /**
37
- * Maximum bit prefix depth for the per-node tree walk (legacy fallback).
42
+ * Maximum bit prefix depth for the per-node tree walk fallback.
38
43
  * At depth 16, each subtree covers ~1/65536 of the key space.
39
44
  */
40
45
  const MAX_DIFF_DEPTH = 16;
@@ -62,7 +67,6 @@ type LiveSubscription = {
62
67
  did: string;
63
68
  dwnUrl: string;
64
69
  delegateDid?: string;
65
- protocol?: string;
66
70
  close: () => Promise<void>;
67
71
  };
68
72
 
@@ -72,10 +76,103 @@ type LocalSubscription = {
72
76
  did: string;
73
77
  dwnUrl: string;
74
78
  delegateDid?: string;
75
- protocol?: string;
76
79
  close: () => Promise<void>;
77
80
  };
78
81
 
82
+ type SyncTarget = {
83
+ did: string;
84
+ dwnUrl: string;
85
+ delegateDid?: string;
86
+ scope: SyncScope;
87
+ authorization: SyncAuthorization;
88
+ authorizationEpoch: string;
89
+ permissionGrantIds?: NonEmptyStringArray;
90
+ };
91
+
92
+ type SyncTargetResolution = Pick<SyncTarget, 'authorization' | 'authorizationEpoch' | 'delegateDid' | 'permissionGrantIds' | 'scope'>;
93
+
94
+ type LinkSyncTarget = SyncTarget & { linkKey: string };
95
+
96
+ enum LinkSubscriptionOpenResult {
97
+ ReadyForLive = 'readyForLive',
98
+ Polling = 'polling',
99
+ Repairing = 'repairing',
100
+ }
101
+
102
+ enum LinkInitializationStatus {
103
+ Active = 'active',
104
+ Failed = 'failed',
105
+ }
106
+
107
+ type LinkInitializationResult =
108
+ | { status: LinkInitializationStatus.Active; durableLinkIdentityKey: string }
109
+ | { status: LinkInitializationStatus.Failed };
110
+
111
+ type PullAcceptanceResult =
112
+ | { accepted: true }
113
+ | { accepted: false; classification: Exclude<SyncScopeClassification, 'in-scope'> };
114
+
115
+ type RecordsWriteProtocolDescriptor = GenericMessage['descriptor'] & { protocol?: unknown };
116
+ type RecordsWriteProtocolMessage = GenericMessage & { descriptor: RecordsWriteProtocolDescriptor };
117
+ type ProtocolsConfigureDefinition = {
118
+ protocol: string;
119
+ uses?: Record<string, unknown>;
120
+ };
121
+ type ProtocolsConfigureDefinitionDescriptor = GenericMessage['descriptor'] & {
122
+ definition: ProtocolsConfigureDefinition;
123
+ };
124
+ type MaybeProtocolsConfigureDefinitionDescriptor = GenericMessage['descriptor'] & {
125
+ definition?: {
126
+ protocol?: string;
127
+ uses?: Record<string, unknown>;
128
+ };
129
+ };
130
+ type SyncDiffEntryWithMessage = MessagesSyncDiffEntry & { message: GenericMessage };
131
+ type SyncDependencyEntryWithMessage = MessagesSyncDependencyEntry & { message: GenericMessage };
132
+ type VerifiedProtocolConfigCandidate = {
133
+ rootMessageCid: string;
134
+ dependency: AuthenticatedProtocolConfigDependency;
135
+ };
136
+ type AuthenticatedProtocolConfigDependency = SyncDependencyEntryWithMessage & { message: ProtocolsConfigureMessage };
137
+ type VerifiedRecordsInitialWriteCandidate = {
138
+ rootMessageCid: string;
139
+ dependency: AuthenticatedRecordsInitialWriteDependency;
140
+ };
141
+ type AuthenticatedRecordsInitialWriteDependency = SyncDependencyEntryWithMessage & { message: RecordsWriteMessage };
142
+
143
+ type ProjectionReconcileTarget = {
144
+ did: string;
145
+ dwnUrl: string;
146
+ delegateDid?: string;
147
+ scope: SyncScope;
148
+ authorization: SyncAuthorization;
149
+ };
150
+
151
+ type ProjectionReconcileOptions = {
152
+ direction?: 'push' | 'pull';
153
+ verifyConvergence?: boolean;
154
+ };
155
+
156
+ type ProjectionReconcileResult = {
157
+ aborted?: boolean;
158
+ converged?: boolean;
159
+ };
160
+
161
+ type ProjectionDiffResult = {
162
+ dependencies?: MessagesSyncDependencyEntry[];
163
+ onlyRemote: MessagesSyncDiffEntry[];
164
+ onlyLocal: string[];
165
+ };
166
+
167
+ type ProtocolSetScope = Extract<SyncScope, { kind: 'protocolSet' }>;
168
+ type RecordsProjectionSyncScope = Extract<SyncScope, { kind: 'recordsProjection' }>;
169
+
170
+ type ProtocolSetDiffPlan = {
171
+ changedProtocols: string[];
172
+ onlyRemote: MessagesSyncDiffEntry[];
173
+ onlyLocal: string[];
174
+ };
175
+
79
176
  // ---------------------------------------------------------------------------
80
177
  // Per-link in-memory delivery-order tracking (not persisted to ledger)
81
178
  // ---------------------------------------------------------------------------
@@ -95,48 +192,6 @@ type InFlightCommit = {
95
192
  committed: boolean;
96
193
  };
97
194
 
98
- /**
99
- * Checks whether a message's protocolPath and contextId match the link's
100
- * subset scope prefixes. Returns true if the message is in scope.
101
- *
102
- * When the scope has no prefixes (or is kind:'full'), all messages match.
103
- * When protocolPathPrefixes or contextIdPrefixes are specified, the message
104
- * must match at least one prefix in each specified set.
105
- *
106
- * This is agent-side filtering for subset scopes. The underlying
107
- * MessagesSubscribe filter only supports protocol-level scoping today —
108
- * protocolPath/contextId prefix filtering at the EventLog level is a
109
- * follow-up (requires dwn-sdk-js MessagesFilter extension).
110
- */
111
- function isEventInScope(message: GenericMessage, scope: SyncScope): boolean {
112
- if (scope.kind === 'full') { return true; }
113
- if (!scope.protocolPathPrefixes && !scope.contextIdPrefixes) { return true; }
114
-
115
- const desc = message.descriptor as Record<string, unknown>;
116
-
117
- // Check protocolPath prefix.
118
- if (scope.protocolPathPrefixes && scope.protocolPathPrefixes.length > 0) {
119
- const protocolPath = desc.protocolPath as string | undefined;
120
- if (!protocolPath) { return false; }
121
- const matches = scope.protocolPathPrefixes.some(
122
- prefix => protocolPath === prefix || protocolPath.startsWith(prefix + '/')
123
- );
124
- if (!matches) { return false; }
125
- }
126
-
127
- // Check contextId prefix.
128
- if (scope.contextIdPrefixes && scope.contextIdPrefixes.length > 0) {
129
- const contextId = (message as any).contextId as string | undefined;
130
- if (!contextId) { return false; }
131
- const matches = scope.contextIdPrefixes.some(
132
- prefix => contextId === prefix || contextId.startsWith(prefix + '/')
133
- );
134
- if (!matches) { return false; }
135
- }
136
-
137
- return true;
138
- }
139
-
140
195
  /**
141
196
  * Per-link runtime state held in memory. Not persisted — on crash,
142
197
  * replay restarts from `contiguousAppliedToken` (idempotent apply).
@@ -150,18 +205,66 @@ type LinkRuntimeState = {
150
205
  inflight: Map<number, InFlightCommit>;
151
206
  };
152
207
 
208
+ type PushRuntimeEntry = { cid: string };
209
+
153
210
  type PushRuntimeState = {
154
211
  did: string;
155
212
  dwnUrl: string;
156
213
  delegateDid?: string;
157
214
  protocol?: string;
158
- entries: { cid: string }[];
215
+ permissionGrantIds?: NonEmptyStringArray;
216
+ entries: PushRuntimeEntry[];
159
217
  retryCount: number;
160
218
  timer?: ReturnType<typeof setTimeout>;
161
219
  /** True while a push HTTP request is in flight for this link. */
162
220
  flushing?: boolean;
163
221
  };
164
222
 
223
+ type PushFlushBatch = {
224
+ pushRuntime: PushRuntimeState;
225
+ pushEntries: PushRuntimeEntry[];
226
+ isStale: () => boolean;
227
+ };
228
+
229
+ type LivePullContext = {
230
+ did: string;
231
+ dwnUrl: string;
232
+ delegateDid?: string;
233
+ eventScope: SyncEventScope;
234
+ linkKey: string;
235
+ link?: ReplicationLinkState;
236
+ permissionGrantIds?: NonEmptyStringArray;
237
+ isStale: () => boolean;
238
+ };
239
+
240
+ type PullDelivery = {
241
+ runtime?: LinkRuntimeState;
242
+ ordinal: number;
243
+ };
244
+
245
+ type LivePullDataStreamFactory = () => Promise<ReadableStream<Uint8Array> | undefined>;
246
+
247
+ type ApplyStatus = {
248
+ code: number;
249
+ detail?: string;
250
+ };
251
+
252
+ function syncEventScope(scope: SyncScope | undefined): SyncEventScope {
253
+ if (scope === undefined) {
254
+ return {};
255
+ }
256
+
257
+ const coveredProtocols = protocolsForSyncScope(scope);
258
+ if (coveredProtocols === undefined) {
259
+ return {};
260
+ }
261
+
262
+ const protocols = [...coveredProtocols] as NonEmptyStringArray;
263
+ return protocols.length === 1
264
+ ? { protocol: protocols[0], protocols }
265
+ : { protocols };
266
+ }
267
+
165
268
  export class SyncEngineLevel implements SyncEngine {
166
269
  /**
167
270
  * Holds the instance of a `EnboxPlatformAgent` that represents the current execution context for
@@ -257,6 +360,9 @@ export class SyncEngineLevel implements SyncEngine {
257
360
  */
258
361
  private readonly _closureContexts: Map<string, ClosureEvaluationContext> = new Map();
259
362
 
363
+ /** Deduplicates concurrent live-sync repairs for the same tenant/protocol. */
364
+ private readonly _protocolMetadataRepairs: Map<string, Promise<boolean>> = new Map();
365
+
260
366
  /** Maximum entries in the echo-loop suppression cache. */
261
367
  private static readonly ECHO_SUPPRESS_MAX_ENTRIES = 10_000;
262
368
 
@@ -273,13 +379,67 @@ export class SyncEngineLevel implements SyncEngine {
273
379
  }
274
380
  }
275
381
 
382
+ private async buildSyncTargetsForEndpoint(did: string, dwnUrl: string, options: SyncIdentityOptions): Promise<SyncTarget[]> {
383
+ const requestedScope = syncScopeFromProtocols(options.protocols);
384
+ const resolutions = await this.buildSyncTargetResolutions(did, requestedScope, options);
385
+
386
+ return resolutions.map(resolution => ({
387
+ did,
388
+ dwnUrl,
389
+ ...resolution,
390
+ }));
391
+ }
392
+
393
+ private async buildSyncTargetResolutions(did: string, requestedScope: SyncScope, options: SyncIdentityOptions): Promise<SyncTargetResolution[]> {
394
+ const { delegateDid } = options;
395
+
396
+ if (delegateDid === undefined) {
397
+ return [{
398
+ scope : requestedScope,
399
+ authorization : { kind: 'owner' },
400
+ authorizationEpoch : await computeAuthorizationEpoch({ kind: 'owner' }),
401
+ }];
402
+ }
403
+
404
+ const resolvedScopes = await resolveMessagesSyncScopes({
405
+ did,
406
+ delegateDid,
407
+ requestedScope,
408
+ messageType : DwnInterface.MessagesSync,
409
+ permissionsApi : this._permissionsApi,
410
+ });
411
+
412
+ return Promise.all(resolvedScopes.map(async ({ scope, permissionGrants }) => {
413
+ const permissionGrantIds = permissionGrantIdsFromEntries(permissionGrants);
414
+ if (permissionGrantIds === undefined) {
415
+ throw new Error(`SyncEngineLevel: delegate ${delegateDid} has no active sync grants for ${did}.`);
416
+ }
417
+
418
+ return {
419
+ scope,
420
+ delegateDid,
421
+ authorization: {
422
+ kind: 'delegate' as const,
423
+ delegateDid,
424
+ permissionGrantIds,
425
+ },
426
+ authorizationEpoch: await computeAuthorizationEpoch({
427
+ kind : 'delegate' as const,
428
+ delegateDid,
429
+ grants : toSyncAuthorizationGrants(permissionGrants),
430
+ }),
431
+ permissionGrantIds,
432
+ };
433
+ }));
434
+ }
435
+
276
436
  /**
277
437
  * Cached sync targets result from the last {@link getSyncTargets} call.
278
438
  * Invalidated on identity registration/unregistration/update.
279
439
  * TTL-based: cleared after 30 seconds to pick up DID document changes.
280
440
  */
281
441
  private _syncTargetsCache?: {
282
- targets: { did: string; dwnUrl: string; delegateDid?: string; protocol?: string }[];
442
+ targets: SyncTarget[];
283
443
  timestamp: number;
284
444
  };
285
445
 
@@ -295,6 +455,9 @@ export class SyncEngineLevel implements SyncEngine {
295
455
  /** TTL for the sync targets cache (30 seconds). */
296
456
  private static readonly SYNC_TARGETS_CACHE_TTL_MS = 30_000;
297
457
 
458
+ /** Backoff schedule for recently published did:dht records. */
459
+ private static readonly DID_RESOLUTION_RETRY_BACKOFF_MS = [2000, 4000, 8000];
460
+
298
461
  /** Count of consecutive SMT sync failures (for backoff in poll mode). */
299
462
  private _consecutiveFailures = 0;
300
463
 
@@ -428,7 +591,12 @@ export class SyncEngineLevel implements SyncEngine {
428
591
 
429
592
  // If live sync is active, hot-add subscriptions for this identity.
430
593
  if (this._syncMode === 'live') {
431
- await this.addIdentityToLiveSync(did, options);
594
+ const currentIdentityKeys = await this.addIdentityToLiveSync(did, options);
595
+ if (currentIdentityKeys.size > 0) {
596
+ await this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
597
+ }
598
+ } else {
599
+ await this.tryPruneSupersededDurableLinksForRegisteredIdentity(did, options);
432
600
  }
433
601
  }
434
602
 
@@ -447,6 +615,7 @@ export class SyncEngineLevel implements SyncEngine {
447
615
  await registeredIdentities.del(did);
448
616
  this._syncTargetsCache = undefined;
449
617
  this._syncTargetsCacheGeneration++;
618
+ await this.pruneSupersededDurableLinksForIdentity(did, new Set());
450
619
  }
451
620
 
452
621
  public async getIdentityOptions(did: string): Promise<SyncIdentityOptions | undefined> {
@@ -480,19 +649,17 @@ export class SyncEngineLevel implements SyncEngine {
480
649
  this._syncTargetsCache = undefined;
481
650
  this._syncTargetsCacheGeneration++;
482
651
 
483
- // Always persist the new delegate to durable links, regardless of
484
- // sync mode. If sync is stopped or polling, existing persisted links
485
- // would otherwise keep the old delegateDid. When live sync starts
486
- // later, initializeLinkTarget() loads the link from LevelDB without
487
- // normalizing delegateDid, so repair/reconcile paths could use stale
488
- // delegate data.
489
- await this.ledger.updateDelegateDid(did, options.delegateDid);
490
-
491
652
  // If live sync is active, tear down and rebuild subscriptions with
492
- // the new options.
653
+ // the new options. Delegate/scope changes derive a new authorization
654
+ // epoch, so existing durable links are not mutated in place.
493
655
  if (this._syncMode === 'live' && this.hasActiveLinksForDid(did)) {
494
656
  await this.removeIdentityFromLiveSync(did);
495
- await this.addIdentityToLiveSync(did, options);
657
+ const currentIdentityKeys = await this.addIdentityToLiveSync(did, options);
658
+ if (currentIdentityKeys.size > 0) {
659
+ await this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
660
+ }
661
+ } else {
662
+ await this.tryPruneSupersededDurableLinksForRegisteredIdentity(did, options);
496
663
  }
497
664
  }
498
665
 
@@ -526,15 +693,12 @@ export class SyncEngineLevel implements SyncEngine {
526
693
 
527
694
  const results = await Promise.allSettled([...byUrl.entries()].map(async ([dwnUrl, targets]) => {
528
695
  for (const target of targets) {
529
- const { did, delegateDid, protocol } = target;
530
696
  try {
531
- await this.createLinkReconciler().reconcile({
532
- did, dwnUrl, delegateDid, protocol,
533
- }, { direction });
697
+ await this.reconcileProjectionTarget(target, { direction });
534
698
  } catch (error: any) {
535
699
  // Skip remaining targets for this DWN endpoint.
536
700
  groupsFailed++;
537
- console.error(`SyncEngineLevel: Error syncing ${did} with ${dwnUrl}`, error);
701
+ console.error(`SyncEngineLevel: Error syncing ${target.did} with ${dwnUrl}`, error);
538
702
  return;
539
703
  }
540
704
  }
@@ -636,7 +800,7 @@ export class SyncEngineLevel implements SyncEngine {
636
800
  }
637
801
 
638
802
  // ---------------------------------------------------------------------------
639
- // Poll-mode sync (legacy)
803
+ // Poll-mode sync
640
804
  // ---------------------------------------------------------------------------
641
805
 
642
806
  private async startPollSync(intervalMilliseconds: number): Promise<void> {
@@ -768,7 +932,7 @@ export class SyncEngineLevel implements SyncEngine {
768
932
  }
769
933
 
770
934
  // ---------------------------------------------------------------------------
771
- // Per-link repair and degraded-poll orchestration (Phase 2)
935
+ // Per-link repair and degraded-poll orchestration
772
936
  // ---------------------------------------------------------------------------
773
937
 
774
938
  /** Maximum consecutive repair attempts before falling back to degraded_poll. */
@@ -811,27 +975,19 @@ export class SyncEngineLevel implements SyncEngine {
811
975
  link: ReplicationLinkState,
812
976
  options?: { resumeToken?: ProgressToken },
813
977
  ): Promise<void> {
814
- const prevStatus = link.status;
815
- const prevConnectivity = link.connectivity;
816
- link.connectivity = 'offline';
817
- await this.ledger.setStatus(link, 'repairing');
818
-
819
- this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevStatus, to: 'repairing' });
820
- if (prevConnectivity !== 'offline') {
821
- this.emitEvent({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevConnectivity, to: 'offline' });
978
+ if (link.status === 'terminal_incomplete') {
979
+ return;
822
980
  }
823
981
 
982
+ await this.setLinkOfflineStatus(link, 'repairing');
983
+
824
984
  if (options?.resumeToken) {
825
985
  this._repairContext.set(linkKey, { resumeToken: options.resumeToken });
826
986
  }
827
987
 
828
988
  // Clear runtime ordinals immediately — stale state must not linger
829
989
  // across repair attempts.
830
- const rt = this._linkRuntimes.get(linkKey);
831
- if (rt) {
832
- rt.inflight.clear();
833
- rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
834
- }
990
+ this.clearLinkRuntimeInflight(linkKey);
835
991
 
836
992
  // Kick off repair with retry scheduling on failure.
837
993
  void this.repairLink(linkKey).catch(() => {
@@ -839,6 +995,68 @@ export class SyncEngineLevel implements SyncEngine {
839
995
  });
840
996
  }
841
997
 
998
+ private async transitionToTerminalIncomplete(
999
+ linkKey: string,
1000
+ link: ReplicationLinkState,
1001
+ ): Promise<void> {
1002
+ if (link.status === 'terminal_incomplete') {
1003
+ return;
1004
+ }
1005
+
1006
+ await this.setLinkOfflineStatus(link, 'terminal_incomplete');
1007
+
1008
+ await this.closeLinkSubscriptions(link);
1009
+
1010
+ this.clearLinkRuntimeInflight(linkKey);
1011
+
1012
+ const retryTimer = this._repairRetryTimers.get(linkKey);
1013
+ if (retryTimer) {
1014
+ clearTimeout(retryTimer);
1015
+ this._repairRetryTimers.delete(linkKey);
1016
+ }
1017
+ const degradedTimer = this._degradedPollTimers.get(linkKey);
1018
+ if (degradedTimer) {
1019
+ clearInterval(degradedTimer);
1020
+ this._degradedPollTimers.delete(linkKey);
1021
+ }
1022
+ const reconcileTimer = this._reconcileTimers.get(linkKey);
1023
+ if (reconcileTimer) {
1024
+ clearTimeout(reconcileTimer);
1025
+ this._reconcileTimers.delete(linkKey);
1026
+ }
1027
+ const pushRuntime = this._pushRuntimes.get(linkKey);
1028
+ if (pushRuntime?.timer) {
1029
+ clearTimeout(pushRuntime.timer);
1030
+ }
1031
+ this._pushRuntimes.delete(linkKey);
1032
+
1033
+ this._repairAttempts.delete(linkKey);
1034
+ this._repairContext.delete(linkKey);
1035
+ }
1036
+
1037
+ private async setLinkOfflineStatus(link: ReplicationLinkState, status: ReplicationLinkState['status']): Promise<void> {
1038
+ const prevStatus = link.status;
1039
+ const prevConnectivity = link.connectivity;
1040
+ link.connectivity = 'offline';
1041
+ await this.ledger.setStatus(link, status);
1042
+
1043
+ const eventScope = syncEventScope(link.scope);
1044
+ this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope, from: prevStatus, to: status });
1045
+ if (prevConnectivity !== 'offline') {
1046
+ this.emitEvent({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope, from: prevConnectivity, to: 'offline' });
1047
+ }
1048
+ }
1049
+
1050
+ private clearLinkRuntimeInflight(linkKey: string): void {
1051
+ const rt = this._linkRuntimes.get(linkKey);
1052
+ if (!rt) {
1053
+ return;
1054
+ }
1055
+
1056
+ rt.inflight.clear();
1057
+ rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
1058
+ }
1059
+
842
1060
  /**
843
1061
  * Schedule a retry for a failed repair. Uses exponential backoff.
844
1062
  * No-op if the link is already in `degraded_poll` (timer loop owns retries)
@@ -925,9 +1143,10 @@ export class SyncEngineLevel implements SyncEngine {
925
1143
  // The old repair closure must not mutate the replacement link's state.
926
1144
  const isStaleLink = (): boolean => this._activeLinks.get(linkKey) !== link;
927
1145
 
928
- const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, protocol } = link;
1146
+ const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, scope, authorization } = link;
1147
+ const eventScope = syncEventScope(scope);
929
1148
 
930
- this.emitEvent({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl, protocol, attempt: (this._repairAttempts.get(linkKey) ?? 0) + 1 });
1149
+ this.emitEvent({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, attempt: (this._repairAttempts.get(linkKey) ?? 0) + 1 });
931
1150
  const attempts = (this._repairAttempts.get(linkKey) ?? 0) + 1;
932
1151
  this._repairAttempts.set(linkKey, attempts);
933
1152
 
@@ -945,9 +1164,13 @@ export class SyncEngineLevel implements SyncEngine {
945
1164
 
946
1165
  try {
947
1166
  // Step 3: Run SMT reconciliation for this link.
948
- const reconcileOutcome = await this.createLinkReconciler(
949
- () => this._engineGeneration === generation && !isStaleLink()
950
- ).reconcile({ did, dwnUrl, delegateDid, protocol });
1167
+ const reconcileOutcome = await this.reconcileProjectionTarget({
1168
+ did,
1169
+ dwnUrl,
1170
+ delegateDid,
1171
+ scope,
1172
+ authorization,
1173
+ }, undefined, () => this._engineGeneration === generation && !isStaleLink());
951
1174
  if (reconcileOutcome.aborted) { return; }
952
1175
 
953
1176
  // Step 4: Determine the post-repair pull resume token.
@@ -972,7 +1195,16 @@ export class SyncEngineLevel implements SyncEngine {
972
1195
  await this.ledger.saveLink(link);
973
1196
  if (this._engineGeneration !== generation || isStaleLink()) { return; }
974
1197
 
975
- const target = { did, dwnUrl, delegateDid, protocol, linkKey };
1198
+ const target = {
1199
+ did,
1200
+ dwnUrl,
1201
+ delegateDid,
1202
+ scope,
1203
+ authorization,
1204
+ authorizationEpoch : link.authorizationEpoch,
1205
+ permissionGrantIds : this.getAuthorizationGrantIds(authorization),
1206
+ linkKey,
1207
+ };
976
1208
  try {
977
1209
  await this.openLivePullSubscription(target);
978
1210
  } catch (pullErr: any) {
@@ -1013,16 +1245,15 @@ export class SyncEngineLevel implements SyncEngine {
1013
1245
  link.connectivity = 'online';
1014
1246
  await this.ledger.setStatus(link, 'live');
1015
1247
 
1016
- // Auto-clear dead letters for this link repair has verified
1017
- // convergence via SMT reconciliation so any previously recorded
1018
- // failures (closure, push-exhausted, pull-processing) for this
1019
- // specific link are no longer current.
1020
- void this.clearDeadLettersForLink(did, dwnUrl, protocol);
1021
- this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, protocol });
1248
+ // Root convergence proves primary CID membership matches, but it does
1249
+ // not prove dependencies are usable. Keep closure failures until a later
1250
+ // successful apply/closure pass clears the specific CID.
1251
+ await this.clearRootConvergenceDeadLettersForScope(did, dwnUrl, scope);
1252
+ this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope });
1022
1253
  if (prevRepairConnectivity !== 'online') {
1023
- this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevRepairConnectivity, to: 'online' });
1254
+ this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: prevRepairConnectivity, to: 'online' });
1024
1255
  }
1025
- this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: 'repairing', to: 'live' });
1256
+ this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: 'repairing', to: 'live' });
1026
1257
 
1027
1258
  } catch (error: any) {
1028
1259
  // If teardown occurred during repair or the link was replaced by a
@@ -1030,7 +1261,7 @@ export class SyncEngineLevel implements SyncEngine {
1030
1261
  if (this._engineGeneration !== generation || isStaleLink()) { return; }
1031
1262
 
1032
1263
  console.error(`SyncEngineLevel: Repair failed for ${did} -> ${dwnUrl} (attempt ${attempts})`, error);
1033
- this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl, protocol, attempt: attempts, error: String(error.message ?? error) });
1264
+ this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, attempt: attempts, error: String(error.message ?? error) });
1034
1265
 
1035
1266
  if (attempts >= SyncEngineLevel.MAX_REPAIR_ATTEMPTS) {
1036
1267
  console.warn(`SyncEngineLevel: Max repair attempts reached for ${did} -> ${dwnUrl}, entering degraded_poll`);
@@ -1048,21 +1279,26 @@ export class SyncEngineLevel implements SyncEngine {
1048
1279
  */
1049
1280
  private async closeLinkSubscriptions(link: ReplicationLinkState): Promise<void> {
1050
1281
  const { tenantDid: did, remoteEndpoint: dwnUrl } = link;
1051
- const linkKey = this.buildLinkKey(did, dwnUrl, link.scopeId);
1282
+ const linkKey = this.buildLinkKey(did, dwnUrl, link.projectionId, link.authorizationEpoch);
1052
1283
 
1053
- // Close pull subscription.
1284
+ await this.closeLiveSubscription(linkKey);
1285
+ await this.closeLocalSubscription(linkKey);
1286
+ }
1287
+
1288
+ private async closeLiveSubscription(linkKey: string): Promise<void> {
1054
1289
  const pullSub = this._liveSubscriptions.find((s) => s.linkKey === linkKey);
1055
- if (pullSub) {
1056
- try { await pullSub.close(); } catch { /* best effort */ }
1057
- this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
1058
- }
1290
+ if (!pullSub) { return; }
1291
+
1292
+ try { await pullSub.close(); } catch { /* best effort */ }
1293
+ this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
1294
+ }
1059
1295
 
1060
- // Close local push subscription.
1296
+ private async closeLocalSubscription(linkKey: string): Promise<void> {
1061
1297
  const pushSub = this._localSubscriptions.find((s) => s.linkKey === linkKey);
1062
- if (pushSub) {
1063
- try { await pushSub.close(); } catch { /* best effort */ }
1064
- this._localSubscriptions = this._localSubscriptions.filter(s => s !== pushSub);
1065
- }
1298
+ if (!pushSub) { return; }
1299
+
1300
+ try { await pushSub.close(); } catch { /* best effort */ }
1301
+ this._localSubscriptions = this._localSubscriptions.filter(s => s !== pushSub);
1066
1302
  }
1067
1303
 
1068
1304
  /**
@@ -1078,8 +1314,9 @@ export class SyncEngineLevel implements SyncEngine {
1078
1314
  const prevDegradedStatus = link.status;
1079
1315
  await this.ledger.setStatus(link, 'degraded_poll');
1080
1316
  this._repairAttempts.delete(linkKey);
1081
- this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevDegradedStatus, to: 'degraded_poll' });
1082
- this.emitEvent({ type: 'degraded-poll:entered', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol });
1317
+ const eventScope = syncEventScope(link.scope);
1318
+ this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope, from: prevDegradedStatus, to: 'degraded_poll' });
1319
+ this.emitEvent({ type: 'degraded-poll:entered', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, ...eventScope });
1083
1320
 
1084
1321
  // Clear any existing timer for this link.
1085
1322
  const existing = this._degradedPollTimers.get(linkKey);
@@ -1194,7 +1431,7 @@ export class SyncEngineLevel implements SyncEngine {
1194
1431
  type : 'link:connectivity-change',
1195
1432
  tenantDid : link.tenantDid,
1196
1433
  remoteEndpoint : link.remoteEndpoint,
1197
- protocol : link.protocol,
1434
+ ...syncEventScope(link.scope),
1198
1435
  from : prev,
1199
1436
  to : 'offline',
1200
1437
  });
@@ -1306,6 +1543,7 @@ export class SyncEngineLevel implements SyncEngine {
1306
1543
 
1307
1544
  // Clear closure evaluation contexts.
1308
1545
  this._closureContexts.clear();
1546
+ this._protocolMetadataRepairs.clear();
1309
1547
  this._recentlyPulledCids.clear();
1310
1548
 
1311
1549
  // Clear the in-memory link and runtime state.
@@ -1319,80 +1557,144 @@ export class SyncEngineLevel implements SyncEngine {
1319
1557
 
1320
1558
  /**
1321
1559
  * Initialize a single replication link target: create or resume the durable
1322
- * link, migrate legacy cursors, open pull + push subscriptions, and
1323
- * transition the link to `'live'`.
1560
+ * link, open pull + push subscriptions, and transition the link to `'live'`.
1324
1561
  */
1325
- private async initializeLinkTarget(target: {
1326
- did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
1327
- }): Promise<void> {
1562
+ private async initializeLinkTarget(target: SyncTarget): Promise<LinkInitializationResult> {
1328
1563
  let link: ReplicationLinkState | undefined;
1329
1564
  try {
1330
- const linkScope: SyncScope = target.protocol
1331
- ? { kind: 'protocol', protocol: target.protocol }
1332
- : { kind: 'full' };
1333
- link = await this.ledger.getOrCreateLink({
1334
- tenantDid : target.did,
1335
- remoteEndpoint : target.dwnUrl,
1336
- scope : linkScope,
1337
- delegateDid : target.delegateDid,
1338
- protocol : target.protocol,
1339
- });
1340
-
1341
- const linkKey = this.buildLinkKey(target.did, target.dwnUrl, link.scopeId);
1565
+ link = await this.getOrCreateReplicationLink(target);
1566
+ const linkKey = this.getReplicationLinkKey(target, link);
1567
+ this._activeLinks.set(linkKey, link);
1568
+ if (link.status === 'terminal_incomplete') {
1569
+ return this.createActiveLinkInitializationResult(link);
1570
+ }
1342
1571
 
1343
- if (!link.pull.contiguousAppliedToken) {
1344
- const legacyKey = buildLegacyCursorKey(target.did, target.dwnUrl, target.protocol);
1345
- const legacyCursor = await this.getCursor(legacyKey);
1346
- if (legacyCursor) {
1347
- ReplicationLedger.resetCheckpoint(link.pull, legacyCursor);
1348
- await this.ledger.saveLink(link);
1349
- await this.deleteLegacyCursor(legacyKey);
1350
- }
1572
+ const subscriptionResult = await this.openLinkSubscriptions({ ...target, linkKey });
1573
+ if (subscriptionResult === LinkSubscriptionOpenResult.ReadyForLive) {
1574
+ await this.markLinkLive(target, link, linkKey);
1575
+ } else if (subscriptionResult === LinkSubscriptionOpenResult.Polling) {
1576
+ await this.markLinkPolling(target, link);
1351
1577
  }
1578
+ return this.createActiveLinkInitializationResult(link);
1579
+ } catch (error: any) {
1580
+ return this.handleInitializeLinkTargetError(target, link, error);
1581
+ }
1582
+ }
1352
1583
 
1353
- this._activeLinks.set(linkKey, link);
1584
+ private async getOrCreateReplicationLink(target: SyncTarget): Promise<ReplicationLinkState> {
1585
+ return this.ledger.getOrCreateLink({
1586
+ tenantDid : target.did,
1587
+ remoteEndpoint : target.dwnUrl,
1588
+ scope : target.scope,
1589
+ authorization : target.authorization,
1590
+ authorizationEpoch : target.authorizationEpoch,
1591
+ delegateDid : target.delegateDid,
1592
+ });
1593
+ }
1354
1594
 
1355
- const targetWithKey = { ...target, linkKey };
1356
- await this.openLivePullSubscription(targetWithKey);
1357
- try {
1358
- await this.openLocalPushSubscription(targetWithKey);
1359
- } catch (pushError) {
1360
- const pullSub = this._liveSubscriptions.find((s) => s.linkKey === linkKey);
1361
- if (pullSub) {
1362
- try { await pullSub.close(); } catch { /* best effort */ }
1363
- this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
1364
- }
1365
- throw pushError;
1366
- }
1595
+ private getReplicationLinkKey(target: SyncTarget, link: ReplicationLinkState): string {
1596
+ return this.buildLinkKey(target.did, target.dwnUrl, link.projectionId, link.authorizationEpoch);
1597
+ }
1367
1598
 
1368
- this.emitEvent({ type: 'link:status-change', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, from: 'initializing', to: 'live' });
1369
- await this.ledger.setStatus(link, 'live');
1599
+ private async openLinkSubscriptions(target: LinkSyncTarget): Promise<LinkSubscriptionOpenResult> {
1600
+ if (!SyncEngineLevel.supportsLiveSubscriptions(target.scope)) {
1601
+ return LinkSubscriptionOpenResult.Polling;
1602
+ }
1370
1603
 
1371
- if (link.needsReconcile) {
1372
- this.scheduleReconcile(linkKey, 1000);
1373
- }
1374
- } catch (error: any) {
1375
- const linkKey = link
1376
- ? this.buildLinkKey(target.did, target.dwnUrl, link.scopeId)
1377
- : buildLegacyCursorKey(target.did, target.dwnUrl, target.protocol);
1378
-
1379
- if (error.isProgressGap && link) {
1380
- console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
1381
- this.emitEvent({ type: 'gap:detected', tenantDid: target.did, remoteEndpoint: target.dwnUrl, protocol: target.protocol, reason: 'ProgressGap' });
1382
- await this.transitionToRepairing(linkKey, link, {
1383
- resumeToken: error.gapInfo?.latestAvailable,
1384
- });
1385
- return;
1386
- }
1604
+ await this.openLivePullSubscription(target);
1605
+ const link = this._activeLinks.get(target.linkKey);
1606
+ if (link?.status === 'repairing') {
1607
+ await this.closeLiveSubscription(target.linkKey);
1608
+ return LinkSubscriptionOpenResult.Repairing;
1609
+ }
1387
1610
 
1388
- console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
1611
+ try {
1612
+ await this.openLocalPushSubscription(target);
1613
+ } catch (error) {
1614
+ await this.closeLiveSubscription(target.linkKey);
1615
+ throw error;
1616
+ }
1617
+ return LinkSubscriptionOpenResult.ReadyForLive;
1618
+ }
1389
1619
 
1390
- this._activeLinks.delete(linkKey);
1391
- this._linkRuntimes.delete(linkKey);
1620
+ private static supportsLiveSubscriptions(scope: SyncScope): boolean {
1621
+ // Records-primary projected links reconcile by root/diff polling until the
1622
+ // DWN has explicit path/context live subscription semantics.
1623
+ return scope.kind !== 'recordsProjection';
1624
+ }
1392
1625
 
1393
- if (this._liveSubscriptions.length === 0) {
1394
- this._connectivityState = 'unknown';
1395
- }
1626
+ private async markLinkLive(target: SyncTarget, link: ReplicationLinkState, linkKey: string): Promise<void> {
1627
+ this.emitEvent({
1628
+ type : 'link:status-change',
1629
+ tenantDid : target.did,
1630
+ remoteEndpoint : target.dwnUrl,
1631
+ ...syncEventScope(target.scope),
1632
+ from : 'initializing',
1633
+ to : 'live'
1634
+ });
1635
+ await this.ledger.setStatus(link, 'live');
1636
+
1637
+ if (link.needsReconcile) {
1638
+ this.scheduleReconcile(linkKey, 1000);
1639
+ }
1640
+ }
1641
+
1642
+ private async markLinkPolling(target: SyncTarget, link: ReplicationLinkState): Promise<void> {
1643
+ this.emitEvent({
1644
+ type : 'link:status-change',
1645
+ tenantDid : target.did,
1646
+ remoteEndpoint : target.dwnUrl,
1647
+ ...syncEventScope(target.scope),
1648
+ from : 'initializing',
1649
+ to : 'polling'
1650
+ });
1651
+ await this.ledger.setStatus(link, 'polling');
1652
+ }
1653
+
1654
+ private async handleInitializeLinkTargetError(
1655
+ target: SyncTarget,
1656
+ link: ReplicationLinkState | undefined,
1657
+ error: any,
1658
+ ): Promise<LinkInitializationResult> {
1659
+ if (error.isProgressGap && link) {
1660
+ const linkKey = this.getReplicationLinkKey(target, link);
1661
+ console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
1662
+ this.emitEvent({
1663
+ type : 'gap:detected',
1664
+ tenantDid : target.did,
1665
+ remoteEndpoint : target.dwnUrl,
1666
+ ...syncEventScope(target.scope),
1667
+ reason : 'ProgressGap'
1668
+ });
1669
+ await this.transitionToRepairing(linkKey, link, {
1670
+ resumeToken: error.gapInfo?.latestAvailable,
1671
+ });
1672
+ return this.createActiveLinkInitializationResult(link);
1673
+ }
1674
+
1675
+ console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
1676
+ if (link) {
1677
+ this.cleanupFailedLinkInitialization(this.getReplicationLinkKey(target, link));
1678
+ }
1679
+ if (this.isDidResolutionFailure(error)) {
1680
+ throw error;
1681
+ }
1682
+ return { status: LinkInitializationStatus.Failed };
1683
+ }
1684
+
1685
+ private createActiveLinkInitializationResult(link: ReplicationLinkState): LinkInitializationResult {
1686
+ return {
1687
+ status : LinkInitializationStatus.Active,
1688
+ durableLinkIdentityKey : this.getDurableLinkIdentityKey(link),
1689
+ };
1690
+ }
1691
+
1692
+ private cleanupFailedLinkInitialization(linkKey: string): void {
1693
+ this._activeLinks.delete(linkKey);
1694
+ this._linkRuntimes.delete(linkKey);
1695
+
1696
+ if (this._liveSubscriptions.length === 0) {
1697
+ this._connectivityState = 'unknown';
1396
1698
  }
1397
1699
  }
1398
1700
 
@@ -1404,31 +1706,31 @@ export class SyncEngineLevel implements SyncEngine {
1404
1706
  * causing a 401. Retrying with exponential backoff lets the
1405
1707
  * propagation settle before giving up.
1406
1708
  */
1407
- private async initializeLinkTargetWithRetry(target: {
1408
- did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
1409
- }): Promise<void> {
1709
+ private async initializeLinkTargetWithRetry(target: SyncTarget): Promise<LinkInitializationResult> {
1410
1710
  try {
1411
- await this.initializeLinkTarget(target);
1711
+ return await this.initializeLinkTarget(target);
1412
1712
  } catch (error: any) {
1413
- const msg = error.message ?? '';
1414
- const isDidResolutionFailure = msg.includes('GetPublicKeyNotFound') || msg.includes('notFound');
1415
- if (!isDidResolutionFailure) { throw error; }
1713
+ if (!this.isDidResolutionFailure(error)) { throw error; }
1416
1714
 
1417
- const delays = [2000, 4000, 8000];
1418
- for (const delay of delays) {
1715
+ for (const delay of SyncEngineLevel.DID_RESOLUTION_RETRY_BACKOFF_MS) {
1419
1716
  await sleep(delay);
1420
1717
  try {
1421
- await this.initializeLinkTarget(target);
1422
- return;
1718
+ return await this.initializeLinkTarget(target);
1423
1719
  } catch {
1424
1720
  // Continue to next attempt.
1425
1721
  }
1426
1722
  }
1427
1723
  // All retries exhausted — the original error was already logged
1428
1724
  // by initializeLinkTarget's catch block.
1725
+ return { status: LinkInitializationStatus.Failed };
1429
1726
  }
1430
1727
  }
1431
1728
 
1729
+ private isDidResolutionFailure(error: any): boolean {
1730
+ const message = error.message ?? '';
1731
+ return message.includes('GetPublicKeyNotFound');
1732
+ }
1733
+
1432
1734
  // ---------------------------------------------------------------------------
1433
1735
  // Hot-add / hot-remove: per-identity live sync management
1434
1736
  // ---------------------------------------------------------------------------
@@ -1447,23 +1749,23 @@ export class SyncEngineLevel implements SyncEngine {
1447
1749
  }
1448
1750
 
1449
1751
  /** Hot-add a single identity to the active live sync session. */
1450
- private async addIdentityToLiveSync(did: string, options: SyncIdentityOptions): Promise<void> {
1451
- const { protocols, delegateDid } = options;
1752
+ private async addIdentityToLiveSync(did: string, options: SyncIdentityOptions): Promise<Set<string>> {
1452
1753
  const dwnEndpointUrls = await this.agent.dwn.getDwnEndpointUrlsForTarget(did);
1453
- if (dwnEndpointUrls.length === 0) { return; }
1754
+ if (dwnEndpointUrls.length === 0) { return new Set(); }
1454
1755
 
1455
- const targets: { did: string; dwnUrl: string; delegateDid?: string; protocol?: string }[] = [];
1756
+ const targets: SyncTarget[] = [];
1456
1757
  for (const dwnUrl of dwnEndpointUrls) {
1457
- if (protocols === 'all') {
1458
- targets.push({ did, delegateDid, dwnUrl });
1459
- } else {
1460
- for (const protocol of protocols) {
1461
- targets.push({ did, delegateDid, dwnUrl, protocol });
1462
- }
1463
- }
1758
+ targets.push(...await this.buildSyncTargetsForEndpoint(did, dwnUrl, options));
1464
1759
  }
1465
1760
 
1466
- await Promise.allSettled(targets.map(t => this.initializeLinkTargetWithRetry(t)));
1761
+ const results = await Promise.allSettled(targets.map(t => this.initializeLinkTargetWithRetry(t)));
1762
+ const currentIdentityKeys = new Set<string>();
1763
+ for (const result of results) {
1764
+ if (result.status === 'fulfilled' && result.value.status === LinkInitializationStatus.Active) {
1765
+ currentIdentityKeys.add(result.value.durableLinkIdentityKey);
1766
+ }
1767
+ }
1768
+ return currentIdentityKeys;
1467
1769
  }
1468
1770
 
1469
1771
  /** Hot-remove a single identity from the active live sync session. */
@@ -1512,6 +1814,36 @@ export class SyncEngineLevel implements SyncEngine {
1512
1814
  this._closureContexts.delete(did);
1513
1815
  }
1514
1816
 
1817
+ private async tryPruneSupersededDurableLinksForRegisteredIdentity(did: string, options: SyncIdentityOptions): Promise<void> {
1818
+ try {
1819
+ const currentIdentityKeys = await this.getDurableLinkIdentityKeysForRegisteredIdentity(did, options);
1820
+ await this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
1821
+ } catch (error: unknown) {
1822
+ console.warn(`SyncEngineLevel: Failed to prune superseded durable links for ${did}`, error);
1823
+ }
1824
+ }
1825
+
1826
+ private async getDurableLinkIdentityKeysForRegisteredIdentity(did: string, options: SyncIdentityOptions): Promise<Set<string>> {
1827
+ const scope = syncScopeFromProtocols(options.protocols);
1828
+ const resolutions = await this.buildSyncTargetResolutions(did, scope, options);
1829
+ const keys = new Set<string>();
1830
+ for (const resolution of resolutions) {
1831
+ const projectionId = await computeProjectionId(did, resolution.scope);
1832
+ keys.add(SyncEngineLevel.durableLinkIdentityKey(did, projectionId, resolution.authorizationEpoch));
1833
+ }
1834
+ return keys;
1835
+ }
1836
+
1837
+ private async pruneSupersededDurableLinksForIdentity(did: string, currentIdentityKeys: Set<string>): Promise<void> {
1838
+ const links = await this.ledger.getLinksForTenant(did);
1839
+ await Promise.all(links.map(async link => {
1840
+ if (currentIdentityKeys.has(this.getDurableLinkIdentityKey(link))) {
1841
+ return;
1842
+ }
1843
+ await this.ledger.deleteLink(link.tenantDid, link.remoteEndpoint, link.projectionId, link.authorizationEpoch);
1844
+ }));
1845
+ }
1846
+
1515
1847
  // ---------------------------------------------------------------------------
1516
1848
  // Live pull: MessagesSubscribe to remote DWN
1517
1849
  // ---------------------------------------------------------------------------
@@ -1520,60 +1852,18 @@ export class SyncEngineLevel implements SyncEngine {
1520
1852
  * Opens a MessagesSubscribe WebSocket subscription to a remote DWN.
1521
1853
  * Incoming events are processed locally as they arrive.
1522
1854
  */
1523
- private async openLivePullSubscription(target: {
1524
- did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
1525
- linkKey: string;
1526
- }): Promise<void> {
1527
- const { did, delegateDid, dwnUrl, protocol } = target;
1855
+ private async openLivePullSubscription(target: LinkSyncTarget): Promise<void> {
1856
+ const { did, delegateDid, dwnUrl } = target;
1857
+ const eventScope = syncEventScope(target.scope);
1528
1858
 
1529
- // Resolve the cursor from the link's durable pull checkpoint.
1530
- // Legacy syncCursors migration happens at link load time in startLiveSync().
1531
1859
  const cursorKey = target.linkKey;
1532
1860
  const link = this._activeLinks.get(cursorKey);
1533
- let cursor = link?.pull.contiguousAppliedToken;
1861
+ const cursor = await this.getInitialPullCursor({ did, dwnUrl, link });
1534
1862
 
1535
- // Guard against corrupted tokens with empty fields — these would fail
1536
- // MessagesSubscribe JSON schema validation (minLength: 1). Discard and
1537
- // start from the beginning rather than crash the subscription.
1538
- if (cursor && (!cursor.streamId || !cursor.messageCid || !cursor.epoch || !cursor.position)) {
1539
- console.warn(`SyncEngineLevel: Discarding stored cursor with empty field(s) for ${did} -> ${dwnUrl}`);
1540
- cursor = undefined;
1541
- if (link) {
1542
- ReplicationLedger.resetCheckpoint(link.pull);
1543
- await this.ledger.saveLink(link);
1544
- }
1545
- }
1546
-
1547
- // Build the MessagesSubscribe filters.
1548
- // When the link has protocolPathPrefixes, include them in the filter so the
1549
- // EventLog delivers only matching events (server-side filtering). This replaces
1550
- // the less efficient agent-side isEventInScope filtering for the pull path.
1551
- // Note: only the first prefix is used as the MessagesFilter field because
1552
- // MessagesFilter.protocolPathPrefix is a single string. Multiple prefixes
1553
- // would need multiple filters (OR semantics) — for now we use the first one.
1554
- const protocolPathPrefix = link?.scope.kind === 'protocol'
1555
- ? link.scope.protocolPathPrefixes?.[0]
1556
- : undefined;
1557
- const filters = protocol
1558
- ? [{ protocol, ...(protocolPathPrefix ? { protocolPathPrefix } : {}) }]
1863
+ const filters = target.scope.kind === 'protocolSet'
1864
+ ? target.scope.protocols.map(protocol => ({ protocol }))
1559
1865
  : [];
1560
1866
 
1561
- // Look up permission grant for MessagesSubscribe if using a delegate.
1562
- // The unified scope matching in AgentPermissionsApi accepts a
1563
- // Messages.Read grant for MessagesSubscribe requests, so a single
1564
- // lookup is sufficient.
1565
- let permissionGrantId: string | undefined;
1566
- if (delegateDid) {
1567
- const grant = await this._permissionsApi.getPermissionForRequest({
1568
- connectedDid : did,
1569
- messageType : DwnInterface.MessagesSubscribe,
1570
- delegateDid,
1571
- protocol,
1572
- cached : true
1573
- });
1574
- permissionGrantId = grant.grant.id;
1575
- }
1576
-
1577
1867
  const handlerGeneration = this._engineGeneration;
1578
1868
 
1579
1869
  // Define the subscription handler that processes incoming events.
@@ -1582,249 +1872,20 @@ export class SyncEngineLevel implements SyncEngine {
1582
1872
  // ensures the checkpoint advances only when all earlier deliveries are committed.
1583
1873
  // Capture the link reference at subscription-open time so we can
1584
1874
  // detect remove+re-add via object identity, not just key existence.
1585
- const capturedLink = link;
1586
- const isStale = (): boolean =>
1587
- this._engineGeneration !== handlerGeneration ||
1588
- !this._activeLinks.has(cursorKey) ||
1589
- (capturedLink !== undefined && this._activeLinks.get(cursorKey) !== capturedLink);
1875
+ const isStale = this.createLinkStalePredicate(cursorKey, link, handlerGeneration);
1876
+ const pullContext: LivePullContext = {
1877
+ did,
1878
+ dwnUrl,
1879
+ delegateDid,
1880
+ eventScope,
1881
+ linkKey : cursorKey,
1882
+ link,
1883
+ permissionGrantIds : target.permissionGrantIds,
1884
+ isStale,
1885
+ };
1590
1886
 
1591
1887
  const subscriptionHandler = async (subMessage: SubscriptionMessage): Promise<void> => {
1592
- if (isStale()) {
1593
- return;
1594
- }
1595
-
1596
- if (subMessage.type === 'eose') {
1597
- // End-of-stored-events — catch-up complete.
1598
- if (link) {
1599
- // Guard: if the link transitioned to repairing while catch-up events
1600
- // were being processed, skip all mutations — repair owns the state now.
1601
- if (link.status !== 'live' && link.status !== 'initializing') {
1602
- return;
1603
- }
1604
-
1605
- if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
1606
- console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
1607
- if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
1608
- return;
1609
- }
1610
- ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
1611
- this.drainCommittedPull(cursorKey);
1612
- if (isStale()) { return; }
1613
- await this.ledger.saveLink(link);
1614
- }
1615
- // Transport is reachable — set connectivity to online.
1616
- if (link) {
1617
- const prevEoseConnectivity = link.connectivity;
1618
- link.connectivity = 'online';
1619
- if (prevEoseConnectivity !== 'online') {
1620
- this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevEoseConnectivity, to: 'online' });
1621
- }
1622
- // If the link was marked dirty, schedule reconciliation now that it's healthy.
1623
- if (link.needsReconcile) {
1624
- this.scheduleReconcile(cursorKey, 500);
1625
- }
1626
- } else {
1627
- this._connectivityState = 'online';
1628
- }
1629
- return;
1630
- }
1631
-
1632
- if (subMessage.type === 'event') {
1633
- const event: MessageEvent = subMessage.event;
1634
-
1635
- // Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
1636
- // skip all processing. Old subscription handlers may still fire after the
1637
- // link transitions — these events should be ignored entirely, not just
1638
- // skipped at the checkpoint level.
1639
- if (link && link.status !== 'live' && link.status !== 'initializing') {
1640
- return;
1641
- }
1642
-
1643
- // Domain validation: reject tokens from a different stream/epoch.
1644
- if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
1645
- console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
1646
- if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
1647
- return;
1648
- }
1649
-
1650
- // Subset scope filtering: if the link has protocolPath/contextId prefixes,
1651
- // skip events that don't match. This is agent-side filtering because
1652
- // MessagesSubscribe only supports protocol-level filtering today.
1653
- //
1654
- // Skipped events MUST advance contiguousAppliedToken — otherwise the
1655
- // link would replay the same filtered-out events indefinitely after
1656
- // reconnect/repair. This is safe because the event is intentionally
1657
- // excluded from this scope and doesn't need processing.
1658
- if (link && !isEventInScope(event.message, link.scope)) {
1659
- if (!isStale()) {
1660
- ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
1661
- ReplicationLedger.commitContiguousToken(link.pull, subMessage.cursor);
1662
- await this.ledger.saveLink(link);
1663
- }
1664
- return;
1665
- }
1666
-
1667
- // Assign a delivery ordinal BEFORE async processing begins.
1668
- // This captures the delivery order even if processing completes out of order.
1669
- const rt = link ? this.getOrCreateRuntime(cursorKey) : undefined;
1670
- const ordinal = rt ? rt.nextDeliveryOrdinal++ : -1;
1671
- if (rt) {
1672
- rt.inflight.set(ordinal, { ordinal, token: subMessage.cursor, committed: false });
1673
- }
1674
-
1675
- try {
1676
- // Extract inline data from the event (available for records <= 30 KB).
1677
- let dataStream = this.extractDataStream(event);
1678
-
1679
- // For large RecordsWrite messages (no inline data), fetch the data
1680
- // from the remote DWN via MessagesRead before storing locally.
1681
- if (!dataStream && isRecordsWrite(event) && (event.message.descriptor as any).dataCid) {
1682
- const messageCid = await Message.getCid(event.message);
1683
- const fetched = await fetchRemoteMessages({
1684
- did, dwnUrl, delegateDid, protocol,
1685
- messageCids : [messageCid],
1686
- agent : this.agent,
1687
- permissionsApi : this._permissionsApi,
1688
- });
1689
- if (fetched.length > 0 && fetched[0].dataStream) {
1690
- dataStream = fetched[0].dataStream;
1691
- }
1692
- }
1693
-
1694
- await this.agent.dwn.processRawMessage(did, event.message, { dataStream });
1695
- if (isStale()) { return; }
1696
-
1697
- // Invalidate closure cache entries that may be affected by this message.
1698
- // Must run before closure validation so subsequent evaluations in the
1699
- // same session see the updated local state.
1700
- const closureCtxForInvalidation = this._closureContexts.get(did);
1701
- if (closureCtxForInvalidation) {
1702
- invalidateClosureCache(closureCtxForInvalidation, event.message);
1703
- }
1704
-
1705
- // Closure validation for scoped subset sync (Phase 3).
1706
- // For protocol-scoped links, verify that all hard dependencies for
1707
- // this operation are locally present before considering it committed.
1708
- // Full-tenant scope bypasses this entirely (returns complete with 0 queries).
1709
- if (link?.scope.kind === 'protocol') {
1710
- const messageStore = this.agent.dwn.node.storage.messageStore;
1711
- let closureCtx = this._closureContexts.get(did);
1712
- if (!closureCtx) {
1713
- closureCtx = createClosureContext(did, undefined, {
1714
- isDelegateSession: !!delegateDid,
1715
- });
1716
- this._closureContexts.set(did, closureCtx);
1717
- }
1718
-
1719
- const closureResult = await evaluateClosure(
1720
- event.message, messageStore, link.scope, closureCtx
1721
- );
1722
-
1723
- if (isStale()) { return; }
1724
-
1725
- if (!closureResult.complete) {
1726
- const failureCode = closureResult.failure!.code;
1727
- const failureDetail = closureResult.failure!.detail;
1728
- console.warn(
1729
- `SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
1730
- `${failureCode} — ${failureDetail}`
1731
- );
1732
-
1733
- // Record the message that triggered the closure failure.
1734
- const closureCid = await Message.getCid(event.message);
1735
- void this.recordDeadLetter({
1736
- messageCid : closureCid,
1737
- tenantDid : did,
1738
- remoteEndpoint : dwnUrl,
1739
- protocol,
1740
- category : 'closure',
1741
- errorCode : failureCode,
1742
- errorDetail : failureDetail,
1743
- });
1744
-
1745
- if (!isStale()) { await this.transitionToRepairing(cursorKey, link); }
1746
- return;
1747
- }
1748
- }
1749
-
1750
- // Squash convergence: processRawMessage triggers the DWN's built-in
1751
- // squash resumable task (performRecordsSquash) which runs inline and
1752
- // handles subset consumers correctly:
1753
- // - If older siblings are locally present → purges them
1754
- // - If squash arrives before older siblings → backstop rejects them (409)
1755
- // - If no older siblings are local → no-op (correct)
1756
- // Both sync orderings (squash-first or siblings-first) converge to
1757
- // the same final state. No additional sync-engine side-effect is needed.
1758
-
1759
- // Track this CID for echo-loop suppression, scoped to the source endpoint.
1760
- const pulledCid = await Message.getCid(event.message);
1761
- this._recentlyPulledCids.set(`${pulledCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
1762
- this.evictExpiredEchoEntries();
1763
-
1764
- // Auto-clear any dead letter for this CID — it was processed
1765
- // successfully, so a previous failure has been self-healed.
1766
- this.clearFailedMessage(pulledCid, dwnUrl).catch(() => { /* teardown race */ });
1767
-
1768
- // Mark this ordinal as committed and drain the checkpoint.
1769
- // Guard: if the link transitioned to repairing while this handler was
1770
- // in-flight (e.g., an earlier ordinal's handler failed concurrently),
1771
- // skip all state mutations — the repair process owns progression now.
1772
- if (link && rt && link.status === 'live' && !isStale()) {
1773
- const entry = rt.inflight.get(ordinal);
1774
- if (entry) { entry.committed = true; }
1775
-
1776
- ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
1777
- const drained = this.drainCommittedPull(cursorKey);
1778
- if (drained > 0) {
1779
- await this.ledger.saveLink(link);
1780
- // Emit after durable save — "advanced" means persisted.
1781
- if (link.pull.contiguousAppliedToken) {
1782
- this.emitEvent({
1783
- type : 'checkpoint:pull-advance',
1784
- tenantDid : link.tenantDid,
1785
- remoteEndpoint : link.remoteEndpoint,
1786
- protocol : link.protocol,
1787
- position : link.pull.contiguousAppliedToken.position,
1788
- messageCid : link.pull.contiguousAppliedToken.messageCid,
1789
- });
1790
- }
1791
- }
1792
-
1793
- // Overflow: too many in-flight ordinals without draining.
1794
- if (rt.inflight.size > MAX_PENDING_TOKENS) {
1795
- console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
1796
- await this.transitionToRepairing(cursorKey, link);
1797
- }
1798
- }
1799
- } catch (error: any) {
1800
- console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
1801
-
1802
- // Record the failing message in the dead letter store before
1803
- // transitioning to repair. The CID identifies which specific
1804
- // message caused the transition.
1805
- try {
1806
- const failedCid = await Message.getCid(event.message);
1807
- void this.recordDeadLetter({
1808
- messageCid : failedCid,
1809
- tenantDid : did,
1810
- remoteEndpoint : dwnUrl,
1811
- protocol,
1812
- category : 'pull-processing',
1813
- errorDetail : error.message ?? String(error),
1814
- });
1815
- } catch {
1816
- // Best effort — don't let dead letter recording block repair.
1817
- }
1818
-
1819
- // A failed processRawMessage means local state is incomplete.
1820
- // Transition to repairing immediately — do NOT advance the checkpoint
1821
- // past this failure or let later ordinals commit past it. SMT
1822
- // reconciliation will discover and fill the gap.
1823
- if (link && !isStale()) {
1824
- await this.transitionToRepairing(cursorKey, link);
1825
- }
1826
- }
1827
- }
1888
+ await this.handleLivePullMessage(pullContext, subMessage);
1828
1889
  };
1829
1890
 
1830
1891
  // Construct the subscribe message and send it directly to the specific
@@ -1837,7 +1898,7 @@ export class SyncEngineLevel implements SyncEngine {
1837
1898
  target : did,
1838
1899
  messageType : DwnInterface.MessagesSubscribe as const,
1839
1900
  granteeDid : delegateDid,
1840
- messageParams : { filters, cursor, permissionGrantId },
1901
+ messageParams : { filters, cursor, permissionGrantIds: toMessagesPermissionGrantIds(target.permissionGrantIds) },
1841
1902
  };
1842
1903
 
1843
1904
  const { message } = await this.agent.dwn.processRequest(subscribeRequest);
@@ -1890,13 +1951,14 @@ export class SyncEngineLevel implements SyncEngine {
1890
1951
  throw new Error(`SyncEngineLevel: MessagesSubscribe failed for ${did} -> ${dwnUrl}: ${reply.status.code} ${reply.status.detail}`);
1891
1952
  }
1892
1953
 
1954
+ const linkKey = cursorKey;
1955
+ const close = async (): Promise<void> => { await reply.subscription!.close(); };
1893
1956
  this._liveSubscriptions.push({
1894
- linkKey : cursorKey,
1957
+ linkKey,
1895
1958
  did,
1896
1959
  dwnUrl,
1897
1960
  delegateDid,
1898
- protocol,
1899
- close : async (): Promise<void> => { await reply.subscription!.close(); },
1961
+ close,
1900
1962
  });
1901
1963
 
1902
1964
  // Set per-link connectivity to online after successful subscription setup.
@@ -1905,301 +1967,2020 @@ export class SyncEngineLevel implements SyncEngine {
1905
1967
  const prevPullConnectivity = pullLink.connectivity;
1906
1968
  pullLink.connectivity = 'online';
1907
1969
  if (prevPullConnectivity !== 'online') {
1908
- this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevPullConnectivity, to: 'online' });
1970
+ this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: prevPullConnectivity, to: 'online' });
1909
1971
  }
1910
1972
  }
1911
1973
  }
1912
1974
 
1913
- // ---------------------------------------------------------------------------
1914
- // Live push: local EventLog subscription for immediate push
1915
- // ---------------------------------------------------------------------------
1975
+ private async getInitialPullCursor({ did, dwnUrl, link }: {
1976
+ did: string;
1977
+ dwnUrl: string;
1978
+ link?: ReplicationLinkState;
1979
+ }): Promise<ProgressToken | undefined> {
1980
+ // Resolve the cursor from the link's durable pull checkpoint.
1981
+ if (!link) {
1982
+ return undefined;
1983
+ }
1916
1984
 
1917
- /**
1918
- * Subscribes to the local DWN's EventLog so that writes by the user are
1919
- * immediately pushed to the remote DWN instead of waiting for the next poll.
1920
- */
1921
- private async openLocalPushSubscription(target: {
1922
- did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
1985
+ const cursor = link.pull.contiguousAppliedToken;
1986
+ if (!cursor || this.isValidProgressToken(cursor)) {
1987
+ return cursor;
1988
+ }
1989
+
1990
+ // Guard against corrupted tokens with empty fields — these would fail
1991
+ // MessagesSubscribe JSON schema validation (minLength: 1). Discard and
1992
+ // start from the beginning rather than crash the subscription.
1993
+ console.warn(`SyncEngineLevel: Discarding stored cursor with empty field(s) for ${did} -> ${dwnUrl}`);
1994
+ ReplicationLedger.resetCheckpoint(link.pull);
1995
+ await this.ledger.saveLink(link);
1996
+ return undefined;
1997
+ }
1998
+
1999
+ private isValidProgressToken(token: ProgressToken): boolean {
2000
+ return !!(token.streamId && token.messageCid && token.epoch && token.position);
2001
+ }
2002
+
2003
+ private createLinkStalePredicate(
2004
+ linkKey: string,
2005
+ capturedLink: ReplicationLinkState | undefined,
2006
+ generation: number,
2007
+ ): () => boolean {
2008
+ return (): boolean =>
2009
+ this._engineGeneration !== generation ||
2010
+ !this._activeLinks.has(linkKey) ||
2011
+ (capturedLink !== undefined && this._activeLinks.get(linkKey) !== capturedLink);
2012
+ }
2013
+
2014
+ private async handleLivePullMessage(context: LivePullContext, subMessage: SubscriptionMessage): Promise<void> {
2015
+ if (context.isStale()) {
2016
+ return;
2017
+ }
2018
+
2019
+ if (subMessage.type === 'eose') {
2020
+ await this.handleLivePullEose(context, subMessage);
2021
+ return;
2022
+ }
2023
+
2024
+ if (subMessage.type === 'error') {
2025
+ await this.handleLivePullSubscriptionError(context, subMessage);
2026
+ return;
2027
+ }
2028
+
2029
+ if (subMessage.type === 'event') {
2030
+ await this.handleLivePullEvent(context, subMessage);
2031
+ }
2032
+ }
2033
+
2034
+ private async handleLivePullEose(
2035
+ { did, dwnUrl, eventScope, linkKey, link, isStale }: LivePullContext,
2036
+ subMessage: Extract<SubscriptionMessage, { type: 'eose' }>,
2037
+ ): Promise<void> {
2038
+ if (link) {
2039
+ // Guard: if the link transitioned to repairing while catch-up events
2040
+ // were being processed, skip all mutations — repair owns the state now.
2041
+ if (link.status !== 'live' && link.status !== 'initializing') {
2042
+ return;
2043
+ }
2044
+
2045
+ if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
2046
+ console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
2047
+ if (!isStale()) { await this.transitionToRepairing(linkKey, link); }
2048
+ return;
2049
+ }
2050
+ ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
2051
+ this.drainCommittedPull(linkKey);
2052
+ if (isStale()) { return; }
2053
+ await this.ledger.saveLink(link);
2054
+ }
2055
+
2056
+ this.markPullLinkOnline({ did, dwnUrl, eventScope, linkKey, link });
2057
+ }
2058
+
2059
+ private markPullLinkOnline({ did, dwnUrl, eventScope, linkKey, link }: {
2060
+ did: string;
2061
+ dwnUrl: string;
2062
+ eventScope: SyncEventScope;
1923
2063
  linkKey: string;
2064
+ link?: ReplicationLinkState;
2065
+ }): void {
2066
+ if (!link) {
2067
+ this._connectivityState = 'online';
2068
+ return;
2069
+ }
2070
+
2071
+ const previous = link.connectivity;
2072
+ link.connectivity = 'online';
2073
+ if (previous !== 'online') {
2074
+ this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope, from: previous, to: 'online' });
2075
+ }
2076
+ if (link.needsReconcile) {
2077
+ this.scheduleReconcile(linkKey, 500);
2078
+ }
2079
+ }
2080
+
2081
+ private async handleLivePullSubscriptionError(
2082
+ { did, dwnUrl, linkKey, link, isStale }: LivePullContext,
2083
+ subMessage: Extract<SubscriptionMessage, { type: 'error' }>,
2084
+ ): Promise<void> {
2085
+ console.warn(`SyncEngineLevel: subscription error for ${did} -> ${dwnUrl}: ${subMessage.error.code}`);
2086
+
2087
+ if (link && !isStale()) {
2088
+ await this.transitionToRepairing(linkKey, link);
2089
+ }
2090
+ }
2091
+
2092
+ private async handleLivePullEvent(
2093
+ context: LivePullContext,
2094
+ subMessage: Extract<SubscriptionMessage, { type: 'event' }>,
2095
+ ): Promise<void> {
2096
+ const event = subMessage.event;
2097
+ if (await this.shouldSkipLivePullEvent(context, subMessage)) {
2098
+ return;
2099
+ }
2100
+
2101
+ const delivery = this.startPullDelivery(context, subMessage.cursor);
2102
+ try {
2103
+ const pulledCid = await this.processLivePullEvent(context, event);
2104
+ if (!pulledCid) { return; }
2105
+
2106
+ this.trackRecentlyPulledMessage(pulledCid, context.dwnUrl);
2107
+ this.clearFailedMessage(pulledCid, context.dwnUrl).catch(() => { /* teardown race */ });
2108
+ await this.commitPullDelivery(context, subMessage.cursor, delivery);
2109
+ } catch (error: any) {
2110
+ await this.handleLivePullProcessingError(context, event, error);
2111
+ }
2112
+ }
2113
+
2114
+ private async shouldSkipLivePullEvent(
2115
+ { did, dwnUrl, linkKey, link, isStale }: LivePullContext,
2116
+ subMessage: Extract<SubscriptionMessage, { type: 'event' }>,
2117
+ ): Promise<boolean> {
2118
+ // Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
2119
+ // skip all processing. Old subscription handlers may still fire after the
2120
+ // link transitions — these events should be ignored entirely, not just
2121
+ // skipped at the checkpoint level.
2122
+ if (link && link.status !== 'live' && link.status !== 'initializing') {
2123
+ return true;
2124
+ }
2125
+
2126
+ // Domain validation: reject tokens from a different stream/epoch.
2127
+ if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
2128
+ console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
2129
+ if (!isStale()) { await this.transitionToRepairing(linkKey, link); }
2130
+ return true;
2131
+ }
2132
+
2133
+ if (link) {
2134
+ const scopeClassification = classifySyncEventScope(subMessage.event, link.scope);
2135
+ if (scopeClassification === 'out-of-scope') {
2136
+ await this.skipOutOfScopePullEvent({ link, cursor: subMessage.cursor, isStale });
2137
+ return true;
2138
+ }
2139
+ if (scopeClassification === 'unknown') {
2140
+ console.warn(`SyncEngineLevel: Unable to classify scoped pull event for ${did} -> ${dwnUrl}, transitioning to repair`);
2141
+ if (!isStale()) { await this.transitionToRepairing(linkKey, link); }
2142
+ return true;
2143
+ }
2144
+ }
2145
+
2146
+ return false;
2147
+ }
2148
+
2149
+ private async skipOutOfScopePullEvent({ link, cursor, isStale }: {
2150
+ link: ReplicationLinkState;
2151
+ cursor: ProgressToken;
2152
+ isStale: () => boolean;
1924
2153
  }): Promise<void> {
1925
- const { did, delegateDid, dwnUrl, protocol } = target;
2154
+ // Skipped events MUST advance contiguousAppliedToken otherwise the link
2155
+ // would replay the same filtered-out events indefinitely after reconnect or
2156
+ // repair. This is safe because the event is intentionally excluded from
2157
+ // this scope and doesn't need processing.
2158
+ if (isStale()) { return; }
2159
+
2160
+ ReplicationLedger.setReceivedToken(link.pull, cursor);
2161
+ ReplicationLedger.commitContiguousToken(link.pull, cursor);
2162
+ await this.ledger.saveLink(link);
2163
+ }
2164
+
2165
+ private startPullDelivery({ linkKey, link }: LivePullContext, cursor: ProgressToken): PullDelivery {
2166
+ // Assign a delivery ordinal BEFORE async processing begins. This captures
2167
+ // delivery order even if processing completes out of order.
2168
+ const runtime = link ? this.getOrCreateRuntime(linkKey) : undefined;
2169
+ const ordinal = runtime ? runtime.nextDeliveryOrdinal++ : -1;
2170
+ if (runtime) {
2171
+ runtime.inflight.set(ordinal, { ordinal, token: cursor, committed: false });
2172
+ }
2173
+ return { runtime, ordinal };
2174
+ }
2175
+
2176
+ private async processLivePullEvent(context: LivePullContext, event: MessageEvent): Promise<string | undefined> {
2177
+ const dataStreamFactory = await this.createLivePullDataStreamFactory(context, event);
2178
+ let applyStatus = await this.applyLivePullEvent(context, event, dataStreamFactory);
2179
+ if (context.isStale()) { return undefined; }
2180
+
2181
+ let applied = SyncEngineLevel.isApplySuccess(applyStatus.code);
2182
+ if (applied) {
2183
+ this.invalidateClosureCacheForMessage(context.did, event.message);
2184
+ }
2185
+
2186
+ if (!await this.ensureClosureComplete(context, event)) {
2187
+ return undefined;
2188
+ }
2189
+
2190
+ if (!applied) {
2191
+ applyStatus = await this.applyLivePullEvent(context, event, dataStreamFactory);
2192
+ if (context.isStale()) { return undefined; }
2193
+
2194
+ applied = SyncEngineLevel.isApplySuccess(applyStatus.code);
2195
+ if (!applied) {
2196
+ throw await this.createLivePullApplyError(event, applyStatus);
2197
+ }
2198
+ this.invalidateClosureCacheForMessage(context.did, event.message);
2199
+ }
2200
+
2201
+ // Squash convergence: processRawMessage triggers the DWN's built-in
2202
+ // squash resumable task (performRecordsSquash), so no additional
2203
+ // sync-engine side effect is needed here.
2204
+ return Message.getCid(event.message);
2205
+ }
2206
+
2207
+ private async applyLivePullEvent(
2208
+ context: LivePullContext,
2209
+ event: MessageEvent,
2210
+ dataStreamFactory: LivePullDataStreamFactory,
2211
+ ): Promise<ApplyStatus> {
2212
+ const dataStream = await dataStreamFactory();
2213
+ const reply = await this.agent.dwn.processRawMessage(context.did, event.message, { dataStream });
2214
+ return reply.status;
2215
+ }
2216
+
2217
+ private async createLivePullDataStreamFactory(
2218
+ context: LivePullContext,
2219
+ event: MessageEvent,
2220
+ ): Promise<LivePullDataStreamFactory> {
2221
+ if (!isRecordsWrite(event)) {
2222
+ return async () => undefined;
2223
+ }
2224
+
2225
+ const encodedData = (event.message as any).encodedData as string | undefined;
2226
+ if (encodedData) {
2227
+ delete (event.message as any).encodedData;
2228
+ const bytes = Encoder.base64UrlToBytes(encodedData);
2229
+ return async () => SyncEngineLevel.dataStreamFromBytes(bytes);
2230
+ }
2231
+
2232
+ const eventData = (event as any).data as ReadableStream<Uint8Array> | undefined;
2233
+ if (eventData) {
2234
+ const bytes = await SyncEngineLevel.readStreamBytes(eventData);
2235
+ return async () => SyncEngineLevel.dataStreamFromBytes(bytes);
2236
+ }
2237
+
2238
+ if (!(event.message.descriptor as any).dataCid) {
2239
+ return async () => undefined;
2240
+ }
2241
+
2242
+ // For large RecordsWrite messages (no inline data), fetch the data from
2243
+ // the remote DWN via MessagesRead before each store attempt. ReadableStream
2244
+ // instances are single-use, so a repair-triggered retry needs a fresh fetch.
2245
+ const { did, dwnUrl, delegateDid, permissionGrantIds } = context;
2246
+ const messageCid = await Message.getCid(event.message);
2247
+ return async () => {
2248
+ const fetched = await fetchRemoteMessages({
2249
+ did,
2250
+ dwnUrl,
2251
+ delegateDid,
2252
+ permissionGrantIds,
2253
+ messageCids : [messageCid],
2254
+ agent : this.agent,
2255
+ });
2256
+ return fetched[0]?.dataStream;
2257
+ };
2258
+ }
2259
+
2260
+ private async createLivePullApplyError(event: MessageEvent, status: ApplyStatus): Promise<Error> {
2261
+ const cid = await Message.getCid(event.message);
2262
+ return new Error(
2263
+ `SyncEngineLevel: live pull apply failed for ${cid}: ${status.code} ${status.detail ?? ''}`.trim()
2264
+ );
2265
+ }
2266
+
2267
+ private static dataStreamFromBytes(bytes: Uint8Array): ReadableStream<Uint8Array> {
2268
+ return new ReadableStream<Uint8Array>({
2269
+ start(controller): void {
2270
+ controller.enqueue(bytes);
2271
+ controller.close();
2272
+ }
2273
+ });
2274
+ }
2275
+
2276
+ private static async readStreamBytes(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
2277
+ const reader = stream.getReader();
2278
+ const chunks: Uint8Array[] = [];
2279
+ let totalSize = 0;
2280
+
2281
+ try {
2282
+ for (;;) {
2283
+ const { done, value } = await reader.read();
2284
+ if (done) { break; }
2285
+ chunks.push(value);
2286
+ totalSize += value.byteLength;
2287
+ }
2288
+ } finally {
2289
+ reader.releaseLock();
2290
+ }
2291
+
2292
+ const bytes = new Uint8Array(totalSize);
2293
+ let offset = 0;
2294
+ for (const chunk of chunks) {
2295
+ bytes.set(chunk, offset);
2296
+ offset += chunk.byteLength;
2297
+ }
2298
+ return bytes;
2299
+ }
2300
+
2301
+ private static isApplySuccess(code: number): boolean {
2302
+ return (code >= 200 && code < 300) || code === 409;
2303
+ }
2304
+
2305
+ private invalidateClosureCacheForMessage(did: string, message: GenericMessage): void {
2306
+ // Must run before closure validation so subsequent evaluations in the same
2307
+ // session see the updated local state.
2308
+ const closureCtx = this._closureContexts.get(did);
2309
+ if (closureCtx) {
2310
+ invalidateClosureCache(closureCtx, message);
2311
+ }
2312
+ }
2313
+
2314
+ private async ensureClosureComplete(context: LivePullContext, event: MessageEvent): Promise<boolean> {
2315
+ const { did, delegateDid, link, isStale } = context;
2316
+ if (!link || link.scope.kind === 'full') {
2317
+ return true;
2318
+ }
2319
+
2320
+ let closureCtx = this._closureContexts.get(did);
2321
+ if (!closureCtx) {
2322
+ closureCtx = createClosureContext(did, undefined, {
2323
+ isDelegateSession: !!delegateDid,
2324
+ });
2325
+ this._closureContexts.set(did, closureCtx);
2326
+ }
2327
+
2328
+ const messageStore = this.agent.dwn.node.storage.messageStore;
2329
+ let closureResult = await evaluateClosure(event.message, messageStore, link.scope, closureCtx);
2330
+ if (isStale()) { return false; }
2331
+ if (closureResult.complete) { return true; }
2332
+
2333
+ if (await this.tryRepairMissingProtocolMetadata(context, closureCtx, closureResult)) {
2334
+ if (isStale()) { return false; }
2335
+
2336
+ closureResult = await evaluateClosure(event.message, messageStore, link.scope, closureCtx);
2337
+ if (isStale()) { return false; }
2338
+ if (closureResult.complete) { return true; }
2339
+ }
2340
+
2341
+ await this.recordClosureFailure(context, event, closureResult.failure!.code, closureResult.failure!.detail);
2342
+ return false;
2343
+ }
2344
+
2345
+ private async tryRepairMissingProtocolMetadata(
2346
+ context: LivePullContext,
2347
+ closureCtx: ClosureEvaluationContext,
2348
+ closureResult: ClosureResult,
2349
+ ): Promise<boolean> {
2350
+ const failure = closureResult.failure;
2351
+ if (!SyncEngineLevel.isRepairableProtocolMetadataFailure(failure)) {
2352
+ return false;
2353
+ }
2354
+
2355
+ const { did } = context;
2356
+ const repairKey = `${did}|${failure.edge.identifier}`;
2357
+ const activeRepair = this._protocolMetadataRepairs.get(repairKey);
2358
+ if (activeRepair) {
2359
+ return activeRepair;
2360
+ }
2361
+
2362
+ const repair = this.repairMissingProtocolMetadata(context, closureCtx, failure.edge.identifier);
2363
+ this._protocolMetadataRepairs.set(repairKey, repair);
2364
+ repair.finally(() => {
2365
+ if (this._protocolMetadataRepairs.get(repairKey) === repair) {
2366
+ this._protocolMetadataRepairs.delete(repairKey);
2367
+ }
2368
+ }).catch(() => { /* caller handles the repair result */ });
2369
+ return repair;
2370
+ }
2371
+
2372
+ private async repairMissingProtocolMetadata(
2373
+ { did, dwnUrl, delegateDid, isStale }: LivePullContext,
2374
+ closureCtx: ClosureEvaluationContext,
2375
+ protocol: string,
2376
+ ): Promise<boolean> {
2377
+ if (isStale()) {
2378
+ return false;
2379
+ }
2380
+
2381
+ const configs = await this.fetchRemoteProtocolConfigClosure({
2382
+ authorDid : delegateDid ?? did,
2383
+ delegateDid,
2384
+ dwnUrl,
2385
+ protocol,
2386
+ tenantDid : did,
2387
+ });
2388
+ if (isStale() || configs.length === 0) {
2389
+ return false;
2390
+ }
2391
+
2392
+ // Live subscriptions can deliver scoped records before the local replica
2393
+ // has the tenant's protocol metadata. Reuse the DWN ProtocolsQuery path and
2394
+ // only install configs that are signed by the tenant, including composed
2395
+ // protocol dependencies needed to authorize the record.
2396
+ let repaired = false;
2397
+ for (const config of configs) {
2398
+ if (isStale()) {
2399
+ return repaired;
2400
+ }
2401
+ const reply = await this.agent.dwn.processRawMessage(did, config);
2402
+ if (isStale()) {
2403
+ return repaired;
2404
+ }
2405
+ if (!SyncEngineLevel.protocolConfigApplySucceeded(reply.status.code)) {
2406
+ return repaired;
2407
+ }
2408
+
2409
+ invalidateClosureCache(closureCtx, config);
2410
+ repaired = true;
2411
+ }
2412
+
2413
+ return repaired;
2414
+ }
2415
+
2416
+ private async fetchRemoteProtocolConfigClosure({
2417
+ authorDid,
2418
+ delegateDid,
2419
+ dwnUrl,
2420
+ protocol,
2421
+ tenantDid,
2422
+ }: {
2423
+ authorDid: string;
2424
+ delegateDid?: string;
2425
+ dwnUrl: string;
2426
+ protocol: string;
2427
+ tenantDid: string;
2428
+ }): Promise<ProtocolsConfigureMessage[]> {
2429
+ const configsByProtocol = new Map<string, ProtocolsConfigureMessage>();
2430
+ const visiting = new Set<string>();
2431
+
2432
+ const visit = async (protocolUri: string): Promise<boolean> => {
2433
+ if (configsByProtocol.has(protocolUri)) {
2434
+ return true;
2435
+ }
2436
+ if (visiting.has(protocolUri)) {
2437
+ return true;
2438
+ }
2439
+ visiting.add(protocolUri);
2440
+
2441
+ const config = await this.fetchRemoteProtocolConfig({
2442
+ authorDid,
2443
+ delegateDid,
2444
+ dwnUrl,
2445
+ protocol: protocolUri,
2446
+ tenantDid,
2447
+ });
2448
+ if (config === undefined) {
2449
+ visiting.delete(protocolUri);
2450
+ return false;
2451
+ }
2452
+
2453
+ for (const usedProtocol of SyncEngineLevel.protocolsConfigureUses(config)) {
2454
+ if (!await visit(usedProtocol)) {
2455
+ visiting.delete(protocolUri);
2456
+ return false;
2457
+ }
2458
+ }
2459
+
2460
+ configsByProtocol.set(protocolUri, config);
2461
+ visiting.delete(protocolUri);
2462
+ return true;
2463
+ };
2464
+
2465
+ return await visit(protocol) ? [...configsByProtocol.values()] : [];
2466
+ }
2467
+
2468
+ private async fetchRemoteProtocolConfig({
2469
+ authorDid,
2470
+ delegateDid,
2471
+ dwnUrl,
2472
+ protocol,
2473
+ tenantDid,
2474
+ }: {
2475
+ authorDid: string;
2476
+ delegateDid?: string;
2477
+ dwnUrl: string;
2478
+ protocol: string;
2479
+ tenantDid: string;
2480
+ }): Promise<ProtocolsConfigureMessage | undefined> {
2481
+ try {
2482
+ const permissionGrantId = await this.getProtocolsQueryPermissionGrantId({
2483
+ delegateDid,
2484
+ protocol,
2485
+ tenantDid,
2486
+ });
2487
+ const { message } = await this.agent.processDwnRequest({
2488
+ author : authorDid,
2489
+ messageParams : {
2490
+ filter: { protocol },
2491
+ ...(permissionGrantId === undefined ? {} : { permissionGrantId }),
2492
+ },
2493
+ messageType : DwnInterface.ProtocolsQuery,
2494
+ store : false,
2495
+ target : tenantDid,
2496
+ });
2497
+
2498
+ const reply = await this.agent.rpc.sendDwnRequest({
2499
+ dwnUrl,
2500
+ message,
2501
+ targetDid: tenantDid,
2502
+ }) as ProtocolsQueryReply;
2503
+ if (reply.status.code !== 200 || reply.entries === undefined) {
2504
+ return undefined;
2505
+ }
2506
+
2507
+ const candidates: ProtocolsConfigureMessage[] = [];
2508
+ for (const entry of reply.entries) {
2509
+ const config = await this.toAuthenticatedTenantProtocolConfig(tenantDid, entry);
2510
+ if (config?.descriptor.definition.protocol === protocol) {
2511
+ candidates.push(config);
2512
+ }
2513
+ }
2514
+
2515
+ return SyncEngineLevel.newestProtocolConfig(candidates);
2516
+ } catch {
2517
+ return undefined;
2518
+ }
2519
+ }
2520
+
2521
+ private async getProtocolsQueryPermissionGrantId({
2522
+ delegateDid,
2523
+ protocol,
2524
+ tenantDid,
2525
+ }: {
2526
+ delegateDid?: string;
2527
+ protocol: string;
2528
+ tenantDid: string;
2529
+ }): Promise<string | undefined> {
2530
+ if (delegateDid === undefined) {
2531
+ return undefined;
2532
+ }
2533
+
2534
+ try {
2535
+ const { grant } = await this._permissionsApi.getPermissionForRequest({
2536
+ connectedDid : tenantDid,
2537
+ delegateDid,
2538
+ protocol,
2539
+ cached : true,
2540
+ messageType : DwnInterface.ProtocolsQuery,
2541
+ });
2542
+ return grant.id;
2543
+ } catch {
2544
+ return undefined;
2545
+ }
2546
+ }
2547
+
2548
+ private async toAuthenticatedTenantProtocolConfig(
2549
+ tenantDid: string,
2550
+ message: GenericMessage,
2551
+ ): Promise<ProtocolsConfigureMessage | undefined> {
2552
+ if (!SyncEngineLevel.isProtocolsConfigureDefinitionMessage(message)) {
2553
+ return undefined;
2554
+ }
2555
+
2556
+ try {
2557
+ await ProtocolsConfigure.parse(message);
2558
+ if (Message.getAuthor(message) !== tenantDid) {
2559
+ return undefined;
2560
+ }
2561
+ await authenticate(message.authorization, this.agent.did);
2562
+ return message;
2563
+ } catch {
2564
+ return undefined;
2565
+ }
2566
+ }
2567
+
2568
+ private static newestProtocolConfig(
2569
+ configs: ProtocolsConfigureMessage[],
2570
+ ): ProtocolsConfigureMessage | undefined {
2571
+ let newest: ProtocolsConfigureMessage | undefined;
2572
+ for (const config of configs) {
2573
+ if (newest === undefined || SyncEngineLevel.isProtocolConfigNewer(config, newest)) {
2574
+ newest = config;
2575
+ }
2576
+ }
2577
+ return newest;
2578
+ }
2579
+
2580
+ private static isProtocolConfigNewer(
2581
+ candidate: ProtocolsConfigureMessage,
2582
+ current: ProtocolsConfigureMessage,
2583
+ ): boolean {
2584
+ return candidate.descriptor.messageTimestamp > current.descriptor.messageTimestamp;
2585
+ }
2586
+
2587
+ private static protocolConfigApplySucceeded(code: number): boolean {
2588
+ return (code >= 200 && code < 300) || code === 409;
2589
+ }
2590
+
2591
+ private static isRepairableProtocolMetadataFailure(
2592
+ failure: ClosureResult['failure'] | undefined,
2593
+ ): failure is NonNullable<ClosureResult['failure']> {
2594
+ return failure?.edge.identifierType === 'protocol' &&
2595
+ (
2596
+ failure.code === ClosureFailureCode.ProtocolMetadataMissing ||
2597
+ failure.code === ClosureFailureCode.CrossProtocolReferenceMissing ||
2598
+ failure.code === ClosureFailureCode.EncryptionDependencyMissing
2599
+ );
2600
+ }
2601
+
2602
+ private async recordClosureFailure(
2603
+ { did, dwnUrl, linkKey, link, isStale }: LivePullContext,
2604
+ event: MessageEvent,
2605
+ failureCode: string,
2606
+ failureDetail: string,
2607
+ ): Promise<void> {
2608
+ console.warn(
2609
+ `SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
2610
+ `${failureCode} — ${failureDetail}`
2611
+ );
2612
+
2613
+ const closureCid = await Message.getCid(event.message);
2614
+ void this.recordDeadLetter({
2615
+ messageCid : closureCid,
2616
+ tenantDid : did,
2617
+ remoteEndpoint : dwnUrl,
2618
+ protocol : (event.message.descriptor as Record<string, unknown>).protocol as string | undefined,
2619
+ category : 'closure',
2620
+ errorCode : failureCode,
2621
+ errorDetail : failureDetail,
2622
+ });
2623
+
2624
+ if (link && !isStale() && isTerminalClosureFailureCode(failureCode)) {
2625
+ await this.transitionToTerminalIncomplete(linkKey, link);
2626
+ return;
2627
+ }
2628
+
2629
+ if (link && !isStale()) {
2630
+ await this.transitionToRepairing(linkKey, link);
2631
+ }
2632
+ }
2633
+
2634
+ private trackRecentlyPulledMessage(messageCid: string, dwnUrl: string): void {
2635
+ this._recentlyPulledCids.set(`${messageCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
2636
+ this.evictExpiredEchoEntries();
2637
+ }
2638
+
2639
+ private async commitPullDelivery(
2640
+ { did, dwnUrl, linkKey, link, isStale }: LivePullContext,
2641
+ cursor: ProgressToken,
2642
+ delivery: PullDelivery,
2643
+ ): Promise<void> {
2644
+ // Guard: if the link transitioned to repairing while this handler was
2645
+ // in-flight, skip all state mutations — the repair process owns progression.
2646
+ if (!link || !delivery.runtime || link.status !== 'live' || isStale()) {
2647
+ return;
2648
+ }
2649
+
2650
+ const entry = delivery.runtime.inflight.get(delivery.ordinal);
2651
+ if (entry) { entry.committed = true; }
2652
+
2653
+ ReplicationLedger.setReceivedToken(link.pull, cursor);
2654
+ const drained = this.drainCommittedPull(linkKey);
2655
+ if (drained > 0) {
2656
+ await this.ledger.saveLink(link);
2657
+ this.emitPullCheckpointAdvance(link);
2658
+ }
2659
+
2660
+ if (delivery.runtime.inflight.size > MAX_PENDING_TOKENS) {
2661
+ console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
2662
+ await this.transitionToRepairing(linkKey, link);
2663
+ }
2664
+ }
2665
+
2666
+ private emitPullCheckpointAdvance(link: ReplicationLinkState): void {
2667
+ if (!link.pull.contiguousAppliedToken) {
2668
+ return;
2669
+ }
2670
+
2671
+ // Emit after durable save — "advanced" means persisted.
2672
+ this.emitEvent({
2673
+ type : 'checkpoint:pull-advance',
2674
+ tenantDid : link.tenantDid,
2675
+ remoteEndpoint : link.remoteEndpoint,
2676
+ ...syncEventScope(link.scope),
2677
+ position : link.pull.contiguousAppliedToken.position,
2678
+ messageCid : link.pull.contiguousAppliedToken.messageCid,
2679
+ });
2680
+ }
2681
+
2682
+ private async handleLivePullProcessingError(
2683
+ { did, dwnUrl, linkKey, link, isStale }: LivePullContext,
2684
+ event: MessageEvent,
2685
+ error: any,
2686
+ ): Promise<void> {
2687
+ console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
2688
+ await this.recordPullProcessingFailure({ did, dwnUrl, event, error });
2689
+
2690
+ // A failed processRawMessage means local state is incomplete. Transition
2691
+ // to repairing immediately — do NOT advance the checkpoint past this
2692
+ // failure or let later ordinals commit past it. SMT reconciliation will
2693
+ // discover and fill the gap.
2694
+ if (link && !isStale()) {
2695
+ await this.transitionToRepairing(linkKey, link);
2696
+ }
2697
+ }
2698
+
2699
+ private async recordPullProcessingFailure({ did, dwnUrl, event, error }: {
2700
+ did: string;
2701
+ dwnUrl: string;
2702
+ event: MessageEvent;
2703
+ error: any;
2704
+ }): Promise<void> {
2705
+ try {
2706
+ const failedCid = await Message.getCid(event.message);
2707
+ void this.recordDeadLetter({
2708
+ messageCid : failedCid,
2709
+ tenantDid : did,
2710
+ remoteEndpoint : dwnUrl,
2711
+ protocol : (event.message.descriptor as Record<string, unknown>).protocol as string | undefined,
2712
+ category : 'pull-processing',
2713
+ errorDetail : error.message ?? String(error),
2714
+ });
2715
+ } catch {
2716
+ // Best effort — don't let dead letter recording block repair.
2717
+ }
2718
+ }
2719
+
2720
+ // ---------------------------------------------------------------------------
2721
+ // Live push: local EventLog subscription for immediate push
2722
+ // ---------------------------------------------------------------------------
2723
+
2724
+ /**
2725
+ * Subscribes to the local DWN's EventLog so that writes by the user are
2726
+ * immediately pushed to the remote DWN instead of waiting for the next poll.
2727
+ */
2728
+ private async openLocalPushSubscription(target: LinkSyncTarget): Promise<void> {
2729
+ const { did, delegateDid, dwnUrl } = target;
2730
+ const protocol = singleProtocolForSyncScope(target.scope);
2731
+
2732
+ const filters = target.scope.kind === 'protocolSet'
2733
+ ? target.scope.protocols.map(protocol => ({ protocol }))
2734
+ : [];
2735
+
2736
+ const handlerGeneration = this._engineGeneration;
2737
+
2738
+ // Capture the link for identity-based staleness detection.
2739
+ const capturedPushLink = this._activeLinks.get(target.linkKey);
2740
+ const isPushStale = (): boolean =>
2741
+ this._engineGeneration !== handlerGeneration ||
2742
+ !this._activeLinks.has(target.linkKey) ||
2743
+ (capturedPushLink !== undefined && this._activeLinks.get(target.linkKey) !== capturedPushLink);
2744
+
2745
+ // Subscribe to the local DWN's EventLog.
2746
+ const subscriptionHandler = async (subMessage: SubscriptionMessage): Promise<void> => {
2747
+ if (isPushStale()) {
2748
+ return;
2749
+ }
2750
+
2751
+ if (subMessage.type !== 'event') {
2752
+ return;
2753
+ }
2754
+
2755
+ // Subset scope filtering: only push events that match the link scope.
2756
+ // Events outside the scope are not this link's responsibility.
2757
+ const pushLinkKey = target.linkKey;
2758
+ const pushLink = this._activeLinks.get(pushLinkKey);
2759
+ if (pushLink) {
2760
+ const scopeClassification = classifySyncEventScope(subMessage.event, pushLink.scope);
2761
+ if (scopeClassification === 'out-of-scope') {
2762
+ return;
2763
+ }
2764
+ if (scopeClassification === 'unknown') {
2765
+ this.markLinkNeedsReconcile(pushLinkKey, pushLink, 'push-scope-unclassified');
2766
+ return;
2767
+ }
2768
+ }
2769
+
2770
+ // Accumulate the message CID for a debounced push.
2771
+ const targetKey = pushLinkKey;
2772
+ const cid = await Message.getCid(subMessage.event.message);
2773
+ if (cid === undefined || isPushStale()) {
2774
+ return;
2775
+ }
2776
+
2777
+ // Echo-loop suppression: skip CIDs that were recently pulled from this
2778
+ // specific remote. A message pulled from Provider A is only suppressed
2779
+ // for push to A — it still fans out to Provider B and C.
2780
+ if (this.isRecentlyPulled(cid, dwnUrl)) {
2781
+ return;
2782
+ }
2783
+
2784
+ const pushRuntime = this.getOrCreatePushRuntime(targetKey, {
2785
+ did,
2786
+ dwnUrl,
2787
+ delegateDid,
2788
+ protocol,
2789
+ permissionGrantIds: target.permissionGrantIds,
2790
+ });
2791
+ pushRuntime.entries.push({ cid });
2792
+
2793
+ // Immediate-first: if no push is in flight and no batch timer is
2794
+ // pending, push immediately. Otherwise, the pending batch timer
2795
+ // or the post-flush drain will pick up the new entry.
2796
+ if (!pushRuntime.flushing && !pushRuntime.timer) {
2797
+ void this.flushPendingPushesForLink(targetKey);
2798
+ }
2799
+ };
2800
+
2801
+ // Subscribe to the local DWN EventLog from "now" — opportunistic push
2802
+ // does not replay from a stored cursor. Any writes missed during outages
2803
+ // are recovered by the post-repair reconciliation path.
2804
+ const response = await this.agent.dwn.processRequest({
2805
+ author : did,
2806
+ target : did,
2807
+ messageType : DwnInterface.MessagesSubscribe,
2808
+ granteeDid : delegateDid,
2809
+ messageParams : { filters, permissionGrantIds: toMessagesPermissionGrantIds(target.permissionGrantIds) },
2810
+ subscriptionHandler : subscriptionHandler as any,
2811
+ });
2812
+
2813
+ const reply = response.reply as MessagesSubscribeReply;
2814
+ if (reply.status.code !== 200 || !reply.subscription) {
2815
+ throw new Error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
2816
+ }
2817
+
2818
+ const close = async (): Promise<void> => { await reply.subscription!.close(); };
2819
+ this._localSubscriptions.push({
2820
+ linkKey: target.linkKey,
2821
+ did,
2822
+ dwnUrl,
2823
+ delegateDid,
2824
+ close,
2825
+ });
2826
+ }
2827
+
2828
+ /**
2829
+ * Flushes accumulated push CIDs to remote DWNs.
2830
+ */
2831
+ private async flushPendingPushes(): Promise<void> {
2832
+ await Promise.all([...this._pushRuntimes.keys()].map(async (linkKey) => {
2833
+ await this.flushPendingPushesForLink(linkKey);
2834
+ }));
2835
+ }
2836
+
2837
+ private async flushPendingPushesForLink(linkKey: string): Promise<void> {
2838
+ const batch = this.takePushFlushBatch(linkKey);
2839
+ if (!batch) { return; }
2840
+
2841
+ const { pushRuntime, pushEntries, isStale } = batch;
2842
+ const { did, dwnUrl, delegateDid, protocol, permissionGrantIds, retryCount } = pushRuntime;
2843
+
2844
+ try {
2845
+ const result = await pushMessages({
2846
+ did,
2847
+ dwnUrl,
2848
+ delegateDid,
2849
+ permissionGrantIds,
2850
+ messageCids : pushEntries.map((entry) => entry.cid),
2851
+ agent : this.agent,
2852
+ });
2853
+
2854
+ await this.handlePushBatchResult(linkKey, batch, result);
2855
+ } catch (error: any) {
2856
+ if (isStale()) { return; }
2857
+ console.error(`SyncEngineLevel: Push batch failed for ${did} -> ${dwnUrl}`, error);
2858
+ this.requeueOrReconcile(linkKey, {
2859
+ did,
2860
+ dwnUrl,
2861
+ delegateDid,
2862
+ protocol,
2863
+ permissionGrantIds,
2864
+ entries : pushEntries,
2865
+ retryCount : retryCount + 1,
2866
+ });
2867
+ } finally {
2868
+ this.finishPushFlush(linkKey, pushRuntime);
2869
+ }
2870
+ }
2871
+
2872
+ private takePushFlushBatch(linkKey: string): PushFlushBatch | undefined {
2873
+ // Guard: bail if this link was hot-removed or is no longer live. Without
2874
+ // this, a stale debounce timer or retry callback could send pushes after
2875
+ // the DID was removed or the link entered repair/terminal state.
2876
+ const flushLink = this._activeLinks.get(linkKey);
2877
+ if (flushLink?.status !== 'live') {
2878
+ const staleRuntime = this._pushRuntimes.get(linkKey);
2879
+ if (staleRuntime?.timer) {
2880
+ clearTimeout(staleRuntime.timer);
2881
+ }
2882
+ this._pushRuntimes.delete(linkKey);
2883
+ return undefined;
2884
+ }
2885
+
2886
+ const pushRuntime = this._pushRuntimes.get(linkKey);
2887
+ if (!pushRuntime) {
2888
+ return undefined;
2889
+ }
2890
+
2891
+ const { entries: pushEntries, retryCount } = pushRuntime;
2892
+ pushRuntime.entries = [];
2893
+
2894
+ if (pushEntries.length === 0) {
2895
+ if (!pushRuntime.timer && !pushRuntime.flushing && retryCount === 0) {
2896
+ this._pushRuntimes.delete(linkKey);
2897
+ }
2898
+ return undefined;
2899
+ }
2900
+
2901
+ // Capture the current active link identity so we can detect
2902
+ // remove+re-add during the await pushMessages() call.
2903
+ const isStale = (): boolean =>
2904
+ !this._activeLinks.has(linkKey) ||
2905
+ (flushLink !== undefined && this._activeLinks.get(linkKey) !== flushLink);
2906
+
2907
+ pushRuntime.flushing = true;
2908
+ return { pushRuntime, pushEntries, isStale };
2909
+ }
2910
+
2911
+ private async handlePushBatchResult(
2912
+ linkKey: string,
2913
+ batch: PushFlushBatch,
2914
+ result: PushResult,
2915
+ ): Promise<void> {
2916
+ if (batch.isStale()) { return; }
2917
+
2918
+ this.clearSucceededPushFailures(result.succeeded, batch.pushRuntime.dwnUrl);
2919
+ await this.recordPermanentPushFailures(batch.pushRuntime, result.permanentlyFailed);
2920
+
2921
+ if (result.failed.length > 0) {
2922
+ this.requeueFailedPushes(linkKey, batch, result.failed);
2923
+ return;
2924
+ }
2925
+
2926
+ this.cleanupSuccessfulPushRuntime(linkKey, batch.pushRuntime);
2927
+ }
2928
+
2929
+ private clearSucceededPushFailures(cids: string[], dwnUrl: string): void {
2930
+ for (const cid of cids) {
2931
+ this.clearFailedMessage(cid, dwnUrl).catch(() => { /* teardown race */ });
2932
+ }
2933
+ }
2934
+
2935
+ private async recordPermanentPushFailures(
2936
+ pushRuntime: PushRuntimeState,
2937
+ permanentlyFailed: PushResult['permanentlyFailed'],
2938
+ ): Promise<void> {
2939
+ for (const entry of permanentlyFailed) {
2940
+ await this.recordDeadLetter({
2941
+ messageCid : entry.cid,
2942
+ tenantDid : pushRuntime.did,
2943
+ remoteEndpoint : pushRuntime.dwnUrl,
2944
+ protocol : pushRuntime.protocol,
2945
+ category : 'push-permanent',
2946
+ errorCode : String(entry.statusCode ?? ''),
2947
+ errorDetail : entry.detail ?? 'permanent push failure',
2948
+ });
2949
+ }
2950
+ }
2951
+
2952
+ private requeueFailedPushes(linkKey: string, batch: PushFlushBatch, failedCids: string[]): void {
2953
+ if (batch.isStale()) { return; }
2954
+
2955
+ const { did, dwnUrl, delegateDid, protocol, permissionGrantIds, retryCount } = batch.pushRuntime;
2956
+ const failedSet = new Set(failedCids);
2957
+ const failedEntries = batch.pushEntries.filter((entry) => failedSet.has(entry.cid));
2958
+ this.requeueOrReconcile(linkKey, {
2959
+ did,
2960
+ dwnUrl,
2961
+ delegateDid,
2962
+ protocol,
2963
+ permissionGrantIds,
2964
+ entries : failedEntries,
2965
+ retryCount : retryCount + 1,
2966
+ });
2967
+ }
2968
+
2969
+ private cleanupSuccessfulPushRuntime(linkKey: string, pushRuntime: PushRuntimeState): void {
2970
+ // Successful push — reset retry count so subsequent unrelated batches on
2971
+ // this link start with a fresh budget.
2972
+ pushRuntime.retryCount = 0;
2973
+ if (!pushRuntime.timer && pushRuntime.entries.length === 0) {
2974
+ this._pushRuntimes.delete(linkKey);
2975
+ }
2976
+ }
2977
+
2978
+ private finishPushFlush(linkKey: string, pushRuntime: PushRuntimeState): void {
2979
+ pushRuntime.flushing = false;
2980
+
2981
+ // If new entries accumulated while this push was in flight, schedule a
2982
+ // short drain to flush them. This gives a brief batching window for burst
2983
+ // writes while keeping single-write latency low.
2984
+ const rt = this._pushRuntimes.get(linkKey);
2985
+ if (rt && rt.entries.length > 0 && !rt.timer) {
2986
+ rt.timer = setTimeout((): void => {
2987
+ rt.timer = undefined;
2988
+ void this.flushPendingPushesForLink(linkKey);
2989
+ }, PUSH_DEBOUNCE_MS);
2990
+ }
2991
+ }
2992
+
2993
+ /** Push retry backoff schedule: immediate, 250ms, 1s, 2s, then give up. */
2994
+ private static readonly PUSH_RETRY_BACKOFF_MS = [0, 250, 1000, 2000];
2995
+ private static readonly ROOT_CONVERGENCE_CLEARABLE_DEAD_LETTER_CATEGORIES: ReadonlySet<DeadLetterCategory> =
2996
+ new Set(['push-permanent', 'push-exhausted', 'pull-processing', 'pull-scope-rejected']);
2997
+
2998
+ /**
2999
+ * Re-queues a failed push batch for retry, or marks the link
3000
+ * `needsReconcile` if retries are exhausted. Bounded to prevent
3001
+ * infinite retry loops.
3002
+ */
3003
+ private requeueOrReconcile(targetKey: string, pending: {
3004
+ did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
3005
+ permissionGrantIds?: NonEmptyStringArray;
3006
+ entries: PushRuntimeEntry[];
3007
+ retryCount: number;
3008
+ }): void {
3009
+ const maxRetries = SyncEngineLevel.PUSH_RETRY_BACKOFF_MS.length;
3010
+ const pushRuntime = this.getOrCreatePushRuntime(targetKey, pending);
3011
+
3012
+ if (pending.retryCount >= maxRetries) {
3013
+ // Retry budget exhausted — record each CID as a dead letter and mark
3014
+ // the link dirty for reconciliation.
3015
+ for (const entry of pending.entries) {
3016
+ void this.recordDeadLetter({
3017
+ messageCid : entry.cid,
3018
+ tenantDid : pending.did,
3019
+ remoteEndpoint : pending.dwnUrl,
3020
+ protocol : pending.protocol,
3021
+ category : 'push-exhausted',
3022
+ errorDetail : `push retries exhausted after ${maxRetries} attempts`,
3023
+ });
3024
+ }
3025
+ if (pushRuntime.timer) {
3026
+ clearTimeout(pushRuntime.timer);
3027
+ }
3028
+ this._pushRuntimes.delete(targetKey);
3029
+ const link = this._activeLinks.get(targetKey);
3030
+ if (link) {
3031
+ this.markLinkNeedsReconcile(targetKey, link, 'push-retry-exhausted');
3032
+ }
3033
+ return;
3034
+ }
3035
+
3036
+ pushRuntime.entries.push(...pending.entries);
3037
+ pushRuntime.retryCount = pending.retryCount;
3038
+ const delayMs = SyncEngineLevel.PUSH_RETRY_BACKOFF_MS[pending.retryCount] ?? 2000;
3039
+ if (pushRuntime.timer) {
3040
+ clearTimeout(pushRuntime.timer);
3041
+ }
3042
+ pushRuntime.timer = setTimeout((): void => {
3043
+ pushRuntime.timer = undefined;
3044
+ void this.flushPendingPushesForLink(targetKey);
3045
+ }, delayMs);
3046
+ }
3047
+
3048
+ private markLinkNeedsReconcile(linkKey: string, link: ReplicationLinkState, reason: string): void {
3049
+ if (link.needsReconcile) {
3050
+ this.scheduleReconcile(linkKey);
3051
+ return;
3052
+ }
3053
+
3054
+ link.needsReconcile = true;
3055
+ void this.ledger.saveLink(link).then(() => {
3056
+ this.emitEvent({
3057
+ type : 'reconcile:needed',
3058
+ tenantDid : link.tenantDid,
3059
+ remoteEndpoint : link.remoteEndpoint,
3060
+ ...syncEventScope(link.scope),
3061
+ reason,
3062
+ });
3063
+ this.scheduleReconcile(linkKey);
3064
+ }).catch((error: unknown) => {
3065
+ console.error(`SyncEngineLevel: Failed to mark link for reconciliation ${link.tenantDid} -> ${link.remoteEndpoint}`, error);
3066
+ });
3067
+ }
3068
+
3069
+ private createLinkReconciler(shouldContinue?: () => boolean): SyncLinkReconciler {
3070
+ return new SyncLinkReconciler({
3071
+ getLocalRoot : async (did, delegateDid, protocol, permissionGrantIds) => this.getLocalRoot(did, delegateDid, protocol, permissionGrantIds),
3072
+ getRemoteRoot : async (did, dwnUrl, delegateDid, protocol, permissionGrantIds) =>
3073
+ this.getRemoteRoot(did, dwnUrl, delegateDid, protocol, permissionGrantIds),
3074
+ diffWithRemote : async (target) => this.diffWithRemote(target),
3075
+ pullMessages : async (params) => this.pullMessages(params),
3076
+ pushMessages : async (params) => this.pushMessages(params),
3077
+ shouldContinue,
3078
+ });
3079
+ }
3080
+
3081
+ private getReconcileProtocols(scope: SyncScope): (string | undefined)[] {
3082
+ return protocolsForSyncScope(scope) ?? [undefined];
3083
+ }
3084
+
3085
+ private getAuthorizationGrantIds(authorization: SyncAuthorization): NonEmptyStringArray | undefined {
3086
+ return authorization.kind === 'delegate' ? authorization.permissionGrantIds : undefined;
3087
+ }
3088
+
3089
+ private async reconcileProjectionTarget(
3090
+ target: ProjectionReconcileTarget,
3091
+ options?: ProjectionReconcileOptions,
3092
+ shouldContinue?: () => boolean,
3093
+ ): Promise<ProjectionReconcileResult> {
3094
+ if (target.scope.kind === 'recordsProjection') {
3095
+ return this.reconcileRecordsProjectionTarget(target, target.scope, options, shouldContinue);
3096
+ }
3097
+
3098
+ if (target.scope.kind === 'protocolSet' && target.scope.protocols.length > 1) {
3099
+ return this.reconcileProtocolSetProjectionTarget(target, options, shouldContinue);
3100
+ }
3101
+
3102
+ let converged = true;
3103
+ const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
3104
+ const reconciler = this.createLinkReconciler(shouldContinue);
3105
+
3106
+ for (const protocol of this.getReconcileProtocols(target.scope)) {
3107
+ const outcome = await reconciler.reconcile({
3108
+ did : target.did,
3109
+ dwnUrl : target.dwnUrl,
3110
+ delegateDid : target.delegateDid,
3111
+ protocol,
3112
+ permissionGrantIds,
3113
+ }, options);
3114
+ if (outcome.aborted) {
3115
+ return { aborted: true };
3116
+ }
3117
+ if (options?.verifyConvergence === true && outcome.converged !== true) {
3118
+ converged = false;
3119
+ }
3120
+ }
3121
+
3122
+ return options?.verifyConvergence === true ? { converged } : {};
3123
+ }
3124
+
3125
+ private async reconcileRecordsProjectionTarget(
3126
+ target: ProjectionReconcileTarget,
3127
+ scope: RecordsProjectionSyncScope,
3128
+ options?: ProjectionReconcileOptions,
3129
+ shouldContinue?: () => boolean,
3130
+ ): Promise<ProjectionReconcileResult> {
3131
+ const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
3132
+ const localRoot = await this.getLocalProjectedRoot(target.did, target.delegateDid, scope.scopes, permissionGrantIds);
3133
+ if (shouldContinue?.() === false) { return { aborted: true }; }
3134
+
3135
+ const remoteRoot = await this.getRemoteProjectedRoot(target.did, target.dwnUrl, target.delegateDid, scope.scopes, permissionGrantIds);
3136
+ if (shouldContinue?.() === false) { return { aborted: true }; }
3137
+
3138
+ if (localRoot !== remoteRoot) {
3139
+ const diff = await this.diffProjectedWithRemote({
3140
+ did : target.did,
3141
+ dwnUrl : target.dwnUrl,
3142
+ delegateDid : target.delegateDid,
3143
+ scopes : scope.scopes,
3144
+ permissionGrantIds,
3145
+ });
3146
+ if (shouldContinue?.() === false) { return { aborted: true }; }
3147
+
3148
+ const aborted = await this.applyProjectedDiff(target, scope, diff, permissionGrantIds, options, shouldContinue);
3149
+ if (aborted) { return { aborted: true }; }
3150
+ }
3151
+
3152
+ if (options?.verifyConvergence !== true) {
3153
+ return {};
3154
+ }
3155
+
3156
+ const postLocalRoot = await this.getLocalProjectedRoot(target.did, target.delegateDid, scope.scopes, permissionGrantIds);
3157
+ if (shouldContinue?.() === false) { return { aborted: true }; }
3158
+
3159
+ const postRemoteRoot = await this.getRemoteProjectedRoot(target.did, target.dwnUrl, target.delegateDid, scope.scopes, permissionGrantIds);
3160
+ if (shouldContinue?.() === false) { return { aborted: true }; }
3161
+
3162
+ return { converged: postLocalRoot === postRemoteRoot };
3163
+ }
3164
+
3165
+ private async reconcileProtocolSetProjectionTarget(
3166
+ target: ProjectionReconcileTarget,
3167
+ options?: ProjectionReconcileOptions,
3168
+ shouldContinue?: () => boolean,
3169
+ ): Promise<ProjectionReconcileResult> {
3170
+ if (target.scope.kind !== 'protocolSet') {
3171
+ return {};
3172
+ }
3173
+
3174
+ const scope = target.scope;
3175
+ const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
3176
+ const diffPlan = await this.collectProtocolSetDiffPlan(target, scope, permissionGrantIds, shouldContinue);
3177
+ if (!diffPlan) {
3178
+ return { aborted: true };
3179
+ }
3180
+
3181
+ if (diffPlan.changedProtocols.length === 0) {
3182
+ return options?.verifyConvergence === true ? { converged: true } : {};
3183
+ }
3184
+
3185
+ const aborted = await this.applyProtocolSetDiffPlan(target, scope, diffPlan, permissionGrantIds, options, shouldContinue);
3186
+ if (aborted) {
3187
+ return { aborted: true };
3188
+ }
3189
+
3190
+ if (options?.verifyConvergence !== true) {
3191
+ return {};
3192
+ }
3193
+
3194
+ return this.verifyProtocolSetConvergence(target, diffPlan.changedProtocols, permissionGrantIds, shouldContinue);
3195
+ }
3196
+
3197
+ private async collectProtocolSetDiffPlan(
3198
+ target: ProjectionReconcileTarget,
3199
+ scope: ProtocolSetScope,
3200
+ permissionGrantIds: NonEmptyStringArray | undefined,
3201
+ shouldContinue?: () => boolean,
3202
+ ): Promise<ProtocolSetDiffPlan | undefined> {
3203
+ const plan: ProtocolSetDiffPlan = { changedProtocols: [], onlyRemote: [], onlyLocal: [] };
3204
+
3205
+ for (const protocol of scope.protocols) {
3206
+ const roots = await this.getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue);
3207
+ if (!roots) { return undefined; }
3208
+
3209
+ if (roots.localRoot === roots.remoteRoot) {
3210
+ continue;
3211
+ }
3212
+
3213
+ plan.changedProtocols.push(protocol);
3214
+ const diff = await this.diffWithRemote({
3215
+ did : target.did,
3216
+ dwnUrl : target.dwnUrl,
3217
+ delegateDid : target.delegateDid,
3218
+ protocol,
3219
+ permissionGrantIds,
3220
+ });
3221
+ if (shouldContinue?.() === false) { return undefined; }
3222
+
3223
+ plan.onlyRemote.push(...diff.onlyRemote);
3224
+ plan.onlyLocal.push(...diff.onlyLocal);
3225
+ }
3226
+
3227
+ return plan;
3228
+ }
3229
+
3230
+ private async getProtocolRoots(
3231
+ target: ProjectionReconcileTarget,
3232
+ protocol: string,
3233
+ permissionGrantIds: NonEmptyStringArray | undefined,
3234
+ shouldContinue?: () => boolean,
3235
+ ): Promise<{ localRoot: string; remoteRoot: string } | undefined> {
3236
+ const localRoot = await this.getLocalRoot(target.did, target.delegateDid, protocol, permissionGrantIds);
3237
+ if (shouldContinue?.() === false) { return undefined; }
3238
+
3239
+ const remoteRoot = await this.getRemoteRoot(target.did, target.dwnUrl, target.delegateDid, protocol, permissionGrantIds);
3240
+ if (shouldContinue?.() === false) { return undefined; }
3241
+
3242
+ return { localRoot, remoteRoot };
3243
+ }
3244
+
3245
+ private async applyProtocolSetDiffPlan(
3246
+ target: ProjectionReconcileTarget,
3247
+ scope: ProtocolSetScope,
3248
+ diffPlan: ProtocolSetDiffPlan,
3249
+ permissionGrantIds: NonEmptyStringArray | undefined,
3250
+ options?: ProjectionReconcileOptions,
3251
+ shouldContinue?: () => boolean,
3252
+ ): Promise<boolean> {
3253
+ // Keep the remote diff combined across protocols so topologicalSort can
3254
+ // order composed protocol configs before records that use them. Any future
3255
+ // chunking for large protocol sets must preserve this global dependency
3256
+ // order instead of reverting to independent per-protocol chunks.
3257
+ if (
3258
+ options?.direction !== 'push' &&
3259
+ diffPlan.onlyRemote.length > 0 &&
3260
+ await this.pullRemoteDiffEntries(target, scope, diffPlan.onlyRemote, permissionGrantIds, shouldContinue)
3261
+ ) {
3262
+ return true;
3263
+ }
3264
+
3265
+ if (options?.direction === 'pull' || diffPlan.onlyLocal.length === 0) {
3266
+ return false;
3267
+ }
3268
+
3269
+ return this.pushLocalDiffEntries(target, diffPlan.onlyLocal, permissionGrantIds, shouldContinue);
3270
+ }
3271
+
3272
+ private async applyProjectedDiff(
3273
+ target: ProjectionReconcileTarget,
3274
+ scope: RecordsProjectionSyncScope,
3275
+ diff: ProjectionDiffResult,
3276
+ permissionGrantIds: NonEmptyStringArray | undefined,
3277
+ options?: ProjectionReconcileOptions,
3278
+ shouldContinue?: () => boolean,
3279
+ ): Promise<boolean> {
3280
+ if (await this.pullProjectedRemoteDiff(target, scope, diff, permissionGrantIds, options, shouldContinue)) {
3281
+ return true;
3282
+ }
3283
+
3284
+ return this.pushProjectedLocalDiff(target, diff.onlyLocal, permissionGrantIds, options, shouldContinue);
3285
+ }
3286
+
3287
+ private async pullProjectedRemoteDiff(
3288
+ target: ProjectionReconcileTarget,
3289
+ scope: RecordsProjectionSyncScope,
3290
+ diff: ProjectionDiffResult,
3291
+ permissionGrantIds: NonEmptyStringArray | undefined,
3292
+ options?: ProjectionReconcileOptions,
3293
+ shouldContinue?: () => boolean,
3294
+ ): Promise<boolean> {
3295
+ if (options?.direction === 'push' || diff.onlyRemote.length === 0) {
3296
+ return false;
3297
+ }
3298
+
3299
+ return this.pullRemoteDiffEntries(target, scope, diff.onlyRemote, permissionGrantIds, shouldContinue, diff.dependencies ?? []);
3300
+ }
3301
+
3302
+ private async pushProjectedLocalDiff(
3303
+ target: ProjectionReconcileTarget,
3304
+ onlyLocal: string[],
3305
+ permissionGrantIds: NonEmptyStringArray | undefined,
3306
+ options?: ProjectionReconcileOptions,
3307
+ shouldContinue?: () => boolean,
3308
+ ): Promise<boolean> {
3309
+ if (options?.direction === 'pull' || onlyLocal.length === 0) {
3310
+ return false;
3311
+ }
3312
+
3313
+ return this.pushLocalDiffEntries(target, onlyLocal, permissionGrantIds, shouldContinue);
3314
+ }
3315
+
3316
+ private async pullRemoteDiffEntries(
3317
+ target: ProjectionReconcileTarget,
3318
+ scope: SyncScope,
3319
+ onlyRemote: MessagesSyncDiffEntry[],
3320
+ permissionGrantIds: NonEmptyStringArray | undefined,
3321
+ shouldContinue?: () => boolean,
3322
+ dependencies: MessagesSyncDependencyEntry[] = [],
3323
+ ): Promise<boolean> {
3324
+ const primaryEntries = SyncEngineLevel.dedupeRemoteEntries(onlyRemote);
3325
+ try {
3326
+ let verifiedInitialWrites: RecordsWriteMessage[] = [];
3327
+ if (scope.kind === 'recordsProjection') {
3328
+ verifiedInitialWrites = await this.pullProjectedDependencyHints(
3329
+ target,
3330
+ scope,
3331
+ primaryEntries,
3332
+ dependencies,
3333
+ permissionGrantIds,
3334
+ shouldContinue,
3335
+ );
3336
+ }
3337
+
3338
+ const { prefetched, needsFetchCids } = partitionRemoteEntries(primaryEntries);
3339
+ await this.pullMessages({
3340
+ did : target.did,
3341
+ dwnUrl : target.dwnUrl,
3342
+ delegateDid : target.delegateDid,
3343
+ scope,
3344
+ permissionGrantIds,
3345
+ messageCids : needsFetchCids,
3346
+ prefetched,
3347
+ verifiedInitialWrites,
3348
+ shouldContinue,
3349
+ });
3350
+ } catch (error) {
3351
+ if (error instanceof SyncPullAbortedError) {
3352
+ return true;
3353
+ }
3354
+ throw error;
3355
+ }
3356
+ return shouldContinue?.() === false;
3357
+ }
3358
+
3359
+ private async pullProjectedDependencyHints(
3360
+ target: ProjectionReconcileTarget,
3361
+ scope: RecordsProjectionSyncScope,
3362
+ primaryEntries: MessagesSyncDiffEntry[],
3363
+ dependencies: MessagesSyncDependencyEntry[],
3364
+ permissionGrantIds: NonEmptyStringArray | undefined,
3365
+ shouldContinue?: () => boolean,
3366
+ ): Promise<RecordsWriteMessage[]> {
3367
+ const verified = await this.verifyProjectedDependencies(target.did, scope, primaryEntries, dependencies);
3368
+ if (verified.length === 0) {
3369
+ return [];
3370
+ }
3371
+
3372
+ await this.pullMessages({
3373
+ did : target.did,
3374
+ dwnUrl : target.dwnUrl,
3375
+ delegateDid : target.delegateDid,
3376
+ scope : SyncEngineLevel.protocolSetScopeForProjectedDependencies(verified),
3377
+ permissionGrantIds,
3378
+ messageCids : [],
3379
+ prefetched : verified,
3380
+ shouldContinue,
3381
+ });
3382
+
3383
+ return SyncEngineLevel.recordsInitialWritesFromVerifiedDependencies(verified);
3384
+ }
3385
+
3386
+ private async verifyProjectedDependencies(
3387
+ tenantDid: string,
3388
+ scope: RecordsProjectionSyncScope,
3389
+ primaryEntries: MessagesSyncDiffEntry[],
3390
+ dependencies: MessagesSyncDependencyEntry[],
3391
+ ): Promise<MessagesSyncDependencyEntry[]> {
3392
+ const primaryByCid = SyncEngineLevel.indexEntriesWithMessage(primaryEntries);
3393
+ const initialWritesByRoot = await this.collectProjectedRecordsInitialWriteDependencies(
3394
+ scope,
3395
+ primaryByCid,
3396
+ dependencies,
3397
+ );
3398
+ const protocolConfigs = await this.verifyProjectedProtocolConfigDependencies(
3399
+ tenantDid,
3400
+ scope,
3401
+ primaryEntries,
3402
+ dependencies,
3403
+ initialWritesByRoot,
3404
+ );
3405
+
3406
+ return SyncEngineLevel.dedupeDependencyEntries([
3407
+ ...protocolConfigs,
3408
+ ...initialWritesByRoot.values(),
3409
+ ]);
3410
+ }
3411
+
3412
+ private async verifyProjectedProtocolConfigDependencies(
3413
+ tenantDid: string,
3414
+ scope: RecordsProjectionSyncScope,
3415
+ primaryEntries: MessagesSyncDiffEntry[],
3416
+ dependencies: MessagesSyncDependencyEntry[],
3417
+ initialWritesByRoot: Map<string, AuthenticatedRecordsInitialWriteDependency> = new Map(),
3418
+ ): Promise<MessagesSyncDependencyEntry[]> {
3419
+ // Projected sync dependency entries are untrusted server hints. Before any
3420
+ // config is applied, bind it to an accepted primary record by CID,
3421
+ // tenant authorship, signature, timestamp, scope, and protocol closure;
3422
+ // malformed or unrelated hints are ignored.
3423
+ const primaryByCid = SyncEngineLevel.indexEntriesWithMessage(primaryEntries);
3424
+ const candidatesByRoot = await this.collectProjectedProtocolConfigCandidates(
3425
+ tenantDid,
3426
+ scope,
3427
+ primaryByCid,
3428
+ dependencies,
3429
+ initialWritesByRoot,
3430
+ );
3431
+ const verified = new Map<string, MessagesSyncDependencyEntry>();
3432
+
3433
+ for (const [rootMessageCid, rootCandidates] of candidatesByRoot) {
3434
+ const primary = primaryByCid.get(rootMessageCid);
3435
+ const rootRecordsWrite = primary === undefined
3436
+ ? undefined
3437
+ : SyncEngineLevel.protocolConfigRootRecordsWrite(primary.message, initialWritesByRoot.get(rootMessageCid)?.message);
3438
+ const rootProtocol = SyncEngineLevel.recordsWriteProtocol(rootRecordsWrite);
3439
+ if (rootProtocol === undefined) {
3440
+ continue;
3441
+ }
3442
+
3443
+ for (const dependency of SyncEngineLevel.filterProtocolConfigClosure(rootProtocol, rootCandidates)) {
3444
+ verified.set(dependency.messageCid, dependency);
3445
+ }
3446
+ }
3447
+
3448
+ return [...verified.values()];
3449
+ }
3450
+
3451
+ private static indexEntriesWithMessage(
3452
+ entries: MessagesSyncDiffEntry[],
3453
+ ): Map<string, SyncDiffEntryWithMessage> {
3454
+ const entriesByCid = new Map<string, SyncDiffEntryWithMessage>();
3455
+ for (const entry of entries) {
3456
+ if (SyncEngineLevel.hasMessage(entry)) {
3457
+ entriesByCid.set(entry.messageCid, entry);
3458
+ }
3459
+ }
3460
+ return entriesByCid;
3461
+ }
3462
+
3463
+ private static recordsInitialWritesFromVerifiedDependencies(
3464
+ entries: MessagesSyncDependencyEntry[],
3465
+ ): RecordsWriteMessage[] {
3466
+ const initialWrites: RecordsWriteMessage[] = [];
3467
+ for (const entry of entries) {
3468
+ if (SyncEngineLevel.hasMessage(entry) && SyncEngineLevel.isRecordsWriteMessage(entry.message)) {
3469
+ initialWrites.push(entry.message);
3470
+ }
3471
+ }
3472
+ return initialWrites;
3473
+ }
3474
+
3475
+ private async collectProjectedRecordsInitialWriteDependencies(
3476
+ scope: RecordsProjectionSyncScope,
3477
+ primaryByCid: Map<string, SyncDiffEntryWithMessage>,
3478
+ dependencies: MessagesSyncDependencyEntry[],
3479
+ ): Promise<Map<string, AuthenticatedRecordsInitialWriteDependency>> {
3480
+ const dependenciesByRoot = new Map<string, AuthenticatedRecordsInitialWriteDependency>();
3481
+ for (const dependency of dependencies) {
3482
+ const verified = await this.verifyRecordsInitialWriteCandidate(scope, primaryByCid, dependency);
3483
+ if (verified === undefined) {
3484
+ continue;
3485
+ }
3486
+
3487
+ dependenciesByRoot.set(verified.rootMessageCid, verified.dependency);
3488
+ }
3489
+ return dependenciesByRoot;
3490
+ }
3491
+
3492
+ private async verifyRecordsInitialWriteCandidate(
3493
+ scope: RecordsProjectionSyncScope,
3494
+ primaryByCid: Map<string, SyncDiffEntryWithMessage>,
3495
+ dependency: MessagesSyncDependencyEntry,
3496
+ ): Promise<VerifiedRecordsInitialWriteCandidate | undefined> {
3497
+ if (dependency.dependencyClass !== 'recordsInitialWrite' ||
3498
+ !SyncEngineLevel.hasMessage(dependency) ||
3499
+ SyncEngineLevel.hasDependencyPayloadBytes(dependency)) {
3500
+ return undefined;
3501
+ }
3502
+
3503
+ const primary = primaryByCid.get(dependency.rootMessageCid);
3504
+ if (primary === undefined ||
3505
+ !SyncEngineLevel.isRecordsDeleteMessage(primary.message) ||
3506
+ !await SyncEngineLevel.projectedDependencyCidsMatch({
3507
+ dependencyCid : dependency.messageCid,
3508
+ dependencyMessage : dependency.message,
3509
+ primaryCid : primary.messageCid,
3510
+ primaryMessage : primary.message,
3511
+ })) {
3512
+ return undefined;
3513
+ }
3514
+
3515
+ const initialWrite = await this.toAuthenticatedRecordsInitialWriteDependency(dependency);
3516
+ if (initialWrite === undefined ||
3517
+ initialWrite.message.recordId !== SyncEngineLevel.recordsDeleteRecordId(primary.message) ||
3518
+ classifySyncMessageScope({ message: primary.message, initialWrite: initialWrite.message, scope }) !== 'in-scope') {
3519
+ return undefined;
3520
+ }
3521
+
3522
+ return { dependency: initialWrite, rootMessageCid: dependency.rootMessageCid };
3523
+ }
3524
+
3525
+ private async toAuthenticatedRecordsInitialWriteDependency(
3526
+ dependency: SyncDependencyEntryWithMessage,
3527
+ ): Promise<AuthenticatedRecordsInitialWriteDependency | undefined> {
3528
+ if (!SyncEngineLevel.isRecordsWriteMessage(dependency.message)) {
3529
+ return undefined;
3530
+ }
3531
+
3532
+ try {
3533
+ const recordsWrite = await RecordsWrite.parse(dependency.message);
3534
+ await authenticate(recordsWrite.message.authorization, this.agent.did, recordsWrite.message.attestation);
3535
+ return await recordsWrite.isInitialWrite()
3536
+ ? { ...dependency, message: recordsWrite.message }
3537
+ : undefined;
3538
+ } catch {
3539
+ return undefined;
3540
+ }
3541
+ }
3542
+
3543
+ private async collectProjectedProtocolConfigCandidates(
3544
+ tenantDid: string,
3545
+ scope: RecordsProjectionSyncScope,
3546
+ primaryByCid: Map<string, SyncDiffEntryWithMessage>,
3547
+ dependencies: MessagesSyncDependencyEntry[],
3548
+ initialWritesByRoot: Map<string, AuthenticatedRecordsInitialWriteDependency>,
3549
+ ): Promise<Map<string, AuthenticatedProtocolConfigDependency[]>> {
3550
+ const candidatesByRoot = new Map<string, AuthenticatedProtocolConfigDependency[]>();
3551
+ for (const dependency of dependencies) {
3552
+ const verified = await this.verifyProtocolConfigCandidate(tenantDid, scope, primaryByCid, dependency, initialWritesByRoot);
3553
+ if (verified === undefined) {
3554
+ continue;
3555
+ }
3556
+
3557
+ const rootCandidates = candidatesByRoot.get(verified.rootMessageCid) ?? [];
3558
+ rootCandidates.push(verified.dependency);
3559
+ candidatesByRoot.set(verified.rootMessageCid, rootCandidates);
3560
+ }
3561
+ return candidatesByRoot;
3562
+ }
3563
+
3564
+ private async verifyProtocolConfigCandidate(
3565
+ tenantDid: string,
3566
+ scope: RecordsProjectionSyncScope,
3567
+ primaryByCid: Map<string, SyncDiffEntryWithMessage>,
3568
+ dependency: MessagesSyncDependencyEntry,
3569
+ initialWritesByRoot: Map<string, AuthenticatedRecordsInitialWriteDependency>,
3570
+ ): Promise<VerifiedProtocolConfigCandidate | undefined> {
3571
+ if (dependency.dependencyClass !== 'protocolsConfigure' || !SyncEngineLevel.hasMessage(dependency)) {
3572
+ return undefined;
3573
+ }
3574
+
3575
+ const primary = primaryByCid.get(dependency.rootMessageCid);
3576
+ if (primary === undefined) {
3577
+ return undefined;
3578
+ }
3579
+
3580
+ const verifiedDependency = await this.verifyProtocolConfigCandidateMessage(
3581
+ tenantDid,
3582
+ scope,
3583
+ primary,
3584
+ dependency,
3585
+ initialWritesByRoot.get(dependency.rootMessageCid)?.message,
3586
+ );
3587
+ return verifiedDependency === undefined
3588
+ ? undefined
3589
+ : { dependency: verifiedDependency, rootMessageCid: dependency.rootMessageCid };
3590
+ }
3591
+
3592
+ private async verifyProtocolConfigCandidateMessage(
3593
+ tenantDid: string,
3594
+ scope: RecordsProjectionSyncScope,
3595
+ primary: SyncDiffEntryWithMessage,
3596
+ dependency: SyncDependencyEntryWithMessage,
3597
+ initialWrite: RecordsWriteMessage | undefined,
3598
+ ): Promise<AuthenticatedProtocolConfigDependency | undefined> {
3599
+ // Protocol authorization is temporal: a record is governed by the protocol
3600
+ // definition active at its creation timestamp. Future configs may add
3601
+ // unrelated `uses` dependencies, so they must not widen this primary's
3602
+ // dependency closure.
3603
+ if (!await SyncEngineLevel.projectedDependencyCidsMatch({
3604
+ dependencyCid : dependency.messageCid,
3605
+ dependencyMessage : dependency.message,
3606
+ primaryCid : primary.messageCid,
3607
+ primaryMessage : primary.message,
3608
+ })) {
3609
+ return undefined;
3610
+ }
3611
+
3612
+ const authenticatedDependency = await this.toAuthenticatedProtocolConfigDependency(tenantDid, dependency);
3613
+ if (authenticatedDependency === undefined) {
3614
+ return undefined;
3615
+ }
3616
+
3617
+ const rootRecordsWrite = SyncEngineLevel.protocolConfigRootRecordsWrite(primary.message, initialWrite);
3618
+ const primaryIsInScope = rootRecordsWrite !== undefined &&
3619
+ classifySyncMessageScope({ message: primary.message, initialWrite, scope }) === 'in-scope';
3620
+ if (!primaryIsInScope ||
3621
+ !SyncEngineLevel.protocolsConfigureIsNotNewerThanRecordsWrite(authenticatedDependency.message, rootRecordsWrite)) {
3622
+ return undefined;
3623
+ }
3624
+
3625
+ return authenticatedDependency;
3626
+ }
3627
+
3628
+ private async toAuthenticatedProtocolConfigDependency(
3629
+ tenantDid: string,
3630
+ dependency: SyncDependencyEntryWithMessage,
3631
+ ): Promise<AuthenticatedProtocolConfigDependency | undefined> {
3632
+ const config = await this.toAuthenticatedTenantProtocolConfig(tenantDid, dependency.message);
3633
+ if (config === undefined) {
3634
+ return undefined;
3635
+ }
3636
+
3637
+ return { ...dependency, message: config };
3638
+ }
3639
+
3640
+ private static async projectedDependencyCidsMatch({
3641
+ dependencyCid,
3642
+ dependencyMessage,
3643
+ primaryCid,
3644
+ primaryMessage,
3645
+ }: {
3646
+ dependencyCid: string;
3647
+ dependencyMessage: GenericMessage;
3648
+ primaryCid: string;
3649
+ primaryMessage: GenericMessage;
3650
+ }): Promise<boolean> {
3651
+ return await Message.getCid(primaryMessage) === primaryCid &&
3652
+ await Message.getCid(dependencyMessage) === dependencyCid;
3653
+ }
3654
+
3655
+ private static recordsWriteProtocol(message: GenericMessage | undefined): string | undefined {
3656
+ if (!SyncEngineLevel.isRecordsWriteProtocolMessage(message)) {
3657
+ return undefined;
3658
+ }
3659
+
3660
+ const { protocol } = message.descriptor;
3661
+ return typeof protocol === 'string' ? protocol : undefined;
3662
+ }
3663
+
3664
+ private static recordsDeleteRecordId(message: GenericMessage): string | undefined {
3665
+ if (!SyncEngineLevel.isRecordsDeleteMessage(message)) {
3666
+ return undefined;
3667
+ }
3668
+
3669
+ const recordId = (message.descriptor as Record<string, unknown>).recordId;
3670
+ return typeof recordId === 'string' ? recordId : undefined;
3671
+ }
3672
+
3673
+ private static protocolConfigRootRecordsWrite(
3674
+ primary: GenericMessage,
3675
+ initialWrite: RecordsWriteMessage | undefined,
3676
+ ): RecordsWriteMessage | undefined {
3677
+ if (SyncEngineLevel.isRecordsWriteMessage(primary)) {
3678
+ return primary;
3679
+ }
3680
+
3681
+ return SyncEngineLevel.isRecordsDeleteMessage(primary) ? initialWrite : undefined;
3682
+ }
3683
+
3684
+ private static protocolsConfigureProtocol(message: ProtocolsConfigureMessage): string {
3685
+ return message.descriptor.definition.protocol;
3686
+ }
3687
+
3688
+ private static protocolsConfigureProtocolFromGenericMessage(message: GenericMessage): string | undefined {
3689
+ if (!SyncEngineLevel.isProtocolsConfigureDefinitionMessage(message)) {
3690
+ return undefined;
3691
+ }
1926
3692
 
1927
- // Build filters scoped to the protocol (if any).
1928
- const filters = protocol ? [{ protocol }] : [];
3693
+ return message.descriptor.definition.protocol;
3694
+ }
1929
3695
 
1930
- // Look up permission grant for local subscription.
1931
- let permissionGrantId: string | undefined;
1932
- if (delegateDid) {
1933
- const grant = await this._permissionsApi.getPermissionForRequest({
1934
- connectedDid : did,
1935
- messageType : DwnInterface.MessagesSubscribe,
1936
- delegateDid,
1937
- protocol,
1938
- cached : true,
1939
- });
1940
- permissionGrantId = grant.grant.id;
1941
- }
3696
+ private static protocolsConfigureIsNotNewerThanRecordsWrite(
3697
+ protocolsConfigureMessage: GenericMessage,
3698
+ recordsWriteMessage: GenericMessage,
3699
+ ): boolean {
3700
+ return protocolsConfigureMessage.descriptor.messageTimestamp <= recordsWriteMessage.descriptor.messageTimestamp;
3701
+ }
1942
3702
 
1943
- const handlerGeneration = this._engineGeneration;
3703
+ private static protocolsConfigureUses(message: ProtocolsConfigureMessage): string[] {
3704
+ const uses = message.descriptor.definition?.uses;
3705
+ return uses === undefined
3706
+ ? []
3707
+ : Object.values(uses).filter((protocol): protocol is string => typeof protocol === 'string');
3708
+ }
1944
3709
 
1945
- // Capture the link for identity-based staleness detection.
1946
- const capturedPushLink = this._activeLinks.get(target.linkKey);
1947
- const isPushStale = (): boolean =>
1948
- this._engineGeneration !== handlerGeneration ||
1949
- !this._activeLinks.has(target.linkKey) ||
1950
- (capturedPushLink !== undefined && this._activeLinks.get(target.linkKey) !== capturedPushLink);
3710
+ private static hasMessage<T extends MessagesSyncDiffEntry>(
3711
+ entry: T | undefined,
3712
+ ): entry is T & { message: GenericMessage } {
3713
+ return entry?.message !== undefined;
3714
+ }
1951
3715
 
1952
- // Subscribe to the local DWN's EventLog.
1953
- const subscriptionHandler = async (subMessage: SubscriptionMessage): Promise<void> => {
1954
- if (isPushStale()) {
1955
- return;
1956
- }
3716
+ private static isRecordsWriteProtocolMessage(message: GenericMessage | undefined): message is RecordsWriteProtocolMessage {
3717
+ return message?.descriptor.interface === DwnInterfaceName.Records &&
3718
+ message.descriptor.method === DwnMethodName.Write;
3719
+ }
1957
3720
 
1958
- if (subMessage.type !== 'event') {
1959
- return;
1960
- }
3721
+ private static isRecordsWriteMessage(message: GenericMessage | undefined): message is RecordsWriteMessage {
3722
+ return SyncEngineLevel.isRecordsWriteProtocolMessage(message) &&
3723
+ 'recordId' in message &&
3724
+ typeof message.recordId === 'string' &&
3725
+ 'contextId' in message &&
3726
+ typeof message.contextId === 'string';
3727
+ }
1961
3728
 
1962
- // Subset scope filtering: only push events that match the link's
1963
- // scope prefixes. Events outside the scope are not our responsibility.
1964
- const pushLinkKey = target.linkKey;
1965
- const pushLink = this._activeLinks.get(pushLinkKey);
1966
- if (pushLink && !isEventInScope(subMessage.event.message, pushLink.scope)) {
1967
- return;
1968
- }
3729
+ private static isRecordsDeleteMessage(message: GenericMessage): boolean {
3730
+ return message.descriptor.interface === DwnInterfaceName.Records &&
3731
+ message.descriptor.method === DwnMethodName.Delete;
3732
+ }
1969
3733
 
1970
- // Accumulate the message CID for a debounced push.
1971
- const targetKey = pushLinkKey;
1972
- const cid = await Message.getCid(subMessage.event.message);
1973
- if (cid === undefined || isPushStale()) {
1974
- return;
1975
- }
3734
+ private static isProtocolsConfigureDefinitionMessage(message: GenericMessage): message is ProtocolsConfigureMessage {
3735
+ return message.descriptor.interface === DwnInterfaceName.Protocols &&
3736
+ message.descriptor.method === DwnMethodName.Configure &&
3737
+ message.authorization !== undefined &&
3738
+ SyncEngineLevel.hasProtocolsConfigureDefinition(message.descriptor);
3739
+ }
1976
3740
 
1977
- // Echo-loop suppression: skip CIDs that were recently pulled from this
1978
- // specific remote. A message pulled from Provider A is only suppressed
1979
- // for push to A — it still fans out to Provider B and C.
1980
- if (this.isRecentlyPulled(cid, dwnUrl)) {
1981
- return;
1982
- }
3741
+ private static hasProtocolsConfigureDefinition(
3742
+ descriptor: MaybeProtocolsConfigureDefinitionDescriptor,
3743
+ ): descriptor is ProtocolsConfigureDefinitionDescriptor {
3744
+ return SyncEngineLevel.isProtocolsConfigureDefinition(descriptor.definition);
3745
+ }
1983
3746
 
1984
- const pushRuntime = this.getOrCreatePushRuntime(targetKey, {
1985
- did, dwnUrl, delegateDid, protocol,
3747
+ private static isProtocolsConfigureDefinition(
3748
+ definition: unknown,
3749
+ ): definition is ProtocolsConfigureDefinition {
3750
+ return typeof definition === 'object' &&
3751
+ definition !== null &&
3752
+ 'protocol' in definition &&
3753
+ typeof definition.protocol === 'string';
3754
+ }
3755
+
3756
+ private static filterProtocolConfigClosure(
3757
+ primaryProtocol: string,
3758
+ candidates: AuthenticatedProtocolConfigDependency[],
3759
+ ): MessagesSyncDependencyEntry[] {
3760
+ // Start from the primary record's protocol and walk only protocols named by
3761
+ // accepted, signed config definitions. This keeps composed-protocol support
3762
+ // narrow: the governing config can admit its `uses` targets, but arbitrary
3763
+ // protocol config hints cannot enter the apply set.
3764
+ const candidatesByProtocol = SyncEngineLevel.groupGoverningProtocolConfigCandidatesByProtocol(candidates);
3765
+ const visitedProtocols = new Set<string>();
3766
+ const pendingProtocols = [primaryProtocol];
3767
+ const accepted = new Map<string, MessagesSyncDependencyEntry>();
3768
+
3769
+ for (
3770
+ let protocol = SyncEngineLevel.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols);
3771
+ protocol !== undefined;
3772
+ protocol = SyncEngineLevel.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols)
3773
+ ) {
3774
+ SyncEngineLevel.acceptProtocolConfigCandidates({
3775
+ protocol,
3776
+ candidatesByProtocol,
3777
+ visitedProtocols,
3778
+ pendingProtocols,
3779
+ accepted,
1986
3780
  });
1987
- pushRuntime.entries.push({ cid });
3781
+ }
1988
3782
 
1989
- // Immediate-first: if no push is in flight and no batch timer is
1990
- // pending, push immediately. Otherwise, the pending batch timer
1991
- // or the post-flush drain will pick up the new entry.
1992
- if (!pushRuntime.flushing && !pushRuntime.timer) {
1993
- void this.flushPendingPushesForLink(targetKey);
1994
- }
1995
- };
3783
+ return [...accepted.values()];
3784
+ }
1996
3785
 
1997
- // Subscribe to the local DWN EventLog from "now" — opportunistic push
1998
- // does not replay from a stored cursor. Any writes missed during outages
1999
- // are recovered by the post-repair reconciliation path.
2000
- const response = await this.agent.dwn.processRequest({
2001
- author : did,
2002
- target : did,
2003
- messageType : DwnInterface.MessagesSubscribe,
2004
- granteeDid : delegateDid,
2005
- messageParams : { filters, permissionGrantId },
2006
- subscriptionHandler : subscriptionHandler as any,
2007
- });
3786
+ private static groupGoverningProtocolConfigCandidatesByProtocol(
3787
+ candidates: AuthenticatedProtocolConfigDependency[],
3788
+ ): Map<string, AuthenticatedProtocolConfigDependency> {
3789
+ const candidatesByProtocol = new Map<string, AuthenticatedProtocolConfigDependency>();
3790
+ for (const candidate of candidates) {
3791
+ const protocol = SyncEngineLevel.protocolsConfigureProtocol(candidate.message);
3792
+ const existing = candidatesByProtocol.get(protocol);
3793
+ if (existing !== undefined && SyncEngineLevel.isProtocolConfigCandidateAtLeastAsNew(existing, candidate)) {
3794
+ continue;
3795
+ }
2008
3796
 
2009
- const reply = response.reply as MessagesSubscribeReply;
2010
- if (reply.status.code !== 200 || !reply.subscription) {
2011
- throw new Error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
3797
+ candidatesByProtocol.set(protocol, candidate);
2012
3798
  }
2013
-
2014
- this._localSubscriptions.push({
2015
- linkKey : target.linkKey ?? buildLegacyCursorKey(did, dwnUrl, protocol),
2016
- did,
2017
- dwnUrl,
2018
- delegateDid,
2019
- protocol,
2020
- close : async (): Promise<void> => { await reply.subscription!.close(); },
2021
- });
3799
+ return candidatesByProtocol;
2022
3800
  }
2023
3801
 
2024
- /**
2025
- * Flushes accumulated push CIDs to remote DWNs.
2026
- */
2027
- private async flushPendingPushes(): Promise<void> {
2028
- await Promise.all([...this._pushRuntimes.keys()].map(async (linkKey) => {
2029
- await this.flushPendingPushesForLink(linkKey);
2030
- }));
3802
+ private static isProtocolConfigCandidateAtLeastAsNew(
3803
+ existing: AuthenticatedProtocolConfigDependency,
3804
+ candidate: AuthenticatedProtocolConfigDependency,
3805
+ ): boolean {
3806
+ const existingTimestamp = existing.message.descriptor.messageTimestamp;
3807
+ const candidateTimestamp = candidate.message.descriptor.messageTimestamp;
3808
+ if (existingTimestamp !== candidateTimestamp) {
3809
+ return existingTimestamp > candidateTimestamp;
3810
+ }
3811
+ return lexicographicalCompare(existing.messageCid, candidate.messageCid) >= 0;
2031
3812
  }
2032
3813
 
2033
- private async flushPendingPushesForLink(linkKey: string): Promise<void> {
2034
- // Guard: bail if this link was hot-removed. Without this, a stale
2035
- // debounce timer or retry callback could send pushes after the DID
2036
- // was removed.
2037
- if (!this._activeLinks.has(linkKey)) {
2038
- return;
3814
+ private static takeNextUnvisitedProtocol(
3815
+ pendingProtocols: string[],
3816
+ visitedProtocols: Set<string>,
3817
+ ): string | undefined {
3818
+ while (pendingProtocols.length > 0) {
3819
+ const protocol = pendingProtocols.shift()!;
3820
+ if (visitedProtocols.has(protocol)) {
3821
+ continue;
3822
+ }
3823
+ visitedProtocols.add(protocol);
3824
+ return protocol;
2039
3825
  }
3826
+ return undefined;
3827
+ }
2040
3828
 
2041
- const pushRuntime = this._pushRuntimes.get(linkKey);
2042
- if (!pushRuntime) {
3829
+ private static acceptProtocolConfigCandidates({
3830
+ protocol,
3831
+ candidatesByProtocol,
3832
+ visitedProtocols,
3833
+ pendingProtocols,
3834
+ accepted,
3835
+ }: {
3836
+ protocol: string;
3837
+ candidatesByProtocol: Map<string, AuthenticatedProtocolConfigDependency>;
3838
+ visitedProtocols: Set<string>;
3839
+ pendingProtocols: string[];
3840
+ accepted: Map<string, MessagesSyncDependencyEntry>;
3841
+ }): void {
3842
+ const candidate = candidatesByProtocol.get(protocol);
3843
+ if (candidate === undefined) {
2043
3844
  return;
2044
3845
  }
2045
3846
 
2046
- // Capture the current active link identity so we can detect
2047
- // remove+re-add during the await pushMessages() call.
2048
- const flushLink = this._activeLinks.get(linkKey);
2049
- const isFlushStale = (): boolean =>
2050
- !this._activeLinks.has(linkKey) ||
2051
- (flushLink !== undefined && this._activeLinks.get(linkKey) !== flushLink);
2052
-
2053
- const { did, dwnUrl, delegateDid, protocol, entries: pushEntries, retryCount } = pushRuntime;
2054
- pushRuntime.entries = [];
3847
+ accepted.set(candidate.messageCid, candidate);
3848
+ SyncEngineLevel.queueUnvisitedProtocols(
3849
+ SyncEngineLevel.protocolsConfigureUses(candidate.message),
3850
+ visitedProtocols,
3851
+ pendingProtocols,
3852
+ );
3853
+ }
2055
3854
 
2056
- if (pushEntries.length === 0) {
2057
- if (!pushRuntime.timer && !pushRuntime.flushing && retryCount === 0) {
2058
- this._pushRuntimes.delete(linkKey);
3855
+ private static queueUnvisitedProtocols(
3856
+ protocols: string[],
3857
+ visitedProtocols: Set<string>,
3858
+ pendingProtocols: string[],
3859
+ ): void {
3860
+ for (const protocol of protocols) {
3861
+ if (!visitedProtocols.has(protocol)) {
3862
+ pendingProtocols.push(protocol);
2059
3863
  }
2060
- return;
2061
3864
  }
3865
+ }
2062
3866
 
2063
- const cids = pushEntries.map((entry) => entry.cid);
2064
- pushRuntime.flushing = true;
3867
+ private static protocolSetScopeForProjectedDependencies(
3868
+ dependencies: MessagesSyncDependencyEntry[],
3869
+ ): Extract<SyncScope, { kind: 'protocolSet' }> {
3870
+ // Verification above is the security boundary. This protocolSet scope only
3871
+ // routes already-verified config dependencies through the existing
3872
+ // pull/apply path, which expects every prefetched message to be accepted by
3873
+ // the supplied sync scope before it reaches processRawMessage().
3874
+ const protocols = SyncEngineLevel.dedupeStrings(
3875
+ dependencies
3876
+ .flatMap(dependency => SyncEngineLevel.projectedDependencyProtocols(dependency))
3877
+ ).sort(lexicographicalCompare);
3878
+ if (protocols.length === 0) {
3879
+ throw new Error('SyncEngineLevel: projected dependency hints contained no protocols.');
3880
+ }
2065
3881
 
2066
- try {
2067
- const result = await pushMessages({
2068
- did, dwnUrl, delegateDid, protocol,
2069
- messageCids : cids,
2070
- agent : this.agent,
2071
- permissionsApi : this._permissionsApi,
2072
- });
3882
+ return {
3883
+ kind : 'protocolSet',
3884
+ protocols : protocols as NonEmptyStringArray,
3885
+ };
3886
+ }
2073
3887
 
2074
- // If the link was replaced during pushMessages, abandon all
2075
- // post-push state mutations — the replacement session owns this key.
2076
- if (isFlushStale()) { return; }
3888
+ private static projectedDependencyProtocols(dependency: MessagesSyncDependencyEntry): string[] {
3889
+ if (dependency.message === undefined) {
3890
+ return [];
3891
+ }
2077
3892
 
2078
- // Auto-clear dead letters for CIDs that succeeded — a previously
2079
- // failed message may have been repaired by reconciliation.
2080
- for (const cid of result.succeeded) {
2081
- this.clearFailedMessage(cid, dwnUrl).catch(() => { /* teardown race */ });
2082
- }
3893
+ const protocol = dependency.dependencyClass === 'protocolsConfigure'
3894
+ ? SyncEngineLevel.protocolsConfigureProtocolFromGenericMessage(dependency.message)
3895
+ : SyncEngineLevel.recordsWriteProtocol(dependency.message);
3896
+ return protocol === undefined ? [] : [protocol];
3897
+ }
2083
3898
 
2084
- // Record permanently failed messages in the dead letter store.
2085
- for (const entry of result.permanentlyFailed) {
2086
- await this.recordDeadLetter({
2087
- messageCid : entry.cid,
2088
- tenantDid : did,
2089
- remoteEndpoint : dwnUrl,
2090
- protocol,
2091
- category : 'push-permanent',
2092
- errorCode : String(entry.statusCode ?? ''),
2093
- errorDetail : entry.detail ?? 'permanent push failure',
2094
- });
2095
- }
3899
+ private static hasDependencyPayloadBytes(dependency: MessagesSyncDependencyEntry): boolean {
3900
+ if (dependency.encodedData !== undefined) {
3901
+ return true;
3902
+ }
2096
3903
 
2097
- if (result.failed.length > 0) {
2098
- if (isFlushStale()) { return; }
2099
- const failedSet = new Set(result.failed);
2100
- const failedEntries = pushEntries.filter((entry) => failedSet.has(entry.cid));
2101
- this.requeueOrReconcile(linkKey, {
2102
- did, dwnUrl, delegateDid, protocol,
2103
- entries : failedEntries,
2104
- retryCount : retryCount + 1,
2105
- });
2106
- } else {
2107
- // Successful push — reset retry count so subsequent unrelated
2108
- // batches on this link start with a fresh budget.
2109
- pushRuntime.retryCount = 0;
2110
- if (!pushRuntime.timer && pushRuntime.entries.length === 0) {
2111
- this._pushRuntimes.delete(linkKey);
2112
- }
2113
- }
2114
- } catch (error: any) {
2115
- if (isFlushStale()) { return; }
2116
- console.error(`SyncEngineLevel: Push batch failed for ${did} -> ${dwnUrl}`, error);
2117
- this.requeueOrReconcile(linkKey, {
2118
- did, dwnUrl, delegateDid, protocol,
2119
- entries : pushEntries,
2120
- retryCount : retryCount + 1,
2121
- });
2122
- } finally {
2123
- pushRuntime.flushing = false;
3904
+ const message = dependency.message as Record<string, unknown> | undefined;
3905
+ return message !== undefined && 'encodedData' in message;
3906
+ }
2124
3907
 
2125
- // If new entries accumulated while this push was in flight, schedule
2126
- // a short drain to flush them. This gives a brief batching window
2127
- // for burst writes while keeping single-write latency low.
2128
- const rt = this._pushRuntimes.get(linkKey);
2129
- if (rt && rt.entries.length > 0 && !rt.timer) {
2130
- rt.timer = setTimeout((): void => {
2131
- rt.timer = undefined;
2132
- void this.flushPendingPushesForLink(linkKey);
2133
- }, PUSH_DEBOUNCE_MS);
2134
- }
3908
+ private static dedupeDependencyEntries(
3909
+ dependencies: Iterable<MessagesSyncDependencyEntry>,
3910
+ ): MessagesSyncDependencyEntry[] {
3911
+ const deduped = new Map<string, MessagesSyncDependencyEntry>();
3912
+ for (const dependency of dependencies) {
3913
+ deduped.set(dependency.messageCid, dependency);
2135
3914
  }
3915
+ return [...deduped.values()];
2136
3916
  }
2137
3917
 
2138
- /** Push retry backoff schedule: immediate, 250ms, 1s, 2s, then give up. */
2139
- private static readonly PUSH_RETRY_BACKOFF_MS = [0, 250, 1000, 2000];
2140
-
2141
- /**
2142
- * Re-queues a failed push batch for retry, or marks the link
2143
- * `needsReconcile` if retries are exhausted. Bounded to prevent
2144
- * infinite retry loops.
2145
- */
2146
- private requeueOrReconcile(targetKey: string, pending: {
2147
- did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
2148
- entries: { cid: string }[];
2149
- retryCount: number;
2150
- }): void {
2151
- const maxRetries = SyncEngineLevel.PUSH_RETRY_BACKOFF_MS.length;
2152
- const pushRuntime = this.getOrCreatePushRuntime(targetKey, pending);
3918
+ private async pushLocalDiffEntries(
3919
+ target: ProjectionReconcileTarget,
3920
+ onlyLocal: string[],
3921
+ permissionGrantIds: NonEmptyStringArray | undefined,
3922
+ shouldContinue?: () => boolean,
3923
+ ): Promise<boolean> {
3924
+ await this.pushMessages({
3925
+ did : target.did,
3926
+ dwnUrl : target.dwnUrl,
3927
+ delegateDid : target.delegateDid,
3928
+ permissionGrantIds,
3929
+ messageCids : SyncEngineLevel.dedupeStrings(onlyLocal),
3930
+ });
3931
+ return shouldContinue?.() === false;
3932
+ }
2153
3933
 
2154
- if (pending.retryCount >= maxRetries) {
2155
- // Retry budget exhausted — record each CID as a dead letter and mark
2156
- // the link dirty for reconciliation.
2157
- for (const entry of pending.entries) {
2158
- void this.recordDeadLetter({
2159
- messageCid : entry.cid,
2160
- tenantDid : pending.did,
2161
- remoteEndpoint : pending.dwnUrl,
2162
- protocol : pending.protocol,
2163
- category : 'push-exhausted',
2164
- errorDetail : `push retries exhausted after ${maxRetries} attempts`,
2165
- });
2166
- }
2167
- if (pushRuntime.timer) {
2168
- clearTimeout(pushRuntime.timer);
2169
- }
2170
- this._pushRuntimes.delete(targetKey);
2171
- const link = this._activeLinks.get(targetKey);
2172
- if (link && !link.needsReconcile) {
2173
- link.needsReconcile = true;
2174
- void this.ledger.saveLink(link).then(() => {
2175
- this.emitEvent({ type: 'reconcile:needed', tenantDid: pending.did, remoteEndpoint: pending.dwnUrl, protocol: pending.protocol, reason: 'push-retry-exhausted' });
2176
- this.scheduleReconcile(targetKey);
2177
- });
3934
+ private async verifyProtocolSetConvergence(
3935
+ target: ProjectionReconcileTarget,
3936
+ changedProtocols: string[],
3937
+ permissionGrantIds: NonEmptyStringArray | undefined,
3938
+ shouldContinue?: () => boolean,
3939
+ ): Promise<ProjectionReconcileResult> {
3940
+ for (const protocol of changedProtocols) {
3941
+ const roots = await this.getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue);
3942
+ if (!roots) { return { aborted: true }; }
3943
+
3944
+ if (roots.localRoot !== roots.remoteRoot) {
3945
+ return { converged: false };
2178
3946
  }
2179
- return;
2180
3947
  }
2181
3948
 
2182
- pushRuntime.entries.push(...pending.entries);
2183
- pushRuntime.retryCount = pending.retryCount;
2184
- const delayMs = SyncEngineLevel.PUSH_RETRY_BACKOFF_MS[pending.retryCount] ?? 2000;
2185
- if (pushRuntime.timer) {
2186
- clearTimeout(pushRuntime.timer);
3949
+ return { converged: true };
3950
+ }
3951
+
3952
+ private static dedupeRemoteEntries(entries: MessagesSyncDiffEntry[]): MessagesSyncDiffEntry[] {
3953
+ const seen = new Set<string>();
3954
+ const unique: MessagesSyncDiffEntry[] = [];
3955
+ for (const entry of entries) {
3956
+ if (seen.has(entry.messageCid)) {
3957
+ continue;
3958
+ }
3959
+ seen.add(entry.messageCid);
3960
+ unique.push(entry);
2187
3961
  }
2188
- pushRuntime.timer = setTimeout((): void => {
2189
- pushRuntime.timer = undefined;
2190
- void this.flushPendingPushesForLink(targetKey);
2191
- }, delayMs);
3962
+ return unique;
2192
3963
  }
2193
3964
 
2194
- private createLinkReconciler(shouldContinue?: () => boolean): SyncLinkReconciler {
2195
- return new SyncLinkReconciler({
2196
- getLocalRoot : async (did, delegateDid, protocol) => this.getLocalRoot(did, delegateDid, protocol),
2197
- getRemoteRoot : async (did, dwnUrl, delegateDid, protocol) => this.getRemoteRoot(did, dwnUrl, delegateDid, protocol),
2198
- diffWithRemote : async (target) => this.diffWithRemote(target),
2199
- pullMessages : async (params) => this.pullMessages(params),
2200
- pushMessages : async (params) => this.pushMessages(params),
2201
- shouldContinue,
2202
- });
3965
+ private static dedupeStrings(values: string[]): string[] {
3966
+ return [...new Set(values)];
3967
+ }
3968
+
3969
+ private async clearRootConvergenceDeadLettersForScope(
3970
+ tenantDid: string,
3971
+ remoteEndpoint: string,
3972
+ scope: SyncScope,
3973
+ ): Promise<void> {
3974
+ if (scope.kind === 'recordsProjection' || (scope.kind === 'protocolSet' && scope.protocols.length > 1)) {
3975
+ // Batched multi-protocol and projected pulls pass the full scope to
3976
+ // pullMessages, so pull dead letters can be recorded without a single
3977
+ // protocol bucket.
3978
+ await this.clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint);
3979
+ }
3980
+
3981
+ for (const protocol of this.getReconcileProtocols(scope)) {
3982
+ await this.clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint, protocol);
3983
+ }
2203
3984
  }
2204
3985
 
2205
3986
  // ---------------------------------------------------------------------------
@@ -2277,21 +4058,30 @@ export class SyncEngineLevel implements SyncEngine {
2277
4058
  // closure's captured `link` reference may no longer be the active
2278
4059
  // link object. Bail before mutating the replacement's state.
2279
4060
  const isStaleLink = (): boolean => this._activeLinks.get(linkKey) !== link;
4061
+ const shouldContinue = (): boolean =>
4062
+ this._engineGeneration === generation &&
4063
+ !isStaleLink() &&
4064
+ link.status === 'live';
2280
4065
 
2281
- const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, protocol } = link;
4066
+ const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, scope, authorization } = link;
4067
+ const eventScope = syncEventScope(scope);
2282
4068
 
2283
4069
  try {
2284
- const reconcileOutcome = await this.createLinkReconciler(
2285
- () => this._engineGeneration === generation && !isStaleLink()
2286
- ).reconcile({ did, dwnUrl, delegateDid, protocol }, { verifyConvergence: true });
4070
+ const reconcileOutcome = await this.reconcileProjectionTarget({
4071
+ did,
4072
+ dwnUrl,
4073
+ delegateDid,
4074
+ scope,
4075
+ authorization,
4076
+ }, { verifyConvergence: true }, shouldContinue);
2287
4077
  if (reconcileOutcome.aborted || isStaleLink()) { return; }
2288
4078
 
2289
4079
  if (reconcileOutcome.converged) {
2290
4080
  await this.ledger.clearNeedsReconcile(link);
2291
- // SMT roots match this link is converged. Clear dead letters
2292
- // scoped to this specific link (tenantDid, remoteEndpoint, protocol).
2293
- void this.clearDeadLettersForLink(did, dwnUrl, protocol);
2294
- this.emitEvent({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl, protocol });
4081
+ // SMT roots match, so transport/apply failures for this link may no
4082
+ // longer be current. Closure failures are not cleared by root equality.
4083
+ await this.clearRootConvergenceDeadLettersForScope(did, dwnUrl, scope);
4084
+ this.emitEvent({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl, ...eventScope });
2295
4085
  } else {
2296
4086
  // Roots still differ — retry after a delay. This can happen when
2297
4087
  // pushMessages() had permanent failures, pullMessages() partially
@@ -2301,136 +4091,46 @@ export class SyncEngineLevel implements SyncEngine {
2301
4091
  } catch (error: any) {
2302
4092
  if (isStaleLink()) { return; }
2303
4093
  console.error(`SyncEngineLevel: Reconciliation failed for ${did} -> ${dwnUrl}`, error);
2304
- // Schedule retry with longer delay.
2305
- this.scheduleReconcile(linkKey, 5000);
2306
- }
2307
- }
2308
-
2309
- private getOrCreatePushRuntime(linkKey: string, params: {
2310
- did: string;
2311
- dwnUrl: string;
2312
- delegateDid?: string;
2313
- protocol?: string;
2314
- }): PushRuntimeState {
2315
- let pushRuntime = this._pushRuntimes.get(linkKey);
2316
- if (!pushRuntime) {
2317
- pushRuntime = {
2318
- ...params,
2319
- entries : [],
2320
- retryCount : 0,
2321
- };
2322
- this._pushRuntimes.set(linkKey, pushRuntime);
2323
- }
2324
-
2325
- return pushRuntime;
2326
- }
2327
-
2328
- // ---------------------------------------------------------------------------
2329
- // Cursor persistence
2330
- // ---------------------------------------------------------------------------
2331
-
2332
- /**
2333
- * Build the runtime key for a replication link.
2334
- *
2335
- * Live-mode subscription methods (`openLivePullSubscription`,
2336
- * `openLocalPushSubscription`) receive `linkKey` directly and never
2337
- * call this. The remaining callers are poll-mode `sync()` and the
2338
- * live-mode startup/error paths that already have `link.scopeId`.
2339
- *
2340
- * The `undefined` fallback (which produces a legacy cursor key) exists
2341
- * only for the no-protocol full-tenant targets in poll mode.
2342
- */
2343
- private buildLinkKey(did: string, dwnUrl: string, scopeIdOrProtocol?: string): string {
2344
- return scopeIdOrProtocol ? buildLinkId(did, dwnUrl, scopeIdOrProtocol) : buildLegacyCursorKey(did, dwnUrl);
2345
- }
2346
-
2347
- /**
2348
- * @deprecated Used by poll-mode sync and one-time migration only. Live mode
2349
- * uses ReplicationLedger checkpoints. Handles migration from old string cursors:
2350
- * if the stored value is a bare string (pre-ProgressToken format), it is treated
2351
- * as absent — the sync engine will do a full SMT reconciliation on first startup
2352
- * after upgrade, which is correct and safe.
2353
- */
2354
- private async getCursor(key: string): Promise<ProgressToken | undefined> {
2355
- const cursors = this._db.sublevel('syncCursors');
2356
- try {
2357
- const raw = await cursors.get(key);
2358
- try {
2359
- const parsed = JSON.parse(raw);
2360
- if (parsed && typeof parsed === 'object' &&
2361
- typeof parsed.streamId === 'string' && parsed.streamId.length > 0 &&
2362
- typeof parsed.epoch === 'string' && parsed.epoch.length > 0 &&
2363
- typeof parsed.position === 'string' && parsed.position.length > 0 &&
2364
- typeof parsed.messageCid === 'string' && parsed.messageCid.length > 0) {
2365
- return parsed as ProgressToken;
2366
- }
2367
- } catch {
2368
- // Not valid JSON (old string cursor) — fall through to delete.
2369
- }
2370
- // Entry exists but is unparseable or has invalid/empty fields. Delete it
2371
- // so subsequent startups don't re-check it on every launch.
2372
- await this.deleteLegacyCursor(key);
2373
- return undefined;
2374
- } catch (error) {
2375
- const e = error as { code: string };
2376
- if (e.code === 'LEVEL_NOT_FOUND') {
2377
- return undefined;
2378
- }
2379
- throw error;
4094
+ // Schedule retry with longer delay.
4095
+ this.scheduleReconcile(linkKey, 5000);
2380
4096
  }
2381
4097
  }
2382
4098
 
2383
-
2384
- /**
2385
- * Delete a legacy cursor from the old syncCursors sublevel.
2386
- * Called as part of one-time migration to ReplicationLedger.
2387
- */
2388
- private async deleteLegacyCursor(key: string): Promise<void> {
2389
- const cursors = this._db.sublevel('syncCursors');
2390
- try {
2391
- await cursors.del(key);
2392
- } catch {
2393
- // Best-effort — ignore LEVEL_NOT_FOUND and transient I/O errors alike.
2394
- // A failed delete leaves the bad entry for one more re-check on the
2395
- // next startup, which is harmless.
4099
+ private getOrCreatePushRuntime(linkKey: string, params: {
4100
+ did: string;
4101
+ dwnUrl: string;
4102
+ delegateDid?: string;
4103
+ protocol?: string;
4104
+ permissionGrantIds?: NonEmptyStringArray;
4105
+ }): PushRuntimeState {
4106
+ let pushRuntime = this._pushRuntimes.get(linkKey);
4107
+ if (!pushRuntime) {
4108
+ pushRuntime = {
4109
+ ...params,
4110
+ entries : [],
4111
+ retryCount : 0,
4112
+ };
4113
+ this._pushRuntimes.set(linkKey, pushRuntime);
2396
4114
  }
4115
+
4116
+ return pushRuntime;
2397
4117
  }
2398
4118
 
2399
4119
  // ---------------------------------------------------------------------------
2400
- // Utility helpers
4120
+ // Cursor persistence
2401
4121
  // ---------------------------------------------------------------------------
2402
4122
 
2403
4123
  /**
2404
- * Extracts a ReadableStream from a MessageEvent if it contains a
2405
- * RecordsWrite with data — either as an inline `encodedData` field
2406
- * (for records <= 30 KB) or as a pre-existing data stream.
4124
+ * Build the runtime key for a replication link.
4125
+ *
4126
+ * Live-mode subscription methods (`openLivePullSubscription`,
4127
+ * `openLocalPushSubscription`) receive `linkKey` directly and never
4128
+ * call this. The remaining callers are poll-mode `sync()` and the
4129
+ * live-mode startup/error paths that already have a projection ID and
4130
+ * authorization epoch.
2407
4131
  */
2408
- private extractDataStream(event: MessageEvent): ReadableStream<Uint8Array> | undefined {
2409
- if (!isRecordsWrite(event)) {
2410
- return undefined;
2411
- }
2412
-
2413
- // Check for inline base64url-encoded data (small records from EventLog).
2414
- // Delete the transport-level field so the DWN schema validator does not
2415
- // reject the message for having unevaluated properties.
2416
- const encodedData = (event.message as any).encodedData as string | undefined;
2417
- if (encodedData) {
2418
- delete (event.message as any).encodedData;
2419
- const bytes = Encoder.base64UrlToBytes(encodedData);
2420
- return new ReadableStream<Uint8Array>({
2421
- start(controller): void {
2422
- controller.enqueue(bytes);
2423
- controller.close();
2424
- }
2425
- });
2426
- }
2427
-
2428
- // Check for a pre-existing data stream (e.g. from a direct message read).
2429
- if ((event as any).data) {
2430
- return (event as any).data;
2431
- }
2432
-
2433
- return undefined;
4132
+ private buildLinkKey(did: string, dwnUrl: string, projectionId: string, authorizationEpoch: string): string {
4133
+ return buildLinkId(did, dwnUrl, projectionId, authorizationEpoch);
2434
4134
  }
2435
4135
 
2436
4136
  // ---------------------------------------------------------------------------
@@ -2492,7 +4192,12 @@ export class SyncEngineLevel implements SyncEngine {
2492
4192
  *
2493
4193
  * Returns a hex-encoded root hash string.
2494
4194
  */
2495
- private async getLocalRoot(did: string, delegateDid?: string, protocol?: string): Promise<string> {
4195
+ private async getLocalRoot(
4196
+ did: string,
4197
+ delegateDid?: string,
4198
+ protocol?: string,
4199
+ permissionGrantIds?: string[],
4200
+ ): Promise<string> {
2496
4201
  const si = this.stateIndex;
2497
4202
  if (si) {
2498
4203
  const rootHash = protocol === undefined
@@ -2502,16 +4207,15 @@ export class SyncEngineLevel implements SyncEngine {
2502
4207
  }
2503
4208
 
2504
4209
  // Remote mode fallback: go through processRequest → RPC.
2505
- const permissionGrantId = await this.getSyncPermissionGrantId(did, delegateDid, protocol);
2506
4210
  const response = await this.agent.dwn.processRequest({
2507
4211
  author : did,
2508
4212
  target : did,
2509
4213
  messageType : DwnInterface.MessagesSync,
2510
4214
  granteeDid : delegateDid,
2511
4215
  messageParams : {
2512
- action: 'root',
4216
+ action : 'root',
2513
4217
  protocol,
2514
- permissionGrantId
4218
+ permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
2515
4219
  }
2516
4220
  });
2517
4221
  const reply = response.reply as MessagesSyncReply;
@@ -2522,9 +4226,13 @@ export class SyncEngineLevel implements SyncEngine {
2522
4226
  * Get the SMT root hash from a remote DWN via a MessagesSync 'root' action.
2523
4227
  * Returns a hex-encoded root hash string.
2524
4228
  */
2525
- private async getRemoteRoot(did: string, dwnUrl: string, delegateDid?: string, protocol?: string): Promise<string> {
2526
- const permissionGrantId = await this.getSyncPermissionGrantId(did, delegateDid, protocol);
2527
-
4229
+ private async getRemoteRoot(
4230
+ did: string,
4231
+ dwnUrl: string,
4232
+ delegateDid?: string,
4233
+ protocol?: string,
4234
+ permissionGrantIds?: string[],
4235
+ ): Promise<string> {
2528
4236
  const syncMessage = await this.agent.dwn.processRequest({
2529
4237
  store : false,
2530
4238
  author : did,
@@ -2532,9 +4240,71 @@ export class SyncEngineLevel implements SyncEngine {
2532
4240
  messageType : DwnInterface.MessagesSync,
2533
4241
  granteeDid : delegateDid,
2534
4242
  messageParams : {
2535
- action: 'root',
4243
+ action : 'root',
2536
4244
  protocol,
2537
- permissionGrantId
4245
+ permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
4246
+ }
4247
+ });
4248
+
4249
+ const reply = await this.agent.rpc.sendDwnRequest({
4250
+ dwnUrl,
4251
+ targetDid : did,
4252
+ message : syncMessage.message,
4253
+ }) as MessagesSyncReply;
4254
+
4255
+ return reply.root ?? '';
4256
+ }
4257
+
4258
+ private async getLocalProjectedRoot(
4259
+ did: string,
4260
+ delegateDid: string | undefined,
4261
+ scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
4262
+ permissionGrantIds?: string[],
4263
+ ): Promise<string> {
4264
+ if (this.stateIndex) {
4265
+ // Local projected roots use the already-derived scope directly. The
4266
+ // remote root/diff request still re-authorizes the invoked grant set.
4267
+ return RecordsProjection.getRootHex({
4268
+ tenant : did,
4269
+ messageStore : this.agent.dwn.node.storage.messageStore,
4270
+ scopes,
4271
+ });
4272
+ }
4273
+
4274
+ const response = await this.agent.dwn.processRequest({
4275
+ author : did,
4276
+ target : did,
4277
+ messageType : DwnInterface.MessagesSync,
4278
+ granteeDid : delegateDid,
4279
+ messageParams : {
4280
+ action : 'root',
4281
+ projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
4282
+ projectionScopes : [...scopes],
4283
+ permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
4284
+ }
4285
+ });
4286
+ const reply = response.reply as MessagesSyncReply;
4287
+ return reply.root ?? '';
4288
+ }
4289
+
4290
+ private async getRemoteProjectedRoot(
4291
+ did: string,
4292
+ dwnUrl: string,
4293
+ delegateDid: string | undefined,
4294
+ scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
4295
+ permissionGrantIds?: string[],
4296
+ ): Promise<string> {
4297
+ const syncMessage = await this.agent.dwn.processRequest({
4298
+ store : false,
4299
+ author : did,
4300
+ target : did,
4301
+ messageType : DwnInterface.MessagesSync,
4302
+ granteeDid : delegateDid,
4303
+ messageParams : {
4304
+ action : 'root',
4305
+ projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
4306
+ projectionScopes : [...scopes],
4307
+ permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
2538
4308
  }
2539
4309
  });
2540
4310
 
@@ -2565,55 +4335,102 @@ export class SyncEngineLevel implements SyncEngine {
2565
4335
  *
2566
4336
  * This replaces `walkTreeDiff()` which required one HTTP call per tree node.
2567
4337
  */
2568
- private async diffWithRemote({ did, dwnUrl, delegateDid, protocol }: {
4338
+ private async diffWithRemote({ did, dwnUrl, delegateDid, protocol, permissionGrantIds }: {
2569
4339
  did: string;
2570
4340
  dwnUrl: string;
2571
4341
  delegateDid?: string;
2572
4342
  protocol?: string;
2573
- }): Promise<{ onlyRemote: MessagesSyncDiffEntry[]; onlyLocal: string[] }> {
4343
+ permissionGrantIds?: string[];
4344
+ }): Promise<ProjectionDiffResult> {
2574
4345
  // Step 1: Collect local subtree hashes at BATCHED_DIFF_DEPTH directly from StateIndex.
2575
- const localHashes = await this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH);
4346
+ const localHashes = await this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH, permissionGrantIds);
2576
4347
 
2577
4348
  // Step 2: Send a single 'diff' request to the remote with our hashes.
2578
- const permissionGrantId = await this.getSyncPermissionGrantId(did, delegateDid, protocol);
4349
+ const messageParams: DwnMessageParams[DwnInterface.MessagesSync] = {
4350
+ action : 'diff',
4351
+ protocol,
4352
+ hashes : localHashes,
4353
+ depth : BATCHED_DIFF_DEPTH,
4354
+ permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
4355
+ };
4356
+
4357
+ // Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
4358
+ // Reuse the same grant set from step 2.
4359
+ return this.diffRemoteMessages(
4360
+ { did, dwnUrl, delegateDid },
4361
+ messageParams,
4362
+ prefix => this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantIds),
4363
+ 'diff',
4364
+ );
4365
+ }
4366
+
4367
+ private async diffProjectedWithRemote({ did, dwnUrl, delegateDid, scopes, permissionGrantIds }: {
4368
+ did: string;
4369
+ dwnUrl: string;
4370
+ delegateDid?: string;
4371
+ scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]];
4372
+ permissionGrantIds?: string[];
4373
+ }): Promise<ProjectionDiffResult> {
4374
+ const localHashes = await this.collectLocalProjectedSubtreeHashes(
4375
+ did,
4376
+ scopes,
4377
+ BATCHED_DIFF_DEPTH,
4378
+ delegateDid,
4379
+ permissionGrantIds,
4380
+ );
4381
+
4382
+ const messageParams: DwnMessageParams[DwnInterface.MessagesSync] = {
4383
+ action : 'diff',
4384
+ projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
4385
+ projectionScopes : [...scopes],
4386
+ hashes : localHashes,
4387
+ depth : BATCHED_DIFF_DEPTH,
4388
+ permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds),
4389
+ };
4390
+
4391
+ return this.diffRemoteMessages(
4392
+ { did, dwnUrl, delegateDid },
4393
+ messageParams,
4394
+ prefix => this.getLocalProjectedLeaves(did, prefix, delegateDid, scopes, permissionGrantIds),
4395
+ 'projected diff',
4396
+ );
4397
+ }
2579
4398
 
4399
+ private async diffRemoteMessages(
4400
+ target: Pick<ProjectionReconcileTarget, 'did' | 'dwnUrl' | 'delegateDid'>,
4401
+ messageParams: DwnMessageParams[DwnInterface.MessagesSync],
4402
+ getLocalLeavesForPrefix: (prefix: string) => Promise<string[]>,
4403
+ operationName: string,
4404
+ ): Promise<ProjectionDiffResult> {
2580
4405
  const syncMessage = await this.agent.dwn.processRequest({
2581
- store : false,
2582
- author : did,
2583
- target : did,
2584
- messageType : DwnInterface.MessagesSync,
2585
- granteeDid : delegateDid,
2586
- messageParams : {
2587
- action : 'diff',
2588
- protocol,
2589
- hashes : localHashes,
2590
- depth : BATCHED_DIFF_DEPTH,
2591
- permissionGrantId,
2592
- }
4406
+ store : false,
4407
+ author : target.did,
4408
+ target : target.did,
4409
+ messageType : DwnInterface.MessagesSync,
4410
+ granteeDid : target.delegateDid,
4411
+ messageParams,
2593
4412
  });
2594
4413
 
2595
4414
  const reply = await this.agent.rpc.sendDwnRequest({
2596
- dwnUrl,
2597
- targetDid : did,
4415
+ dwnUrl : target.dwnUrl,
4416
+ targetDid : target.did,
2598
4417
  message : syncMessage.message,
2599
4418
  }) as MessagesSyncReply;
2600
4419
 
2601
4420
  if (reply.status.code !== 200) {
2602
- throw new Error(`SyncEngineLevel: diff failed with ${reply.status.code}: ${reply.status.detail}`);
4421
+ throw new Error(`SyncEngineLevel: ${operationName} failed with ${reply.status.code}: ${reply.status.detail}`);
2603
4422
  }
2604
4423
 
2605
- // Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
2606
- // Reuse the same grant ID from step 2 (avoids redundant lookup).
2607
- const permissionGrantIdForLeaves = permissionGrantId;
2608
4424
  const onlyLocalCids: string[] = [];
2609
4425
  for (const prefix of reply.onlyLocal ?? []) {
2610
- const leaves = await this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantIdForLeaves);
4426
+ const leaves = await getLocalLeavesForPrefix(prefix);
2611
4427
  onlyLocalCids.push(...leaves);
2612
4428
  }
2613
4429
 
2614
4430
  return {
2615
- onlyRemote : reply.onlyRemote ?? [],
2616
- onlyLocal : onlyLocalCids,
4431
+ dependencies : reply.dependencies ?? [],
4432
+ onlyRemote : reply.onlyRemote ?? [],
4433
+ onlyLocal : onlyLocalCids,
2617
4434
  };
2618
4435
  }
2619
4436
 
@@ -2629,6 +4446,7 @@ export class SyncEngineLevel implements SyncEngine {
2629
4446
  did: string,
2630
4447
  protocol: string | undefined,
2631
4448
  depth: number,
4449
+ permissionGrantIds?: string[],
2632
4450
  ): Promise<Record<string, string>> {
2633
4451
  const result: Record<string, string> = {};
2634
4452
  const defaultHash = await this.getDefaultHashHex(depth);
@@ -2646,7 +4464,7 @@ export class SyncEngineLevel implements SyncEngine {
2646
4464
  hexHash = hashToHex(hash);
2647
4465
  } else {
2648
4466
  // Remote mode fallback.
2649
- hexHash = await this.getLocalSubtreeHash(did, prefix, undefined, protocol);
4467
+ hexHash = await this.getLocalSubtreeHash(did, prefix, undefined, protocol, permissionGrantIds);
2650
4468
  }
2651
4469
 
2652
4470
  if (hexHash === defaultHash) {
@@ -2670,6 +4488,52 @@ export class SyncEngineLevel implements SyncEngine {
2670
4488
  return result;
2671
4489
  }
2672
4490
 
4491
+ private async collectLocalProjectedSubtreeHashes(
4492
+ did: string,
4493
+ scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
4494
+ depth: number,
4495
+ delegateDid?: string,
4496
+ permissionGrantIds?: string[],
4497
+ ): Promise<Record<string, string>> {
4498
+ const result: Record<string, string> = {};
4499
+ const snapshot = this.stateIndex
4500
+ ? await RecordsProjection.createSnapshot({
4501
+ tenant : did,
4502
+ messageStore : this.agent.dwn.node.storage.messageStore,
4503
+ scopes,
4504
+ })
4505
+ : undefined;
4506
+
4507
+ try {
4508
+ const walk = async (prefix: string, currentDepth: number): Promise<void> => {
4509
+ const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
4510
+ const hexHash = snapshot
4511
+ ? hashToHex(await snapshot.getSubtreeHash(bitPath))
4512
+ : await this.getLocalProjectedSubtreeHash(did, prefix, delegateDid, scopes, permissionGrantIds);
4513
+ const defaultHash = await this.getDefaultHashHex(currentDepth);
4514
+
4515
+ if (hexHash === defaultHash) {
4516
+ return;
4517
+ }
4518
+
4519
+ if (currentDepth >= depth) {
4520
+ result[prefix] = hexHash;
4521
+ return;
4522
+ }
4523
+
4524
+ await Promise.all([
4525
+ walk(prefix + '0', currentDepth + 1),
4526
+ walk(prefix + '1', currentDepth + 1),
4527
+ ]);
4528
+ };
4529
+
4530
+ await walk('', 0);
4531
+ return result;
4532
+ } finally {
4533
+ await snapshot?.close();
4534
+ }
4535
+ }
4536
+
2673
4537
  /**
2674
4538
  * Get the subtree hash at a given bit prefix from the local DWN.
2675
4539
  *
@@ -2677,7 +4541,7 @@ export class SyncEngineLevel implements SyncEngine {
2677
4541
  * In remote mode: constructs a signed MessagesSync message and routes through RPC.
2678
4542
  */
2679
4543
  private async getLocalSubtreeHash(
2680
- did: string, prefix: string, delegateDid?: string, protocol?: string, permissionGrantId?: string
4544
+ did: string, prefix: string, delegateDid?: string, protocol?: string, permissionGrantIds?: string[]
2681
4545
  ): Promise<string> {
2682
4546
  const si = this.stateIndex;
2683
4547
  if (si) {
@@ -2695,10 +4559,34 @@ export class SyncEngineLevel implements SyncEngine {
2695
4559
  messageType : DwnInterface.MessagesSync,
2696
4560
  granteeDid : delegateDid,
2697
4561
  messageParams : {
2698
- action: 'subtree',
4562
+ action : 'subtree',
2699
4563
  prefix,
2700
4564
  protocol,
2701
- permissionGrantId
4565
+ permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
4566
+ }
4567
+ });
4568
+ const reply = response.reply as MessagesSyncReply;
4569
+ return reply.hash ?? '';
4570
+ }
4571
+
4572
+ private async getLocalProjectedSubtreeHash(
4573
+ did: string,
4574
+ prefix: string,
4575
+ delegateDid: string | undefined,
4576
+ scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
4577
+ permissionGrantIds?: string[],
4578
+ ): Promise<string> {
4579
+ const response = await this.agent.dwn.processRequest({
4580
+ author : did,
4581
+ target : did,
4582
+ messageType : DwnInterface.MessagesSync,
4583
+ granteeDid : delegateDid,
4584
+ messageParams : {
4585
+ action : 'subtree',
4586
+ prefix,
4587
+ projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
4588
+ projectionScopes : [...scopes],
4589
+ permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
2702
4590
  }
2703
4591
  });
2704
4592
  const reply = response.reply as MessagesSyncReply;
@@ -2712,7 +4600,7 @@ export class SyncEngineLevel implements SyncEngine {
2712
4600
  * In remote mode: constructs a signed MessagesSync message and routes through RPC.
2713
4601
  */
2714
4602
  private async getLocalLeaves(
2715
- did: string, prefix: string, delegateDid?: string, protocol?: string, permissionGrantId?: string
4603
+ did: string, prefix: string, delegateDid?: string, protocol?: string, permissionGrantIds?: string[]
2716
4604
  ): Promise<string[]> {
2717
4605
  const si = this.stateIndex;
2718
4606
  if (si) {
@@ -2729,10 +4617,43 @@ export class SyncEngineLevel implements SyncEngine {
2729
4617
  messageType : DwnInterface.MessagesSync,
2730
4618
  granteeDid : delegateDid,
2731
4619
  messageParams : {
2732
- action: 'leaves',
4620
+ action : 'leaves',
2733
4621
  prefix,
2734
4622
  protocol,
2735
- permissionGrantId
4623
+ permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
4624
+ }
4625
+ });
4626
+ const reply = response.reply as MessagesSyncReply;
4627
+ return reply.entries ?? [];
4628
+ }
4629
+
4630
+ private async getLocalProjectedLeaves(
4631
+ did: string,
4632
+ prefix: string,
4633
+ delegateDid: string | undefined,
4634
+ scopes: readonly [RecordsProjectionScope, ...RecordsProjectionScope[]],
4635
+ permissionGrantIds?: string[],
4636
+ ): Promise<string[]> {
4637
+ if (this.stateIndex) {
4638
+ return RecordsProjection.getLeaves({
4639
+ tenant : did,
4640
+ messageStore : this.agent.dwn.node.storage.messageStore,
4641
+ scopes,
4642
+ prefix : SyncEngineLevel.parseBitPrefix(prefix),
4643
+ });
4644
+ }
4645
+
4646
+ const response = await this.agent.dwn.processRequest({
4647
+ author : did,
4648
+ target : did,
4649
+ messageType : DwnInterface.MessagesSync,
4650
+ granteeDid : delegateDid,
4651
+ messageParams : {
4652
+ action : 'leaves',
4653
+ prefix,
4654
+ projectionRootVersion : RECORDS_PROJECTION_ROOT_VERSION,
4655
+ projectionScopes : [...scopes],
4656
+ permissionGrantIds : toMessagesPermissionGrantIds(permissionGrantIds)
2736
4657
  }
2737
4658
  });
2738
4659
  const reply = response.reply as MessagesSyncReply;
@@ -2751,33 +4672,161 @@ export class SyncEngineLevel implements SyncEngine {
2751
4672
  * they are processed directly without additional HTTP round-trips.
2752
4673
  * Only `messageCids` that were NOT prefetched are fetched individually.
2753
4674
  */
2754
- private async pullMessages({ did, dwnUrl, delegateDid, protocol, messageCids, prefetched }: {
4675
+ private async pullMessages({
4676
+ did,
4677
+ dwnUrl,
4678
+ delegateDid,
4679
+ protocol,
4680
+ scope,
4681
+ permissionGrantIds,
4682
+ messageCids,
4683
+ prefetched,
4684
+ verifiedInitialWrites,
4685
+ shouldContinue,
4686
+ }: {
2755
4687
  did: string;
2756
4688
  dwnUrl: string;
2757
4689
  delegateDid?: string;
2758
4690
  protocol?: string;
4691
+ scope?: SyncScope;
4692
+ permissionGrantIds?: string[];
2759
4693
  messageCids: string[];
2760
4694
  prefetched?: MessagesSyncDiffEntry[];
4695
+ verifiedInitialWrites?: RecordsWriteMessage[];
4696
+ shouldContinue?: () => boolean;
2761
4697
  }): Promise<void> {
4698
+ const acceptanceScope: SyncScope = scope ?? (protocol === undefined
4699
+ ? { kind: 'full' }
4700
+ : { kind: 'protocolSet', protocols: [protocol] });
4701
+ const rejectedPullEntries = new Map<string, Extract<PullAcceptanceResult, { accepted: false }>>();
2762
4702
  const failedCids = await pullMessages({
2763
- did, dwnUrl, delegateDid, protocol, messageCids, prefetched,
2764
- agent : this.agent,
2765
- permissionsApi : this._permissionsApi,
4703
+ did,
4704
+ dwnUrl,
4705
+ delegateDid,
4706
+ permissionGrantIds,
4707
+ messageCids,
4708
+ prefetched,
4709
+ shouldContinue,
4710
+ agent : this.agent,
4711
+ acceptEntry : async (entry, entries) => {
4712
+ const result = await this.acceptPulledSyncEntry(did, acceptanceScope, entry, entries, verifiedInitialWrites);
4713
+ if (!result.accepted) {
4714
+ rejectedPullEntries.set(await getMessageCid(entry.message), result);
4715
+ }
4716
+ return result.accepted;
4717
+ },
2766
4718
  });
2767
4719
 
2768
4720
  // Record permanently failed pull entries in the dead letter store.
2769
4721
  for (const cid of failedCids) {
4722
+ const rejection = rejectedPullEntries.get(cid);
2770
4723
  await this.recordDeadLetter({
2771
4724
  messageCid : cid,
2772
4725
  tenantDid : did,
2773
4726
  remoteEndpoint : dwnUrl,
2774
4727
  protocol,
2775
- category : 'pull-processing',
2776
- errorDetail : 'pull processing failed after retry passes exhausted',
4728
+ category : rejection ? 'pull-scope-rejected' : 'pull-processing',
4729
+ errorCode : rejection?.classification,
4730
+ errorDetail : rejection
4731
+ ? `pulled message rejected by ${rejection.classification} sync scope gate`
4732
+ : 'pull processing failed after retry passes exhausted',
2777
4733
  });
2778
4734
  }
2779
4735
  }
2780
4736
 
4737
+ private async acceptPulledSyncEntry(
4738
+ did: string,
4739
+ scope: SyncScope,
4740
+ entry: SyncMessageEntry,
4741
+ entries: SyncMessageEntry[],
4742
+ verifiedInitialWrites: RecordsWriteMessage[] = [],
4743
+ ): Promise<PullAcceptanceResult> {
4744
+ if (scope.kind === 'full') {
4745
+ return { accepted: true };
4746
+ }
4747
+
4748
+ const initialWrite = await this.resolvePulledDeleteInitialWrite(did, entry.message, entries, verifiedInitialWrites);
4749
+ const classification = classifySyncMessageScope({
4750
+ message: entry.message,
4751
+ initialWrite,
4752
+ scope,
4753
+ });
4754
+
4755
+ if (classification === 'in-scope') {
4756
+ return { accepted: true };
4757
+ }
4758
+
4759
+ const messageCid = await getMessageCid(entry.message);
4760
+ console.warn(`SyncEngineLevel: refusing to apply ${classification} pulled message ${messageCid}`);
4761
+ return { accepted: false, classification };
4762
+ }
4763
+
4764
+ private async resolvePulledDeleteInitialWrite(
4765
+ did: string,
4766
+ message: GenericMessage,
4767
+ entries: SyncMessageEntry[],
4768
+ verifiedInitialWrites: RecordsWriteMessage[] = [],
4769
+ ): Promise<RecordsWriteMessage | undefined> {
4770
+ const descriptor = message.descriptor as Record<string, unknown>;
4771
+ if (
4772
+ descriptor.interface !== DwnInterfaceName.Records ||
4773
+ descriptor.method !== DwnMethodName.Delete ||
4774
+ typeof descriptor.recordId !== 'string'
4775
+ ) {
4776
+ return undefined;
4777
+ }
4778
+
4779
+ if (!this.agent.dwn.isRemoteMode) {
4780
+ const localInitialWrite = await RecordsWrite.fetchInitialRecordsWriteMessage(
4781
+ this.agent.dwn.node.storage.messageStore,
4782
+ did,
4783
+ descriptor.recordId,
4784
+ );
4785
+ if (localInitialWrite) {
4786
+ return localInitialWrite;
4787
+ }
4788
+ }
4789
+
4790
+ const verifiedInitialWrite = SyncEngineLevel.findInitialWriteByRecordId(descriptor.recordId, verifiedInitialWrites);
4791
+ if (verifiedInitialWrite !== undefined) {
4792
+ return verifiedInitialWrite;
4793
+ }
4794
+
4795
+ // Batch entries are only used when the initial write has not been applied
4796
+ // locally yet. Verified dependency hints cover the projected remote-mode
4797
+ // path where the initial write was applied in the previous pull batch and
4798
+ // no embedded local message store is available. Batch entries are still
4799
+ // parsed as initial RecordsWrite messages, and processRawMessage
4800
+ // authenticates the delete before any local mutation occurs.
4801
+ return this.findInitialWriteInPullBatch(descriptor.recordId, entries);
4802
+ }
4803
+
4804
+ private static findInitialWriteByRecordId(
4805
+ recordId: string,
4806
+ initialWrites: RecordsWriteMessage[],
4807
+ ): RecordsWriteMessage | undefined {
4808
+ return initialWrites.find(initialWrite => initialWrite.recordId === recordId);
4809
+ }
4810
+
4811
+ private async findInitialWriteInPullBatch(
4812
+ recordId: string,
4813
+ entries: SyncMessageEntry[],
4814
+ ): Promise<RecordsWriteMessage | undefined> {
4815
+ for (const entry of entries) {
4816
+ if (entry.message.descriptor.interface !== DwnInterfaceName.Records ||
4817
+ entry.message.descriptor.method !== DwnMethodName.Write) {
4818
+ continue;
4819
+ }
4820
+
4821
+ const candidate = entry.message as RecordsWriteMessage;
4822
+ if (candidate.recordId === recordId && await RecordsWrite.isInitialWrite(candidate)) {
4823
+ return candidate;
4824
+ }
4825
+ }
4826
+
4827
+ return undefined;
4828
+ }
4829
+
2781
4830
  // ---------------------------------------------------------------------------
2782
4831
  // Echo-loop suppression
2783
4832
  // ---------------------------------------------------------------------------
@@ -2828,17 +4877,16 @@ export class SyncEngineLevel implements SyncEngine {
2828
4877
  * Reads missing messages from the local DWN and pushes them to the remote DWN
2829
4878
  * in dependency order (topological sort).
2830
4879
  */
2831
- private async pushMessages({ did, dwnUrl, delegateDid, protocol, messageCids }: {
4880
+ private async pushMessages({ did, dwnUrl, delegateDid, permissionGrantIds, messageCids }: {
2832
4881
  did: string;
2833
4882
  dwnUrl: string;
2834
4883
  delegateDid?: string;
2835
- protocol?: string;
4884
+ permissionGrantIds?: string[];
2836
4885
  messageCids: string[];
2837
4886
  }): Promise<PushResult> {
2838
4887
  return pushMessages({
2839
- did, dwnUrl, delegateDid, protocol, messageCids,
2840
- agent : this.agent,
2841
- permissionsApi : this._permissionsApi,
4888
+ did, dwnUrl, delegateDid, permissionGrantIds, messageCids,
4889
+ agent: this.agent,
2842
4890
  });
2843
4891
  }
2844
4892
 
@@ -2867,14 +4915,20 @@ export class SyncEngineLevel implements SyncEngine {
2867
4915
  * When `protocol` is undefined (full-tenant link), clears entries that
2868
4916
  * also have no protocol.
2869
4917
  */
2870
- private async clearDeadLettersForLink(tenantDid: string, remoteEndpoint: string, protocol?: string): Promise<void> {
4918
+ private async clearDeadLettersForLink(
4919
+ tenantDid: string,
4920
+ remoteEndpoint: string,
4921
+ protocol?: string,
4922
+ options: { categories?: ReadonlySet<DeadLetterCategory> } = {},
4923
+ ): Promise<void> {
2871
4924
  const batch: { type: 'del'; key: string }[] = [];
2872
4925
  try {
2873
4926
  for await (const [key, value] of this._deadLetters.iterator()) {
2874
4927
  const entry = JSON.parse(value) as DeadLetterEntry;
2875
4928
  if (entry.tenantDid === tenantDid &&
2876
4929
  entry.remoteEndpoint === remoteEndpoint &&
2877
- entry.protocol === protocol) {
4930
+ entry.protocol === protocol &&
4931
+ (options.categories === undefined || options.categories.has(entry.category))) {
2878
4932
  batch.push({ type: 'del', key });
2879
4933
  }
2880
4934
  }
@@ -2887,6 +4941,20 @@ export class SyncEngineLevel implements SyncEngine {
2887
4941
  }
2888
4942
  }
2889
4943
 
4944
+ private async clearRootConvergenceDeadLetters(
4945
+ tenantDid: string,
4946
+ remoteEndpoint: string,
4947
+ protocol?: string,
4948
+ ): Promise<void> {
4949
+ try {
4950
+ await this.clearDeadLettersForLink(tenantDid, remoteEndpoint, protocol, {
4951
+ categories: SyncEngineLevel.ROOT_CONVERGENCE_CLEARABLE_DEAD_LETTER_CATEGORIES,
4952
+ });
4953
+ } catch (error) {
4954
+ console.warn(`SyncEngineLevel: Failed to clear root-convergence dead letters for ${tenantDid} -> ${remoteEndpoint}`, error);
4955
+ }
4956
+ }
4957
+
2890
4958
  /**
2891
4959
  * Build a compound dead letter key. Different remotes can fail the same CID
2892
4960
  * for different reasons, so the key includes the remote endpoint.
@@ -2929,7 +4997,7 @@ export class SyncEngineLevel implements SyncEngine {
2929
4997
  }
2930
4998
  }
2931
4999
  // Deterministic ordering: newest first so apps see the most recent failures.
2932
- entries.sort((a, b) => b.failedAt.localeCompare(a.failedAt));
5000
+ entries.sort((a, b) => lexicographicalCompare(b.failedAt, a.failedAt));
2933
5001
  return entries;
2934
5002
  }
2935
5003
 
@@ -2984,44 +5052,87 @@ export class SyncEngineLevel implements SyncEngine {
2984
5052
 
2985
5053
  public async getSyncHealth(): Promise<SyncHealthSummary> {
2986
5054
  let failedMessageCount = 0;
2987
- for await (const _ of this._deadLetters.iterator()) {
5055
+ let closureFailureCount = 0;
5056
+ for await (const [, value] of this._deadLetters.iterator()) {
2988
5057
  failedMessageCount++;
5058
+ const entry = JSON.parse(value) as DeadLetterEntry;
5059
+ if (entry.category === 'closure') {
5060
+ closureFailureCount++;
5061
+ }
2989
5062
  }
2990
5063
 
2991
- // Count degraded links from the durable ledger, not just in-memory
2992
- // _activeLinks. Links persist across restarts; a repairing/degraded_poll
2993
- // link from a previous session must still be reported.
5064
+ // Superseded authorization epochs can leave durable link state behind. Only
5065
+ // links that still belong to the current registered projection/epoch should
5066
+ // affect health. Endpoint-level orphan cleanup is a separate GC concern.
5067
+ const currentLinkIdentityKeys = await this.getCurrentDurableLinkIdentityKeys();
2994
5068
  let degradedLinkCount = 0;
2995
5069
  const allLinks = await this.ledger.getAllLinks();
2996
5070
  for (const link of allLinks) {
2997
- if (link.status === 'repairing' || link.status === 'degraded_poll') {
5071
+ const isCurrentLink = currentLinkIdentityKeys === undefined || currentLinkIdentityKeys.has(this.getDurableLinkIdentityKey(link));
5072
+ if (isCurrentLink && SyncEngineLevel.isUnhealthyLinkStatus(link.status)) {
2998
5073
  degradedLinkCount++;
2999
5074
  }
3000
5075
  }
3001
5076
 
3002
5077
  return {
3003
- connectivity: this.connectivityState,
3004
- failedMessageCount,
3005
- degradedLinkCount,
5078
+ connectivity : this.connectivityState,
5079
+ failedMessageCount : failedMessageCount,
5080
+ closureFailureCount : closureFailureCount,
5081
+ degradedLinkCount : degradedLinkCount,
5082
+ syncHealthy : failedMessageCount === 0 && degradedLinkCount === 0,
3006
5083
  };
3007
5084
  }
3008
5085
 
5086
+ private async getCurrentDurableLinkIdentityKeys(): Promise<Set<string> | undefined> {
5087
+ try {
5088
+ const identityKeys = new Set<string>();
5089
+ for await (const [did, options] of this._db.sublevel('registeredIdentities').iterator()) {
5090
+ let parsed: SyncIdentityOptions;
5091
+ try {
5092
+ parsed = JSON.parse(options) as SyncIdentityOptions;
5093
+ } catch (error: unknown) {
5094
+ console.warn(`SyncEngineLevel: Corrupt sync options for ${did}, skipping health target:`, error);
5095
+ continue;
5096
+ }
5097
+
5098
+ const scope = syncScopeFromProtocols(parsed.protocols);
5099
+ const resolutions = await this.buildSyncTargetResolutions(did, scope, parsed);
5100
+ for (const resolution of resolutions) {
5101
+ const projectionId = await computeProjectionId(did, resolution.scope);
5102
+ identityKeys.add(SyncEngineLevel.durableLinkIdentityKey(did, projectionId, resolution.authorizationEpoch));
5103
+ }
5104
+ }
5105
+ return identityKeys;
5106
+ } catch (error: unknown) {
5107
+ console.warn('SyncEngineLevel: Failed to resolve current durable link identity keys for health; falling back to all durable links', error);
5108
+ return undefined;
5109
+ }
5110
+ }
5111
+
5112
+ private getDurableLinkIdentityKey(link: ReplicationLinkState): string {
5113
+ return SyncEngineLevel.durableLinkIdentityKey(link.tenantDid, link.projectionId, link.authorizationEpoch);
5114
+ }
5115
+
5116
+ private static durableLinkIdentityKey(tenantDid: string, projectionId: string, authorizationEpoch: string): string {
5117
+ return `${tenantDid}^${projectionId}^${authorizationEpoch}`;
5118
+ }
5119
+
5120
+ private static isUnhealthyLinkStatus(status: ReplicationLinkState['status']): boolean {
5121
+ return status === 'repairing' || status === 'degraded_poll' || status === 'terminal_incomplete';
5122
+ }
5123
+
3009
5124
  // ---------------------------------------------------------------------------
3010
5125
  // Sync targets
3011
5126
  // ---------------------------------------------------------------------------
3012
5127
 
3013
5128
  /**
3014
- * Returns the list of sync targets: (did, dwnUrl, delegateDid?, protocol?) tuples.
5129
+ * Returns the list of sync targets: one canonical projection target per
5130
+ * registered DID and resolved DWN endpoint.
3015
5131
  * Results are cached for up to 30 seconds to avoid redundant DID resolution
3016
5132
  * on every sync tick. The cache is invalidated when identities are registered,
3017
5133
  * unregistered, or updated.
3018
5134
  */
3019
- private async getSyncTargets(): Promise<{
3020
- did: string;
3021
- dwnUrl: string;
3022
- delegateDid?: string;
3023
- protocol?: string;
3024
- }[]> {
5135
+ private async getSyncTargets(): Promise<SyncTarget[]> {
3025
5136
  // Return cached targets if still valid.
3026
5137
  if (this._syncTargetsCache
3027
5138
  && (Date.now() - this._syncTargetsCache.timestamp) < SyncEngineLevel.SYNC_TARGETS_CACHE_TTL_MS) {
@@ -3033,7 +5144,7 @@ export class SyncEngineLevel implements SyncEngine {
3033
5144
  // make our result stale.
3034
5145
  const generationAtStart = this._syncTargetsCacheGeneration;
3035
5146
 
3036
- const targets: { did: string; dwnUrl: string; delegateDid?: string; protocol?: string }[] = [];
5147
+ const targets: SyncTarget[] = [];
3037
5148
  let hasRegisteredIdentities = false;
3038
5149
  let anyEndpointMissing = false;
3039
5150
 
@@ -3047,8 +5158,6 @@ export class SyncEngineLevel implements SyncEngine {
3047
5158
  continue;
3048
5159
  }
3049
5160
 
3050
- const { protocols, delegateDid } = parsed;
3051
-
3052
5161
  const dwnEndpointUrls = await this.agent.dwn.getDwnEndpointUrlsForTarget(did);
3053
5162
  if (dwnEndpointUrls.length === 0) {
3054
5163
  anyEndpointMissing = true;
@@ -3056,14 +5165,7 @@ export class SyncEngineLevel implements SyncEngine {
3056
5165
  }
3057
5166
 
3058
5167
  for (const dwnUrl of dwnEndpointUrls) {
3059
- if (protocols === 'all') {
3060
- // Sync all protocols (global tree).
3061
- targets.push({ did, delegateDid, dwnUrl });
3062
- } else {
3063
- for (const protocol of protocols) {
3064
- targets.push({ did, delegateDid, dwnUrl, protocol });
3065
- }
3066
- }
5168
+ targets.push(...await this.buildSyncTargetsForEndpoint(did, dwnUrl, parsed));
3067
5169
  }
3068
5170
  }
3069
5171
 
@@ -3081,22 +5183,4 @@ export class SyncEngineLevel implements SyncEngine {
3081
5183
  return targets;
3082
5184
  }
3083
5185
 
3084
- /**
3085
- * Gets the permission grant ID for MessagesSync if a delegateDid is provided.
3086
- * Returns undefined if no delegate is in use (owner access).
3087
- */
3088
- private async getSyncPermissionGrantId(did: string, delegateDid?: string, protocol?: string): Promise<string | undefined> {
3089
- if (!delegateDid) {
3090
- return undefined;
3091
- }
3092
-
3093
- const messagesSyncGrant = await this._permissionsApi.getPermissionForRequest({
3094
- connectedDid : did,
3095
- messageType : DwnInterface.MessagesSync,
3096
- delegateDid,
3097
- protocol,
3098
- cached : true
3099
- });
3100
- return messagesSyncGrant.grant.id;
3101
- }
3102
5186
  }