@enbox/agent 0.7.7 → 0.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.mjs +9 -9
- package/dist/browser.mjs.map +4 -4
- package/dist/esm/dwn-api.js +3 -2
- package/dist/esm/dwn-api.js.map +1 -1
- package/dist/esm/enbox-connect-protocol.js +5 -5
- package/dist/esm/enbox-connect-protocol.js.map +1 -1
- package/dist/esm/index.js +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/permissions-api.js +7 -34
- package/dist/esm/permissions-api.js.map +1 -1
- package/dist/esm/sync-closure-resolver.js +229 -110
- package/dist/esm/sync-closure-resolver.js.map +1 -1
- package/dist/esm/sync-closure-types.js +39 -7
- package/dist/esm/sync-closure-types.js.map +1 -1
- package/dist/esm/sync-engine-level.js +2242 -797
- package/dist/esm/sync-engine-level.js.map +1 -1
- package/dist/esm/sync-link-id.js +4 -13
- package/dist/esm/sync-link-id.js.map +1 -1
- package/dist/esm/sync-link-reconciler.js +26 -8
- package/dist/esm/sync-link-reconciler.js.map +1 -1
- package/dist/esm/sync-messages.js +218 -154
- package/dist/esm/sync-messages.js.map +1 -1
- package/dist/esm/sync-permission-grants.js +208 -0
- package/dist/esm/sync-permission-grants.js.map +1 -0
- package/dist/esm/sync-replication-ledger.js +23 -40
- package/dist/esm/sync-replication-ledger.js.map +1 -1
- package/dist/esm/sync-scope-acceptance.js +126 -0
- package/dist/esm/sync-scope-acceptance.js.map +1 -0
- package/dist/esm/sync-topological-sort.js +57 -15
- package/dist/esm/sync-topological-sort.js.map +1 -1
- package/dist/esm/types/sync.js +130 -22
- package/dist/esm/types/sync.js.map +1 -1
- package/dist/types/dwn-api.d.ts.map +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/permissions-api.d.ts +1 -2
- package/dist/types/permissions-api.d.ts.map +1 -1
- package/dist/types/sync-closure-resolver.d.ts.map +1 -1
- package/dist/types/sync-closure-types.d.ts +14 -3
- package/dist/types/sync-closure-types.d.ts.map +1 -1
- package/dist/types/sync-engine-level.d.ts +144 -31
- package/dist/types/sync-engine-level.d.ts.map +1 -1
- package/dist/types/sync-link-id.d.ts +3 -9
- package/dist/types/sync-link-id.d.ts.map +1 -1
- package/dist/types/sync-link-reconciler.d.ts +12 -2
- package/dist/types/sync-link-reconciler.d.ts.map +1 -1
- package/dist/types/sync-messages.d.ts +16 -13
- package/dist/types/sync-messages.d.ts.map +1 -1
- package/dist/types/sync-permission-grants.d.ts +52 -0
- package/dist/types/sync-permission-grants.d.ts.map +1 -0
- package/dist/types/sync-replication-ledger.d.ts +5 -13
- package/dist/types/sync-replication-ledger.d.ts.map +1 -1
- package/dist/types/sync-scope-acceptance.d.ts +28 -0
- package/dist/types/sync-scope-acceptance.d.ts.map +1 -0
- package/dist/types/sync-topological-sort.d.ts +2 -1
- package/dist/types/sync-topological-sort.d.ts.map +1 -1
- package/dist/types/types/permissions.d.ts +2 -0
- package/dist/types/types/permissions.d.ts.map +1 -1
- package/dist/types/types/sync.d.ts +137 -75
- package/dist/types/types/sync.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/dwn-api.ts +3 -2
- package/src/enbox-connect-protocol.ts +5 -5
- package/src/index.ts +10 -1
- package/src/permissions-api.ts +11 -42
- package/src/sync-closure-resolver.ts +306 -126
- package/src/sync-closure-types.ts +54 -9
- package/src/sync-engine-level.ts +3051 -967
- package/src/sync-link-id.ts +9 -14
- package/src/sync-link-reconciler.ts +43 -10
- package/src/sync-messages.ts +263 -159
- package/src/sync-permission-grants.ts +297 -0
- package/src/sync-replication-ledger.ts +55 -50
- package/src/sync-scope-acceptance.ts +186 -0
- package/src/sync-topological-sort.ts +89 -21
- package/src/types/permissions.ts +2 -0
- package/src/types/sync.ts +235 -62
|
@@ -17,20 +17,22 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
|
17
17
|
import ms from 'ms';
|
|
18
18
|
import { Level } from 'level';
|
|
19
19
|
import { sleep } from '@enbox/common';
|
|
20
|
-
import { Encoder, hashToHex, initDefaultHashes, Message } from '@enbox/dwn-sdk-js';
|
|
21
|
-
import { evaluateClosure } from './sync-closure-resolver.js';
|
|
22
|
-
import { MAX_PENDING_TOKENS } from './types/sync.js';
|
|
23
|
-
import { ReplicationLedger } from './sync-replication-ledger.js';
|
|
24
|
-
import { createClosureContext, invalidateClosureCache } from './sync-closure-types.js';
|
|
20
|
+
import { authenticate, DwnInterfaceName, DwnMethodName, Encoder, hashToHex, initDefaultHashes, Message, ProtocolsConfigure, RECORDS_PROJECTION_ROOT_VERSION, RecordsProjection, RecordsWrite } from '@enbox/dwn-sdk-js';
|
|
25
21
|
import { AgentPermissionsApi } from './permissions-api.js';
|
|
22
|
+
import { buildLinkId } from './sync-link-id.js';
|
|
26
23
|
import { DwnInterface } from './types/dwn.js';
|
|
24
|
+
import { evaluateClosure } from './sync-closure-resolver.js';
|
|
27
25
|
import { isRecordsWrite } from './utils.js';
|
|
28
|
-
import {
|
|
26
|
+
import { ReplicationLedger } from './sync-replication-ledger.js';
|
|
29
27
|
import { topologicalSort } from './sync-topological-sort.js';
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
28
|
+
import { classifySyncEventScope, classifySyncMessageScope } from './sync-scope-acceptance.js';
|
|
29
|
+
import { ClosureFailureCode, createClosureContext, invalidateClosureCache, isTerminalClosureFailureCode } from './sync-closure-types.js';
|
|
30
|
+
import { computeAuthorizationEpoch, computeProjectionId, lexicographicalCompare, MAX_PENDING_TOKENS, protocolsForSyncScope, singleProtocolForSyncScope, syncScopeFromProtocols } from './types/sync.js';
|
|
31
|
+
import { fetchRemoteMessages, getMessageCid, pullMessages, pushMessages, SyncPullAbortedError } from './sync-messages.js';
|
|
32
|
+
import { partitionRemoteEntries, SyncLinkReconciler } from './sync-link-reconciler.js';
|
|
33
|
+
import { permissionGrantIdsFromEntries, resolveMessagesSyncScopes, toMessagesPermissionGrantIds, toSyncAuthorizationGrants } from './sync-permission-grants.js';
|
|
32
34
|
/**
|
|
33
|
-
* Maximum bit prefix depth for the per-node tree walk
|
|
35
|
+
* Maximum bit prefix depth for the per-node tree walk fallback.
|
|
34
36
|
* At depth 16, each subtree covers ~1/65536 of the key space.
|
|
35
37
|
*/
|
|
36
38
|
const MAX_DIFF_DEPTH = 16;
|
|
@@ -49,50 +51,29 @@ const BATCHED_DIFF_DEPTH = 8;
|
|
|
49
51
|
* once the in-flight push completes.
|
|
50
52
|
*/
|
|
51
53
|
const PUSH_DEBOUNCE_MS = 100;
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
if (scope.kind === 'full') {
|
|
67
|
-
return true;
|
|
68
|
-
}
|
|
69
|
-
if (!scope.protocolPathPrefixes && !scope.contextIdPrefixes) {
|
|
70
|
-
return true;
|
|
71
|
-
}
|
|
72
|
-
const desc = message.descriptor;
|
|
73
|
-
// Check protocolPath prefix.
|
|
74
|
-
if (scope.protocolPathPrefixes && scope.protocolPathPrefixes.length > 0) {
|
|
75
|
-
const protocolPath = desc.protocolPath;
|
|
76
|
-
if (!protocolPath) {
|
|
77
|
-
return false;
|
|
78
|
-
}
|
|
79
|
-
const matches = scope.protocolPathPrefixes.some(prefix => protocolPath === prefix || protocolPath.startsWith(prefix + '/'));
|
|
80
|
-
if (!matches) {
|
|
81
|
-
return false;
|
|
82
|
-
}
|
|
54
|
+
var LinkSubscriptionOpenResult;
|
|
55
|
+
(function (LinkSubscriptionOpenResult) {
|
|
56
|
+
LinkSubscriptionOpenResult["ReadyForLive"] = "readyForLive";
|
|
57
|
+
LinkSubscriptionOpenResult["Polling"] = "polling";
|
|
58
|
+
LinkSubscriptionOpenResult["Repairing"] = "repairing";
|
|
59
|
+
})(LinkSubscriptionOpenResult || (LinkSubscriptionOpenResult = {}));
|
|
60
|
+
var LinkInitializationStatus;
|
|
61
|
+
(function (LinkInitializationStatus) {
|
|
62
|
+
LinkInitializationStatus["Active"] = "active";
|
|
63
|
+
LinkInitializationStatus["Failed"] = "failed";
|
|
64
|
+
})(LinkInitializationStatus || (LinkInitializationStatus = {}));
|
|
65
|
+
function syncEventScope(scope) {
|
|
66
|
+
if (scope === undefined) {
|
|
67
|
+
return {};
|
|
83
68
|
}
|
|
84
|
-
|
|
85
|
-
if (
|
|
86
|
-
|
|
87
|
-
if (!contextId) {
|
|
88
|
-
return false;
|
|
89
|
-
}
|
|
90
|
-
const matches = scope.contextIdPrefixes.some(prefix => contextId === prefix || contextId.startsWith(prefix + '/'));
|
|
91
|
-
if (!matches) {
|
|
92
|
-
return false;
|
|
93
|
-
}
|
|
69
|
+
const coveredProtocols = protocolsForSyncScope(scope);
|
|
70
|
+
if (coveredProtocols === undefined) {
|
|
71
|
+
return {};
|
|
94
72
|
}
|
|
95
|
-
|
|
73
|
+
const protocols = [...coveredProtocols];
|
|
74
|
+
return protocols.length === 1
|
|
75
|
+
? { protocol: protocols[0], protocols }
|
|
76
|
+
: { protocols };
|
|
96
77
|
}
|
|
97
78
|
export class SyncEngineLevel {
|
|
98
79
|
/** Validate `SyncIdentityOptions` for `registerIdentity` and `updateIdentityOptions`. */
|
|
@@ -107,6 +88,54 @@ export class SyncEngineLevel {
|
|
|
107
88
|
throw new Error('SyncEngineLevel: protocols must be \'all\' or a non-empty array of protocol URIs. An empty array is ambiguous.');
|
|
108
89
|
}
|
|
109
90
|
}
|
|
91
|
+
buildSyncTargetsForEndpoint(did, dwnUrl, options) {
|
|
92
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
93
|
+
const requestedScope = syncScopeFromProtocols(options.protocols);
|
|
94
|
+
const resolutions = yield this.buildSyncTargetResolutions(did, requestedScope, options);
|
|
95
|
+
return resolutions.map(resolution => (Object.assign({ did,
|
|
96
|
+
dwnUrl }, resolution)));
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
buildSyncTargetResolutions(did, requestedScope, options) {
|
|
100
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
101
|
+
const { delegateDid } = options;
|
|
102
|
+
if (delegateDid === undefined) {
|
|
103
|
+
return [{
|
|
104
|
+
scope: requestedScope,
|
|
105
|
+
authorization: { kind: 'owner' },
|
|
106
|
+
authorizationEpoch: yield computeAuthorizationEpoch({ kind: 'owner' }),
|
|
107
|
+
}];
|
|
108
|
+
}
|
|
109
|
+
const resolvedScopes = yield resolveMessagesSyncScopes({
|
|
110
|
+
did,
|
|
111
|
+
delegateDid,
|
|
112
|
+
requestedScope,
|
|
113
|
+
messageType: DwnInterface.MessagesSync,
|
|
114
|
+
permissionsApi: this._permissionsApi,
|
|
115
|
+
});
|
|
116
|
+
return Promise.all(resolvedScopes.map((_a) => __awaiter(this, [_a], void 0, function* ({ scope, permissionGrants }) {
|
|
117
|
+
const permissionGrantIds = permissionGrantIdsFromEntries(permissionGrants);
|
|
118
|
+
if (permissionGrantIds === undefined) {
|
|
119
|
+
throw new Error(`SyncEngineLevel: delegate ${delegateDid} has no active sync grants for ${did}.`);
|
|
120
|
+
}
|
|
121
|
+
return {
|
|
122
|
+
scope,
|
|
123
|
+
delegateDid,
|
|
124
|
+
authorization: {
|
|
125
|
+
kind: 'delegate',
|
|
126
|
+
delegateDid,
|
|
127
|
+
permissionGrantIds,
|
|
128
|
+
},
|
|
129
|
+
authorizationEpoch: yield computeAuthorizationEpoch({
|
|
130
|
+
kind: 'delegate',
|
|
131
|
+
delegateDid,
|
|
132
|
+
grants: toSyncAuthorizationGrants(permissionGrants),
|
|
133
|
+
}),
|
|
134
|
+
permissionGrantIds,
|
|
135
|
+
};
|
|
136
|
+
})));
|
|
137
|
+
});
|
|
138
|
+
}
|
|
110
139
|
constructor({ agent, dataPath, db }) {
|
|
111
140
|
this._syncLock = false;
|
|
112
141
|
/**
|
|
@@ -157,6 +186,8 @@ export class SyncEngineLevel {
|
|
|
157
186
|
* tenant. Keyed by tenantDid to prevent cross-tenant cache pollution.
|
|
158
187
|
*/
|
|
159
188
|
this._closureContexts = new Map();
|
|
189
|
+
/** Deduplicates concurrent live-sync repairs for the same tenant/protocol. */
|
|
190
|
+
this._protocolMetadataRepairs = new Map();
|
|
160
191
|
/**
|
|
161
192
|
* Monotonic generation counter for sync target cache invalidation.
|
|
162
193
|
* Bumped on every invalidation (register/unregister/update/clear/close/stopSync).
|
|
@@ -300,7 +331,13 @@ export class SyncEngineLevel {
|
|
|
300
331
|
this._syncTargetsCacheGeneration++;
|
|
301
332
|
// If live sync is active, hot-add subscriptions for this identity.
|
|
302
333
|
if (this._syncMode === 'live') {
|
|
303
|
-
yield this.addIdentityToLiveSync(did, options);
|
|
334
|
+
const currentIdentityKeys = yield this.addIdentityToLiveSync(did, options);
|
|
335
|
+
if (currentIdentityKeys.size > 0) {
|
|
336
|
+
yield this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
else {
|
|
340
|
+
yield this.tryPruneSupersededDurableLinksForRegisteredIdentity(did, options);
|
|
304
341
|
}
|
|
305
342
|
});
|
|
306
343
|
}
|
|
@@ -318,6 +355,7 @@ export class SyncEngineLevel {
|
|
|
318
355
|
yield registeredIdentities.del(did);
|
|
319
356
|
this._syncTargetsCache = undefined;
|
|
320
357
|
this._syncTargetsCacheGeneration++;
|
|
358
|
+
yield this.pruneSupersededDurableLinksForIdentity(did, new Set());
|
|
321
359
|
});
|
|
322
360
|
}
|
|
323
361
|
getIdentityOptions(did) {
|
|
@@ -352,18 +390,18 @@ export class SyncEngineLevel {
|
|
|
352
390
|
yield registeredIdentities.put(did, JSON.stringify(options));
|
|
353
391
|
this._syncTargetsCache = undefined;
|
|
354
392
|
this._syncTargetsCacheGeneration++;
|
|
355
|
-
// Always persist the new delegate to durable links, regardless of
|
|
356
|
-
// sync mode. If sync is stopped or polling, existing persisted links
|
|
357
|
-
// would otherwise keep the old delegateDid. When live sync starts
|
|
358
|
-
// later, initializeLinkTarget() loads the link from LevelDB without
|
|
359
|
-
// normalizing delegateDid, so repair/reconcile paths could use stale
|
|
360
|
-
// delegate data.
|
|
361
|
-
yield this.ledger.updateDelegateDid(did, options.delegateDid);
|
|
362
393
|
// If live sync is active, tear down and rebuild subscriptions with
|
|
363
|
-
// the new options.
|
|
394
|
+
// the new options. Delegate/scope changes derive a new authorization
|
|
395
|
+
// epoch, so existing durable links are not mutated in place.
|
|
364
396
|
if (this._syncMode === 'live' && this.hasActiveLinksForDid(did)) {
|
|
365
397
|
yield this.removeIdentityFromLiveSync(did);
|
|
366
|
-
yield this.addIdentityToLiveSync(did, options);
|
|
398
|
+
const currentIdentityKeys = yield this.addIdentityToLiveSync(did, options);
|
|
399
|
+
if (currentIdentityKeys.size > 0) {
|
|
400
|
+
yield this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
else {
|
|
404
|
+
yield this.tryPruneSupersededDurableLinksForRegisteredIdentity(did, options);
|
|
367
405
|
}
|
|
368
406
|
});
|
|
369
407
|
}
|
|
@@ -394,16 +432,13 @@ export class SyncEngineLevel {
|
|
|
394
432
|
let groupsFailed = 0;
|
|
395
433
|
const results = yield Promise.allSettled([...byUrl.entries()].map((_a) => __awaiter(this, [_a], void 0, function* ([dwnUrl, targets]) {
|
|
396
434
|
for (const target of targets) {
|
|
397
|
-
const { did, delegateDid, protocol } = target;
|
|
398
435
|
try {
|
|
399
|
-
yield this.
|
|
400
|
-
did, dwnUrl, delegateDid, protocol,
|
|
401
|
-
}, { direction });
|
|
436
|
+
yield this.reconcileProjectionTarget(target, { direction });
|
|
402
437
|
}
|
|
403
438
|
catch (error) {
|
|
404
439
|
// Skip remaining targets for this DWN endpoint.
|
|
405
440
|
groupsFailed++;
|
|
406
|
-
console.error(`SyncEngineLevel: Error syncing ${did} with ${dwnUrl}`, error);
|
|
441
|
+
console.error(`SyncEngineLevel: Error syncing ${target.did} with ${dwnUrl}`, error);
|
|
407
442
|
return;
|
|
408
443
|
}
|
|
409
444
|
}
|
|
@@ -502,7 +537,7 @@ export class SyncEngineLevel {
|
|
|
502
537
|
});
|
|
503
538
|
}
|
|
504
539
|
// ---------------------------------------------------------------------------
|
|
505
|
-
// Poll-mode sync
|
|
540
|
+
// Poll-mode sync
|
|
506
541
|
// ---------------------------------------------------------------------------
|
|
507
542
|
startPollSync(intervalMilliseconds) {
|
|
508
543
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -635,30 +670,75 @@ export class SyncEngineLevel {
|
|
|
635
670
|
*/
|
|
636
671
|
transitionToRepairing(linkKey, link, options) {
|
|
637
672
|
return __awaiter(this, void 0, void 0, function* () {
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
link.connectivity = 'offline';
|
|
641
|
-
yield this.ledger.setStatus(link, 'repairing');
|
|
642
|
-
this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevStatus, to: 'repairing' });
|
|
643
|
-
if (prevConnectivity !== 'offline') {
|
|
644
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevConnectivity, to: 'offline' });
|
|
673
|
+
if (link.status === 'terminal_incomplete') {
|
|
674
|
+
return;
|
|
645
675
|
}
|
|
676
|
+
yield this.setLinkOfflineStatus(link, 'repairing');
|
|
646
677
|
if (options === null || options === void 0 ? void 0 : options.resumeToken) {
|
|
647
678
|
this._repairContext.set(linkKey, { resumeToken: options.resumeToken });
|
|
648
679
|
}
|
|
649
680
|
// Clear runtime ordinals immediately — stale state must not linger
|
|
650
681
|
// across repair attempts.
|
|
651
|
-
|
|
652
|
-
if (rt) {
|
|
653
|
-
rt.inflight.clear();
|
|
654
|
-
rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
|
|
655
|
-
}
|
|
682
|
+
this.clearLinkRuntimeInflight(linkKey);
|
|
656
683
|
// Kick off repair with retry scheduling on failure.
|
|
657
684
|
void this.repairLink(linkKey).catch(() => {
|
|
658
685
|
this.scheduleRepairRetry(linkKey);
|
|
659
686
|
});
|
|
660
687
|
});
|
|
661
688
|
}
|
|
689
|
+
transitionToTerminalIncomplete(linkKey, link) {
|
|
690
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
691
|
+
if (link.status === 'terminal_incomplete') {
|
|
692
|
+
return;
|
|
693
|
+
}
|
|
694
|
+
yield this.setLinkOfflineStatus(link, 'terminal_incomplete');
|
|
695
|
+
yield this.closeLinkSubscriptions(link);
|
|
696
|
+
this.clearLinkRuntimeInflight(linkKey);
|
|
697
|
+
const retryTimer = this._repairRetryTimers.get(linkKey);
|
|
698
|
+
if (retryTimer) {
|
|
699
|
+
clearTimeout(retryTimer);
|
|
700
|
+
this._repairRetryTimers.delete(linkKey);
|
|
701
|
+
}
|
|
702
|
+
const degradedTimer = this._degradedPollTimers.get(linkKey);
|
|
703
|
+
if (degradedTimer) {
|
|
704
|
+
clearInterval(degradedTimer);
|
|
705
|
+
this._degradedPollTimers.delete(linkKey);
|
|
706
|
+
}
|
|
707
|
+
const reconcileTimer = this._reconcileTimers.get(linkKey);
|
|
708
|
+
if (reconcileTimer) {
|
|
709
|
+
clearTimeout(reconcileTimer);
|
|
710
|
+
this._reconcileTimers.delete(linkKey);
|
|
711
|
+
}
|
|
712
|
+
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
713
|
+
if (pushRuntime === null || pushRuntime === void 0 ? void 0 : pushRuntime.timer) {
|
|
714
|
+
clearTimeout(pushRuntime.timer);
|
|
715
|
+
}
|
|
716
|
+
this._pushRuntimes.delete(linkKey);
|
|
717
|
+
this._repairAttempts.delete(linkKey);
|
|
718
|
+
this._repairContext.delete(linkKey);
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
setLinkOfflineStatus(link, status) {
|
|
722
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
723
|
+
const prevStatus = link.status;
|
|
724
|
+
const prevConnectivity = link.connectivity;
|
|
725
|
+
link.connectivity = 'offline';
|
|
726
|
+
yield this.ledger.setStatus(link, status);
|
|
727
|
+
const eventScope = syncEventScope(link.scope);
|
|
728
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, eventScope), { from: prevStatus, to: status }));
|
|
729
|
+
if (prevConnectivity !== 'offline') {
|
|
730
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, eventScope), { from: prevConnectivity, to: 'offline' }));
|
|
731
|
+
}
|
|
732
|
+
});
|
|
733
|
+
}
|
|
734
|
+
clearLinkRuntimeInflight(linkKey) {
|
|
735
|
+
const rt = this._linkRuntimes.get(linkKey);
|
|
736
|
+
if (!rt) {
|
|
737
|
+
return;
|
|
738
|
+
}
|
|
739
|
+
rt.inflight.clear();
|
|
740
|
+
rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
|
|
741
|
+
}
|
|
662
742
|
/**
|
|
663
743
|
* Schedule a retry for a failed repair. Uses exponential backoff.
|
|
664
744
|
* No-op if the link is already in `degraded_poll` (timer loop owns retries)
|
|
@@ -748,8 +828,9 @@ export class SyncEngineLevel {
|
|
|
748
828
|
// `_activeLinks` may contain a *different* link object for the same key.
|
|
749
829
|
// The old repair closure must not mutate the replacement link's state.
|
|
750
830
|
const isStaleLink = () => this._activeLinks.get(linkKey) !== link;
|
|
751
|
-
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid,
|
|
752
|
-
|
|
831
|
+
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, scope, authorization } = link;
|
|
832
|
+
const eventScope = syncEventScope(scope);
|
|
833
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { attempt: ((_a = this._repairAttempts.get(linkKey)) !== null && _a !== void 0 ? _a : 0) + 1 }));
|
|
753
834
|
const attempts = ((_b = this._repairAttempts.get(linkKey)) !== null && _b !== void 0 ? _b : 0) + 1;
|
|
754
835
|
this._repairAttempts.set(linkKey, attempts);
|
|
755
836
|
// Step 1: Close existing subscriptions FIRST to stop old events from
|
|
@@ -766,7 +847,13 @@ export class SyncEngineLevel {
|
|
|
766
847
|
rt.nextCommitOrdinal = 0;
|
|
767
848
|
try {
|
|
768
849
|
// Step 3: Run SMT reconciliation for this link.
|
|
769
|
-
const reconcileOutcome = yield this.
|
|
850
|
+
const reconcileOutcome = yield this.reconcileProjectionTarget({
|
|
851
|
+
did,
|
|
852
|
+
dwnUrl,
|
|
853
|
+
delegateDid,
|
|
854
|
+
scope,
|
|
855
|
+
authorization,
|
|
856
|
+
}, undefined, () => this._engineGeneration === generation && !isStaleLink());
|
|
770
857
|
if (reconcileOutcome.aborted) {
|
|
771
858
|
return;
|
|
772
859
|
}
|
|
@@ -794,7 +881,16 @@ export class SyncEngineLevel {
|
|
|
794
881
|
if (this._engineGeneration !== generation || isStaleLink()) {
|
|
795
882
|
return;
|
|
796
883
|
}
|
|
797
|
-
const target = {
|
|
884
|
+
const target = {
|
|
885
|
+
did,
|
|
886
|
+
dwnUrl,
|
|
887
|
+
delegateDid,
|
|
888
|
+
scope,
|
|
889
|
+
authorization,
|
|
890
|
+
authorizationEpoch: link.authorizationEpoch,
|
|
891
|
+
permissionGrantIds: this.getAuthorizationGrantIds(authorization),
|
|
892
|
+
linkKey,
|
|
893
|
+
};
|
|
798
894
|
try {
|
|
799
895
|
yield this.openLivePullSubscription(target);
|
|
800
896
|
}
|
|
@@ -847,16 +943,15 @@ export class SyncEngineLevel {
|
|
|
847
943
|
const prevRepairConnectivity = link.connectivity;
|
|
848
944
|
link.connectivity = 'online';
|
|
849
945
|
yield this.ledger.setStatus(link, 'live');
|
|
850
|
-
//
|
|
851
|
-
//
|
|
852
|
-
//
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, protocol });
|
|
946
|
+
// Root convergence proves primary CID membership matches, but it does
|
|
947
|
+
// not prove dependencies are usable. Keep closure failures until a later
|
|
948
|
+
// successful apply/closure pass clears the specific CID.
|
|
949
|
+
yield this.clearRootConvergenceDeadLettersForScope(did, dwnUrl, scope);
|
|
950
|
+
this.emitEvent(Object.assign({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope));
|
|
856
951
|
if (prevRepairConnectivity !== 'online') {
|
|
857
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
952
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { from: prevRepairConnectivity, to: 'online' }));
|
|
858
953
|
}
|
|
859
|
-
this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
954
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { from: 'repairing', to: 'live' }));
|
|
860
955
|
}
|
|
861
956
|
catch (error) {
|
|
862
957
|
// If teardown occurred during repair or the link was replaced by a
|
|
@@ -865,7 +960,7 @@ export class SyncEngineLevel {
|
|
|
865
960
|
return;
|
|
866
961
|
}
|
|
867
962
|
console.error(`SyncEngineLevel: Repair failed for ${did} -> ${dwnUrl} (attempt ${attempts})`, error);
|
|
868
|
-
this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
963
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { attempt: attempts, error: String((_d = error.message) !== null && _d !== void 0 ? _d : error) }));
|
|
869
964
|
if (attempts >= SyncEngineLevel.MAX_REPAIR_ATTEMPTS) {
|
|
870
965
|
console.warn(`SyncEngineLevel: Max repair attempts reached for ${did} -> ${dwnUrl}, entering degraded_poll`);
|
|
871
966
|
yield this.enterDegradedPoll(linkKey);
|
|
@@ -882,25 +977,35 @@ export class SyncEngineLevel {
|
|
|
882
977
|
closeLinkSubscriptions(link) {
|
|
883
978
|
return __awaiter(this, void 0, void 0, function* () {
|
|
884
979
|
const { tenantDid: did, remoteEndpoint: dwnUrl } = link;
|
|
885
|
-
const linkKey = this.buildLinkKey(did, dwnUrl, link.
|
|
886
|
-
|
|
980
|
+
const linkKey = this.buildLinkKey(did, dwnUrl, link.projectionId, link.authorizationEpoch);
|
|
981
|
+
yield this.closeLiveSubscription(linkKey);
|
|
982
|
+
yield this.closeLocalSubscription(linkKey);
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
closeLiveSubscription(linkKey) {
|
|
986
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
887
987
|
const pullSub = this._liveSubscriptions.find((s) => s.linkKey === linkKey);
|
|
888
|
-
if (pullSub) {
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
|
|
988
|
+
if (!pullSub) {
|
|
989
|
+
return;
|
|
990
|
+
}
|
|
991
|
+
try {
|
|
992
|
+
yield pullSub.close();
|
|
894
993
|
}
|
|
895
|
-
|
|
994
|
+
catch ( /* best effort */_a) { /* best effort */ }
|
|
995
|
+
this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
|
|
996
|
+
});
|
|
997
|
+
}
|
|
998
|
+
closeLocalSubscription(linkKey) {
|
|
999
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
896
1000
|
const pushSub = this._localSubscriptions.find((s) => s.linkKey === linkKey);
|
|
897
|
-
if (pushSub) {
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
this._localSubscriptions = this._localSubscriptions.filter(s => s !== pushSub);
|
|
1001
|
+
if (!pushSub) {
|
|
1002
|
+
return;
|
|
1003
|
+
}
|
|
1004
|
+
try {
|
|
1005
|
+
yield pushSub.close();
|
|
903
1006
|
}
|
|
1007
|
+
catch ( /* best effort */_a) { /* best effort */ }
|
|
1008
|
+
this._localSubscriptions = this._localSubscriptions.filter(s => s !== pushSub);
|
|
904
1009
|
});
|
|
905
1010
|
}
|
|
906
1011
|
/**
|
|
@@ -918,8 +1023,9 @@ export class SyncEngineLevel {
|
|
|
918
1023
|
const prevDegradedStatus = link.status;
|
|
919
1024
|
yield this.ledger.setStatus(link, 'degraded_poll');
|
|
920
1025
|
this._repairAttempts.delete(linkKey);
|
|
921
|
-
|
|
922
|
-
this.emitEvent({ type: '
|
|
1026
|
+
const eventScope = syncEventScope(link.scope);
|
|
1027
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, eventScope), { from: prevDegradedStatus, to: 'degraded_poll' }));
|
|
1028
|
+
this.emitEvent(Object.assign({ type: 'degraded-poll:entered', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, eventScope));
|
|
923
1029
|
// Clear any existing timer for this link.
|
|
924
1030
|
const existing = this._degradedPollTimers.get(linkKey);
|
|
925
1031
|
if (existing) {
|
|
@@ -1025,14 +1131,7 @@ export class SyncEngineLevel {
|
|
|
1025
1131
|
const prev = link.connectivity;
|
|
1026
1132
|
if (prev !== 'offline') {
|
|
1027
1133
|
link.connectivity = 'offline';
|
|
1028
|
-
this.emitEvent({
|
|
1029
|
-
type: 'link:connectivity-change',
|
|
1030
|
-
tenantDid: link.tenantDid,
|
|
1031
|
-
remoteEndpoint: link.remoteEndpoint,
|
|
1032
|
-
protocol: link.protocol,
|
|
1033
|
-
from: prev,
|
|
1034
|
-
to: 'offline',
|
|
1035
|
-
});
|
|
1134
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, syncEventScope(link.scope)), { from: prev, to: 'offline' }));
|
|
1036
1135
|
}
|
|
1037
1136
|
}
|
|
1038
1137
|
};
|
|
@@ -1132,6 +1231,7 @@ export class SyncEngineLevel {
|
|
|
1132
1231
|
this._reconcileInFlight.clear();
|
|
1133
1232
|
// Clear closure evaluation contexts.
|
|
1134
1233
|
this._closureContexts.clear();
|
|
1234
|
+
this._protocolMetadataRepairs.clear();
|
|
1135
1235
|
this._recentlyPulledCids.clear();
|
|
1136
1236
|
// Clear the in-memory link and runtime state.
|
|
1137
1237
|
this._activeLinks.clear();
|
|
@@ -1143,78 +1243,123 @@ export class SyncEngineLevel {
|
|
|
1143
1243
|
// ---------------------------------------------------------------------------
|
|
1144
1244
|
/**
|
|
1145
1245
|
* Initialize a single replication link target: create or resume the durable
|
|
1146
|
-
* link,
|
|
1147
|
-
* transition the link to `'live'`.
|
|
1246
|
+
* link, open pull + push subscriptions, and transition the link to `'live'`.
|
|
1148
1247
|
*/
|
|
1149
1248
|
initializeLinkTarget(target) {
|
|
1150
1249
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1151
|
-
var _a;
|
|
1152
1250
|
let link;
|
|
1153
1251
|
try {
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
: { kind: 'full' };
|
|
1157
|
-
link = yield this.ledger.getOrCreateLink({
|
|
1158
|
-
tenantDid: target.did,
|
|
1159
|
-
remoteEndpoint: target.dwnUrl,
|
|
1160
|
-
scope: linkScope,
|
|
1161
|
-
delegateDid: target.delegateDid,
|
|
1162
|
-
protocol: target.protocol,
|
|
1163
|
-
});
|
|
1164
|
-
const linkKey = this.buildLinkKey(target.did, target.dwnUrl, link.scopeId);
|
|
1165
|
-
if (!link.pull.contiguousAppliedToken) {
|
|
1166
|
-
const legacyKey = buildLegacyCursorKey(target.did, target.dwnUrl, target.protocol);
|
|
1167
|
-
const legacyCursor = yield this.getCursor(legacyKey);
|
|
1168
|
-
if (legacyCursor) {
|
|
1169
|
-
ReplicationLedger.resetCheckpoint(link.pull, legacyCursor);
|
|
1170
|
-
yield this.ledger.saveLink(link);
|
|
1171
|
-
yield this.deleteLegacyCursor(legacyKey);
|
|
1172
|
-
}
|
|
1173
|
-
}
|
|
1252
|
+
link = yield this.getOrCreateReplicationLink(target);
|
|
1253
|
+
const linkKey = this.getReplicationLinkKey(target, link);
|
|
1174
1254
|
this._activeLinks.set(linkKey, link);
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
try {
|
|
1178
|
-
yield this.openLocalPushSubscription(targetWithKey);
|
|
1255
|
+
if (link.status === 'terminal_incomplete') {
|
|
1256
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1179
1257
|
}
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
try {
|
|
1184
|
-
yield pullSub.close();
|
|
1185
|
-
}
|
|
1186
|
-
catch ( /* best effort */_b) { /* best effort */ }
|
|
1187
|
-
this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
|
|
1188
|
-
}
|
|
1189
|
-
throw pushError;
|
|
1258
|
+
const subscriptionResult = yield this.openLinkSubscriptions(Object.assign(Object.assign({}, target), { linkKey }));
|
|
1259
|
+
if (subscriptionResult === LinkSubscriptionOpenResult.ReadyForLive) {
|
|
1260
|
+
yield this.markLinkLive(target, link, linkKey);
|
|
1190
1261
|
}
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
if (link.needsReconcile) {
|
|
1194
|
-
this.scheduleReconcile(linkKey, 1000);
|
|
1262
|
+
else if (subscriptionResult === LinkSubscriptionOpenResult.Polling) {
|
|
1263
|
+
yield this.markLinkPolling(target, link);
|
|
1195
1264
|
}
|
|
1265
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1196
1266
|
}
|
|
1197
1267
|
catch (error) {
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1268
|
+
return this.handleInitializeLinkTargetError(target, link, error);
|
|
1269
|
+
}
|
|
1270
|
+
});
|
|
1271
|
+
}
|
|
1272
|
+
getOrCreateReplicationLink(target) {
|
|
1273
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1274
|
+
return this.ledger.getOrCreateLink({
|
|
1275
|
+
tenantDid: target.did,
|
|
1276
|
+
remoteEndpoint: target.dwnUrl,
|
|
1277
|
+
scope: target.scope,
|
|
1278
|
+
authorization: target.authorization,
|
|
1279
|
+
authorizationEpoch: target.authorizationEpoch,
|
|
1280
|
+
delegateDid: target.delegateDid,
|
|
1281
|
+
});
|
|
1282
|
+
});
|
|
1283
|
+
}
|
|
1284
|
+
getReplicationLinkKey(target, link) {
|
|
1285
|
+
return this.buildLinkKey(target.did, target.dwnUrl, link.projectionId, link.authorizationEpoch);
|
|
1286
|
+
}
|
|
1287
|
+
openLinkSubscriptions(target) {
|
|
1288
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1289
|
+
if (!SyncEngineLevel.supportsLiveSubscriptions(target.scope)) {
|
|
1290
|
+
return LinkSubscriptionOpenResult.Polling;
|
|
1291
|
+
}
|
|
1292
|
+
yield this.openLivePullSubscription(target);
|
|
1293
|
+
const link = this._activeLinks.get(target.linkKey);
|
|
1294
|
+
if ((link === null || link === void 0 ? void 0 : link.status) === 'repairing') {
|
|
1295
|
+
yield this.closeLiveSubscription(target.linkKey);
|
|
1296
|
+
return LinkSubscriptionOpenResult.Repairing;
|
|
1297
|
+
}
|
|
1298
|
+
try {
|
|
1299
|
+
yield this.openLocalPushSubscription(target);
|
|
1300
|
+
}
|
|
1301
|
+
catch (error) {
|
|
1302
|
+
yield this.closeLiveSubscription(target.linkKey);
|
|
1303
|
+
throw error;
|
|
1304
|
+
}
|
|
1305
|
+
return LinkSubscriptionOpenResult.ReadyForLive;
|
|
1306
|
+
});
|
|
1307
|
+
}
|
|
1308
|
+
static supportsLiveSubscriptions(scope) {
|
|
1309
|
+
// Records-primary projected links reconcile by root/diff polling until the
|
|
1310
|
+
// DWN has explicit path/context live subscription semantics.
|
|
1311
|
+
return scope.kind !== 'recordsProjection';
|
|
1312
|
+
}
|
|
1313
|
+
markLinkLive(target, link, linkKey) {
|
|
1314
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1315
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:status-change', tenantDid: target.did, remoteEndpoint: target.dwnUrl }, syncEventScope(target.scope)), { from: 'initializing', to: 'live' }));
|
|
1316
|
+
yield this.ledger.setStatus(link, 'live');
|
|
1317
|
+
if (link.needsReconcile) {
|
|
1318
|
+
this.scheduleReconcile(linkKey, 1000);
|
|
1215
1319
|
}
|
|
1216
1320
|
});
|
|
1217
1321
|
}
|
|
1322
|
+
markLinkPolling(target, link) {
|
|
1323
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1324
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:status-change', tenantDid: target.did, remoteEndpoint: target.dwnUrl }, syncEventScope(target.scope)), { from: 'initializing', to: 'polling' }));
|
|
1325
|
+
yield this.ledger.setStatus(link, 'polling');
|
|
1326
|
+
});
|
|
1327
|
+
}
|
|
1328
|
+
handleInitializeLinkTargetError(target, link, error) {
|
|
1329
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1330
|
+
var _a;
|
|
1331
|
+
if (error.isProgressGap && link) {
|
|
1332
|
+
const linkKey = this.getReplicationLinkKey(target, link);
|
|
1333
|
+
console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
|
|
1334
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'gap:detected', tenantDid: target.did, remoteEndpoint: target.dwnUrl }, syncEventScope(target.scope)), { reason: 'ProgressGap' }));
|
|
1335
|
+
yield this.transitionToRepairing(linkKey, link, {
|
|
1336
|
+
resumeToken: (_a = error.gapInfo) === null || _a === void 0 ? void 0 : _a.latestAvailable,
|
|
1337
|
+
});
|
|
1338
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1339
|
+
}
|
|
1340
|
+
console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
|
|
1341
|
+
if (link) {
|
|
1342
|
+
this.cleanupFailedLinkInitialization(this.getReplicationLinkKey(target, link));
|
|
1343
|
+
}
|
|
1344
|
+
if (this.isDidResolutionFailure(error)) {
|
|
1345
|
+
throw error;
|
|
1346
|
+
}
|
|
1347
|
+
return { status: LinkInitializationStatus.Failed };
|
|
1348
|
+
});
|
|
1349
|
+
}
|
|
1350
|
+
createActiveLinkInitializationResult(link) {
|
|
1351
|
+
return {
|
|
1352
|
+
status: LinkInitializationStatus.Active,
|
|
1353
|
+
durableLinkIdentityKey: this.getDurableLinkIdentityKey(link),
|
|
1354
|
+
};
|
|
1355
|
+
}
|
|
1356
|
+
cleanupFailedLinkInitialization(linkKey) {
|
|
1357
|
+
this._activeLinks.delete(linkKey);
|
|
1358
|
+
this._linkRuntimes.delete(linkKey);
|
|
1359
|
+
if (this._liveSubscriptions.length === 0) {
|
|
1360
|
+
this._connectivityState = 'unknown';
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1218
1363
|
/**
|
|
1219
1364
|
* Wrapper around {@link initializeLinkTarget} that retries on DID
|
|
1220
1365
|
* resolution failures. Newly published `did:dht` DIDs take a few
|
|
@@ -1225,32 +1370,33 @@ export class SyncEngineLevel {
|
|
|
1225
1370
|
*/
|
|
1226
1371
|
initializeLinkTargetWithRetry(target) {
|
|
1227
1372
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1228
|
-
var _a;
|
|
1229
1373
|
try {
|
|
1230
|
-
yield this.initializeLinkTarget(target);
|
|
1374
|
+
return yield this.initializeLinkTarget(target);
|
|
1231
1375
|
}
|
|
1232
1376
|
catch (error) {
|
|
1233
|
-
|
|
1234
|
-
const isDidResolutionFailure = msg.includes('GetPublicKeyNotFound') || msg.includes('notFound');
|
|
1235
|
-
if (!isDidResolutionFailure) {
|
|
1377
|
+
if (!this.isDidResolutionFailure(error)) {
|
|
1236
1378
|
throw error;
|
|
1237
1379
|
}
|
|
1238
|
-
const
|
|
1239
|
-
for (const delay of delays) {
|
|
1380
|
+
for (const delay of SyncEngineLevel.DID_RESOLUTION_RETRY_BACKOFF_MS) {
|
|
1240
1381
|
yield sleep(delay);
|
|
1241
1382
|
try {
|
|
1242
|
-
yield this.initializeLinkTarget(target);
|
|
1243
|
-
return;
|
|
1383
|
+
return yield this.initializeLinkTarget(target);
|
|
1244
1384
|
}
|
|
1245
|
-
catch (
|
|
1385
|
+
catch (_a) {
|
|
1246
1386
|
// Continue to next attempt.
|
|
1247
1387
|
}
|
|
1248
1388
|
}
|
|
1249
1389
|
// All retries exhausted — the original error was already logged
|
|
1250
1390
|
// by initializeLinkTarget's catch block.
|
|
1391
|
+
return { status: LinkInitializationStatus.Failed };
|
|
1251
1392
|
}
|
|
1252
1393
|
});
|
|
1253
1394
|
}
|
|
1395
|
+
isDidResolutionFailure(error) {
|
|
1396
|
+
var _a;
|
|
1397
|
+
const message = (_a = error.message) !== null && _a !== void 0 ? _a : '';
|
|
1398
|
+
return message.includes('GetPublicKeyNotFound');
|
|
1399
|
+
}
|
|
1254
1400
|
// ---------------------------------------------------------------------------
|
|
1255
1401
|
// Hot-add / hot-remove: per-identity live sync management
|
|
1256
1402
|
// ---------------------------------------------------------------------------
|
|
@@ -1270,23 +1416,22 @@ export class SyncEngineLevel {
|
|
|
1270
1416
|
/** Hot-add a single identity to the active live sync session. */
|
|
1271
1417
|
addIdentityToLiveSync(did, options) {
|
|
1272
1418
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1273
|
-
const { protocols, delegateDid } = options;
|
|
1274
1419
|
const dwnEndpointUrls = yield this.agent.dwn.getDwnEndpointUrlsForTarget(did);
|
|
1275
1420
|
if (dwnEndpointUrls.length === 0) {
|
|
1276
|
-
return;
|
|
1421
|
+
return new Set();
|
|
1277
1422
|
}
|
|
1278
1423
|
const targets = [];
|
|
1279
1424
|
for (const dwnUrl of dwnEndpointUrls) {
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1425
|
+
targets.push(...yield this.buildSyncTargetsForEndpoint(did, dwnUrl, options));
|
|
1426
|
+
}
|
|
1427
|
+
const results = yield Promise.allSettled(targets.map(t => this.initializeLinkTargetWithRetry(t)));
|
|
1428
|
+
const currentIdentityKeys = new Set();
|
|
1429
|
+
for (const result of results) {
|
|
1430
|
+
if (result.status === 'fulfilled' && result.value.status === LinkInitializationStatus.Active) {
|
|
1431
|
+
currentIdentityKeys.add(result.value.durableLinkIdentityKey);
|
|
1287
1432
|
}
|
|
1288
1433
|
}
|
|
1289
|
-
|
|
1434
|
+
return currentIdentityKeys;
|
|
1290
1435
|
});
|
|
1291
1436
|
}
|
|
1292
1437
|
/** Hot-remove a single identity from the active live sync session. */
|
|
@@ -1361,6 +1506,40 @@ export class SyncEngineLevel {
|
|
|
1361
1506
|
this._closureContexts.delete(did);
|
|
1362
1507
|
});
|
|
1363
1508
|
}
|
|
1509
|
+
tryPruneSupersededDurableLinksForRegisteredIdentity(did, options) {
|
|
1510
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1511
|
+
try {
|
|
1512
|
+
const currentIdentityKeys = yield this.getDurableLinkIdentityKeysForRegisteredIdentity(did, options);
|
|
1513
|
+
yield this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
1514
|
+
}
|
|
1515
|
+
catch (error) {
|
|
1516
|
+
console.warn(`SyncEngineLevel: Failed to prune superseded durable links for ${did}`, error);
|
|
1517
|
+
}
|
|
1518
|
+
});
|
|
1519
|
+
}
|
|
1520
|
+
getDurableLinkIdentityKeysForRegisteredIdentity(did, options) {
|
|
1521
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1522
|
+
const scope = syncScopeFromProtocols(options.protocols);
|
|
1523
|
+
const resolutions = yield this.buildSyncTargetResolutions(did, scope, options);
|
|
1524
|
+
const keys = new Set();
|
|
1525
|
+
for (const resolution of resolutions) {
|
|
1526
|
+
const projectionId = yield computeProjectionId(did, resolution.scope);
|
|
1527
|
+
keys.add(SyncEngineLevel.durableLinkIdentityKey(did, projectionId, resolution.authorizationEpoch));
|
|
1528
|
+
}
|
|
1529
|
+
return keys;
|
|
1530
|
+
});
|
|
1531
|
+
}
|
|
1532
|
+
pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys) {
|
|
1533
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1534
|
+
const links = yield this.ledger.getLinksForTenant(did);
|
|
1535
|
+
yield Promise.all(links.map((link) => __awaiter(this, void 0, void 0, function* () {
|
|
1536
|
+
if (currentIdentityKeys.has(this.getDurableLinkIdentityKey(link))) {
|
|
1537
|
+
return;
|
|
1538
|
+
}
|
|
1539
|
+
yield this.ledger.deleteLink(link.tenantDid, link.remoteEndpoint, link.projectionId, link.authorizationEpoch);
|
|
1540
|
+
})));
|
|
1541
|
+
});
|
|
1542
|
+
}
|
|
1364
1543
|
// ---------------------------------------------------------------------------
|
|
1365
1544
|
// Live pull: MessagesSubscribe to remote DWN
|
|
1366
1545
|
// ---------------------------------------------------------------------------
|
|
@@ -1370,52 +1549,14 @@ export class SyncEngineLevel {
|
|
|
1370
1549
|
*/
|
|
1371
1550
|
openLivePullSubscription(target) {
|
|
1372
1551
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1373
|
-
|
|
1374
|
-
const
|
|
1375
|
-
// Resolve the cursor from the link's durable pull checkpoint.
|
|
1376
|
-
// Legacy syncCursors migration happens at link load time in startLiveSync().
|
|
1552
|
+
const { did, delegateDid, dwnUrl } = target;
|
|
1553
|
+
const eventScope = syncEventScope(target.scope);
|
|
1377
1554
|
const cursorKey = target.linkKey;
|
|
1378
1555
|
const link = this._activeLinks.get(cursorKey);
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
// start from the beginning rather than crash the subscription.
|
|
1383
|
-
if (cursor && (!cursor.streamId || !cursor.messageCid || !cursor.epoch || !cursor.position)) {
|
|
1384
|
-
console.warn(`SyncEngineLevel: Discarding stored cursor with empty field(s) for ${did} -> ${dwnUrl}`);
|
|
1385
|
-
cursor = undefined;
|
|
1386
|
-
if (link) {
|
|
1387
|
-
ReplicationLedger.resetCheckpoint(link.pull);
|
|
1388
|
-
yield this.ledger.saveLink(link);
|
|
1389
|
-
}
|
|
1390
|
-
}
|
|
1391
|
-
// Build the MessagesSubscribe filters.
|
|
1392
|
-
// When the link has protocolPathPrefixes, include them in the filter so the
|
|
1393
|
-
// EventLog delivers only matching events (server-side filtering). This replaces
|
|
1394
|
-
// the less efficient agent-side isEventInScope filtering for the pull path.
|
|
1395
|
-
// Note: only the first prefix is used as the MessagesFilter field because
|
|
1396
|
-
// MessagesFilter.protocolPathPrefix is a single string. Multiple prefixes
|
|
1397
|
-
// would need multiple filters (OR semantics) — for now we use the first one.
|
|
1398
|
-
const protocolPathPrefix = (link === null || link === void 0 ? void 0 : link.scope.kind) === 'protocol'
|
|
1399
|
-
? (_a = link.scope.protocolPathPrefixes) === null || _a === void 0 ? void 0 : _a[0]
|
|
1400
|
-
: undefined;
|
|
1401
|
-
const filters = protocol
|
|
1402
|
-
? [Object.assign({ protocol }, (protocolPathPrefix ? { protocolPathPrefix } : {}))]
|
|
1556
|
+
const cursor = yield this.getInitialPullCursor({ did, dwnUrl, link });
|
|
1557
|
+
const filters = target.scope.kind === 'protocolSet'
|
|
1558
|
+
? target.scope.protocols.map(protocol => ({ protocol }))
|
|
1403
1559
|
: [];
|
|
1404
|
-
// Look up permission grant for MessagesSubscribe if using a delegate.
|
|
1405
|
-
// The unified scope matching in AgentPermissionsApi accepts a
|
|
1406
|
-
// Messages.Read grant for MessagesSubscribe requests, so a single
|
|
1407
|
-
// lookup is sufficient.
|
|
1408
|
-
let permissionGrantId;
|
|
1409
|
-
if (delegateDid) {
|
|
1410
|
-
const grant = yield this._permissionsApi.getPermissionForRequest({
|
|
1411
|
-
connectedDid: did,
|
|
1412
|
-
messageType: DwnInterface.MessagesSubscribe,
|
|
1413
|
-
delegateDid,
|
|
1414
|
-
protocol,
|
|
1415
|
-
cached: true
|
|
1416
|
-
});
|
|
1417
|
-
permissionGrantId = grant.grant.id;
|
|
1418
|
-
}
|
|
1419
1560
|
const handlerGeneration = this._engineGeneration;
|
|
1420
1561
|
// Define the subscription handler that processes incoming events.
|
|
1421
1562
|
// NOTE: The WebSocket client fires handlers without awaiting (fire-and-forget),
|
|
@@ -1423,236 +1564,19 @@ export class SyncEngineLevel {
|
|
|
1423
1564
|
// ensures the checkpoint advances only when all earlier deliveries are committed.
|
|
1424
1565
|
// Capture the link reference at subscription-open time so we can
|
|
1425
1566
|
// detect remove+re-add via object identity, not just key existence.
|
|
1426
|
-
const
|
|
1427
|
-
const
|
|
1428
|
-
|
|
1429
|
-
|
|
1567
|
+
const isStale = this.createLinkStalePredicate(cursorKey, link, handlerGeneration);
|
|
1568
|
+
const pullContext = {
|
|
1569
|
+
did,
|
|
1570
|
+
dwnUrl,
|
|
1571
|
+
delegateDid,
|
|
1572
|
+
eventScope,
|
|
1573
|
+
linkKey: cursorKey,
|
|
1574
|
+
link,
|
|
1575
|
+
permissionGrantIds: target.permissionGrantIds,
|
|
1576
|
+
isStale,
|
|
1577
|
+
};
|
|
1430
1578
|
const subscriptionHandler = (subMessage) => __awaiter(this, void 0, void 0, function* () {
|
|
1431
|
-
|
|
1432
|
-
if (isStale()) {
|
|
1433
|
-
return;
|
|
1434
|
-
}
|
|
1435
|
-
if (subMessage.type === 'eose') {
|
|
1436
|
-
// End-of-stored-events — catch-up complete.
|
|
1437
|
-
if (link) {
|
|
1438
|
-
// Guard: if the link transitioned to repairing while catch-up events
|
|
1439
|
-
// were being processed, skip all mutations — repair owns the state now.
|
|
1440
|
-
if (link.status !== 'live' && link.status !== 'initializing') {
|
|
1441
|
-
return;
|
|
1442
|
-
}
|
|
1443
|
-
if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1444
|
-
console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1445
|
-
if (!isStale()) {
|
|
1446
|
-
yield this.transitionToRepairing(cursorKey, link);
|
|
1447
|
-
}
|
|
1448
|
-
return;
|
|
1449
|
-
}
|
|
1450
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1451
|
-
this.drainCommittedPull(cursorKey);
|
|
1452
|
-
if (isStale()) {
|
|
1453
|
-
return;
|
|
1454
|
-
}
|
|
1455
|
-
yield this.ledger.saveLink(link);
|
|
1456
|
-
}
|
|
1457
|
-
// Transport is reachable — set connectivity to online.
|
|
1458
|
-
if (link) {
|
|
1459
|
-
const prevEoseConnectivity = link.connectivity;
|
|
1460
|
-
link.connectivity = 'online';
|
|
1461
|
-
if (prevEoseConnectivity !== 'online') {
|
|
1462
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevEoseConnectivity, to: 'online' });
|
|
1463
|
-
}
|
|
1464
|
-
// If the link was marked dirty, schedule reconciliation now that it's healthy.
|
|
1465
|
-
if (link.needsReconcile) {
|
|
1466
|
-
this.scheduleReconcile(cursorKey, 500);
|
|
1467
|
-
}
|
|
1468
|
-
}
|
|
1469
|
-
else {
|
|
1470
|
-
this._connectivityState = 'online';
|
|
1471
|
-
}
|
|
1472
|
-
return;
|
|
1473
|
-
}
|
|
1474
|
-
if (subMessage.type === 'event') {
|
|
1475
|
-
const event = subMessage.event;
|
|
1476
|
-
// Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
|
|
1477
|
-
// skip all processing. Old subscription handlers may still fire after the
|
|
1478
|
-
// link transitions — these events should be ignored entirely, not just
|
|
1479
|
-
// skipped at the checkpoint level.
|
|
1480
|
-
if (link && link.status !== 'live' && link.status !== 'initializing') {
|
|
1481
|
-
return;
|
|
1482
|
-
}
|
|
1483
|
-
// Domain validation: reject tokens from a different stream/epoch.
|
|
1484
|
-
if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1485
|
-
console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1486
|
-
if (!isStale()) {
|
|
1487
|
-
yield this.transitionToRepairing(cursorKey, link);
|
|
1488
|
-
}
|
|
1489
|
-
return;
|
|
1490
|
-
}
|
|
1491
|
-
// Subset scope filtering: if the link has protocolPath/contextId prefixes,
|
|
1492
|
-
// skip events that don't match. This is agent-side filtering because
|
|
1493
|
-
// MessagesSubscribe only supports protocol-level filtering today.
|
|
1494
|
-
//
|
|
1495
|
-
// Skipped events MUST advance contiguousAppliedToken — otherwise the
|
|
1496
|
-
// link would replay the same filtered-out events indefinitely after
|
|
1497
|
-
// reconnect/repair. This is safe because the event is intentionally
|
|
1498
|
-
// excluded from this scope and doesn't need processing.
|
|
1499
|
-
if (link && !isEventInScope(event.message, link.scope)) {
|
|
1500
|
-
if (!isStale()) {
|
|
1501
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1502
|
-
ReplicationLedger.commitContiguousToken(link.pull, subMessage.cursor);
|
|
1503
|
-
yield this.ledger.saveLink(link);
|
|
1504
|
-
}
|
|
1505
|
-
return;
|
|
1506
|
-
}
|
|
1507
|
-
// Assign a delivery ordinal BEFORE async processing begins.
|
|
1508
|
-
// This captures the delivery order even if processing completes out of order.
|
|
1509
|
-
const rt = link ? this.getOrCreateRuntime(cursorKey) : undefined;
|
|
1510
|
-
const ordinal = rt ? rt.nextDeliveryOrdinal++ : -1;
|
|
1511
|
-
if (rt) {
|
|
1512
|
-
rt.inflight.set(ordinal, { ordinal, token: subMessage.cursor, committed: false });
|
|
1513
|
-
}
|
|
1514
|
-
try {
|
|
1515
|
-
// Extract inline data from the event (available for records <= 30 KB).
|
|
1516
|
-
let dataStream = this.extractDataStream(event);
|
|
1517
|
-
// For large RecordsWrite messages (no inline data), fetch the data
|
|
1518
|
-
// from the remote DWN via MessagesRead before storing locally.
|
|
1519
|
-
if (!dataStream && isRecordsWrite(event) && event.message.descriptor.dataCid) {
|
|
1520
|
-
const messageCid = yield Message.getCid(event.message);
|
|
1521
|
-
const fetched = yield fetchRemoteMessages({
|
|
1522
|
-
did, dwnUrl, delegateDid, protocol,
|
|
1523
|
-
messageCids: [messageCid],
|
|
1524
|
-
agent: this.agent,
|
|
1525
|
-
permissionsApi: this._permissionsApi,
|
|
1526
|
-
});
|
|
1527
|
-
if (fetched.length > 0 && fetched[0].dataStream) {
|
|
1528
|
-
dataStream = fetched[0].dataStream;
|
|
1529
|
-
}
|
|
1530
|
-
}
|
|
1531
|
-
yield this.agent.dwn.processRawMessage(did, event.message, { dataStream });
|
|
1532
|
-
if (isStale()) {
|
|
1533
|
-
return;
|
|
1534
|
-
}
|
|
1535
|
-
// Invalidate closure cache entries that may be affected by this message.
|
|
1536
|
-
// Must run before closure validation so subsequent evaluations in the
|
|
1537
|
-
// same session see the updated local state.
|
|
1538
|
-
const closureCtxForInvalidation = this._closureContexts.get(did);
|
|
1539
|
-
if (closureCtxForInvalidation) {
|
|
1540
|
-
invalidateClosureCache(closureCtxForInvalidation, event.message);
|
|
1541
|
-
}
|
|
1542
|
-
// Closure validation for scoped subset sync (Phase 3).
|
|
1543
|
-
// For protocol-scoped links, verify that all hard dependencies for
|
|
1544
|
-
// this operation are locally present before considering it committed.
|
|
1545
|
-
// Full-tenant scope bypasses this entirely (returns complete with 0 queries).
|
|
1546
|
-
if ((link === null || link === void 0 ? void 0 : link.scope.kind) === 'protocol') {
|
|
1547
|
-
const messageStore = this.agent.dwn.node.storage.messageStore;
|
|
1548
|
-
let closureCtx = this._closureContexts.get(did);
|
|
1549
|
-
if (!closureCtx) {
|
|
1550
|
-
closureCtx = createClosureContext(did, undefined, {
|
|
1551
|
-
isDelegateSession: !!delegateDid,
|
|
1552
|
-
});
|
|
1553
|
-
this._closureContexts.set(did, closureCtx);
|
|
1554
|
-
}
|
|
1555
|
-
const closureResult = yield evaluateClosure(event.message, messageStore, link.scope, closureCtx);
|
|
1556
|
-
if (isStale()) {
|
|
1557
|
-
return;
|
|
1558
|
-
}
|
|
1559
|
-
if (!closureResult.complete) {
|
|
1560
|
-
const failureCode = closureResult.failure.code;
|
|
1561
|
-
const failureDetail = closureResult.failure.detail;
|
|
1562
|
-
console.warn(`SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
|
|
1563
|
-
`${failureCode} — ${failureDetail}`);
|
|
1564
|
-
// Record the message that triggered the closure failure.
|
|
1565
|
-
const closureCid = yield Message.getCid(event.message);
|
|
1566
|
-
void this.recordDeadLetter({
|
|
1567
|
-
messageCid: closureCid,
|
|
1568
|
-
tenantDid: did,
|
|
1569
|
-
remoteEndpoint: dwnUrl,
|
|
1570
|
-
protocol,
|
|
1571
|
-
category: 'closure',
|
|
1572
|
-
errorCode: failureCode,
|
|
1573
|
-
errorDetail: failureDetail,
|
|
1574
|
-
});
|
|
1575
|
-
if (!isStale()) {
|
|
1576
|
-
yield this.transitionToRepairing(cursorKey, link);
|
|
1577
|
-
}
|
|
1578
|
-
return;
|
|
1579
|
-
}
|
|
1580
|
-
}
|
|
1581
|
-
// Squash convergence: processRawMessage triggers the DWN's built-in
|
|
1582
|
-
// squash resumable task (performRecordsSquash) which runs inline and
|
|
1583
|
-
// handles subset consumers correctly:
|
|
1584
|
-
// - If older siblings are locally present → purges them
|
|
1585
|
-
// - If squash arrives before older siblings → backstop rejects them (409)
|
|
1586
|
-
// - If no older siblings are local → no-op (correct)
|
|
1587
|
-
// Both sync orderings (squash-first or siblings-first) converge to
|
|
1588
|
-
// the same final state. No additional sync-engine side-effect is needed.
|
|
1589
|
-
// Track this CID for echo-loop suppression, scoped to the source endpoint.
|
|
1590
|
-
const pulledCid = yield Message.getCid(event.message);
|
|
1591
|
-
this._recentlyPulledCids.set(`${pulledCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
|
|
1592
|
-
this.evictExpiredEchoEntries();
|
|
1593
|
-
// Auto-clear any dead letter for this CID — it was processed
|
|
1594
|
-
// successfully, so a previous failure has been self-healed.
|
|
1595
|
-
this.clearFailedMessage(pulledCid, dwnUrl).catch(() => { });
|
|
1596
|
-
// Mark this ordinal as committed and drain the checkpoint.
|
|
1597
|
-
// Guard: if the link transitioned to repairing while this handler was
|
|
1598
|
-
// in-flight (e.g., an earlier ordinal's handler failed concurrently),
|
|
1599
|
-
// skip all state mutations — the repair process owns progression now.
|
|
1600
|
-
if (link && rt && link.status === 'live' && !isStale()) {
|
|
1601
|
-
const entry = rt.inflight.get(ordinal);
|
|
1602
|
-
if (entry) {
|
|
1603
|
-
entry.committed = true;
|
|
1604
|
-
}
|
|
1605
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1606
|
-
const drained = this.drainCommittedPull(cursorKey);
|
|
1607
|
-
if (drained > 0) {
|
|
1608
|
-
yield this.ledger.saveLink(link);
|
|
1609
|
-
// Emit after durable save — "advanced" means persisted.
|
|
1610
|
-
if (link.pull.contiguousAppliedToken) {
|
|
1611
|
-
this.emitEvent({
|
|
1612
|
-
type: 'checkpoint:pull-advance',
|
|
1613
|
-
tenantDid: link.tenantDid,
|
|
1614
|
-
remoteEndpoint: link.remoteEndpoint,
|
|
1615
|
-
protocol: link.protocol,
|
|
1616
|
-
position: link.pull.contiguousAppliedToken.position,
|
|
1617
|
-
messageCid: link.pull.contiguousAppliedToken.messageCid,
|
|
1618
|
-
});
|
|
1619
|
-
}
|
|
1620
|
-
}
|
|
1621
|
-
// Overflow: too many in-flight ordinals without draining.
|
|
1622
|
-
if (rt.inflight.size > MAX_PENDING_TOKENS) {
|
|
1623
|
-
console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1624
|
-
yield this.transitionToRepairing(cursorKey, link);
|
|
1625
|
-
}
|
|
1626
|
-
}
|
|
1627
|
-
}
|
|
1628
|
-
catch (error) {
|
|
1629
|
-
console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
|
|
1630
|
-
// Record the failing message in the dead letter store before
|
|
1631
|
-
// transitioning to repair. The CID identifies which specific
|
|
1632
|
-
// message caused the transition.
|
|
1633
|
-
try {
|
|
1634
|
-
const failedCid = yield Message.getCid(event.message);
|
|
1635
|
-
void this.recordDeadLetter({
|
|
1636
|
-
messageCid: failedCid,
|
|
1637
|
-
tenantDid: did,
|
|
1638
|
-
remoteEndpoint: dwnUrl,
|
|
1639
|
-
protocol,
|
|
1640
|
-
category: 'pull-processing',
|
|
1641
|
-
errorDetail: (_a = error.message) !== null && _a !== void 0 ? _a : String(error),
|
|
1642
|
-
});
|
|
1643
|
-
}
|
|
1644
|
-
catch (_b) {
|
|
1645
|
-
// Best effort — don't let dead letter recording block repair.
|
|
1646
|
-
}
|
|
1647
|
-
// A failed processRawMessage means local state is incomplete.
|
|
1648
|
-
// Transition to repairing immediately — do NOT advance the checkpoint
|
|
1649
|
-
// past this failure or let later ordinals commit past it. SMT
|
|
1650
|
-
// reconciliation will discover and fill the gap.
|
|
1651
|
-
if (link && !isStale()) {
|
|
1652
|
-
yield this.transitionToRepairing(cursorKey, link);
|
|
1653
|
-
}
|
|
1654
|
-
}
|
|
1655
|
-
}
|
|
1579
|
+
yield this.handleLivePullMessage(pullContext, subMessage);
|
|
1656
1580
|
});
|
|
1657
1581
|
// Construct the subscribe message and send it directly to the specific
|
|
1658
1582
|
// dwnUrl via WebSocket. We do NOT use agent.dwn.sendRequest() because
|
|
@@ -1664,7 +1588,7 @@ export class SyncEngineLevel {
|
|
|
1664
1588
|
target: did,
|
|
1665
1589
|
messageType: DwnInterface.MessagesSubscribe,
|
|
1666
1590
|
granteeDid: delegateDid,
|
|
1667
|
-
messageParams: { filters, cursor,
|
|
1591
|
+
messageParams: { filters, cursor, permissionGrantIds: toMessagesPermissionGrantIds(target.permissionGrantIds) },
|
|
1668
1592
|
};
|
|
1669
1593
|
const { message } = yield this.agent.dwn.processRequest(subscribeRequest);
|
|
1670
1594
|
if (!message) {
|
|
@@ -1710,13 +1634,14 @@ export class SyncEngineLevel {
|
|
|
1710
1634
|
if (reply.status.code !== 200 || !reply.subscription) {
|
|
1711
1635
|
throw new Error(`SyncEngineLevel: MessagesSubscribe failed for ${did} -> ${dwnUrl}: ${reply.status.code} ${reply.status.detail}`);
|
|
1712
1636
|
}
|
|
1637
|
+
const linkKey = cursorKey;
|
|
1638
|
+
const close = () => __awaiter(this, void 0, void 0, function* () { yield reply.subscription.close(); });
|
|
1713
1639
|
this._liveSubscriptions.push({
|
|
1714
|
-
linkKey
|
|
1640
|
+
linkKey,
|
|
1715
1641
|
did,
|
|
1716
1642
|
dwnUrl,
|
|
1717
1643
|
delegateDid,
|
|
1718
|
-
|
|
1719
|
-
close: () => __awaiter(this, void 0, void 0, function* () { yield reply.subscription.close(); }),
|
|
1644
|
+
close,
|
|
1720
1645
|
});
|
|
1721
1646
|
// Set per-link connectivity to online after successful subscription setup.
|
|
1722
1647
|
const pullLink = this._activeLinks.get(cursorKey);
|
|
@@ -1724,36 +1649,642 @@ export class SyncEngineLevel {
|
|
|
1724
1649
|
const prevPullConnectivity = pullLink.connectivity;
|
|
1725
1650
|
pullLink.connectivity = 'online';
|
|
1726
1651
|
if (prevPullConnectivity !== 'online') {
|
|
1727
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1652
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { from: prevPullConnectivity, to: 'online' }));
|
|
1728
1653
|
}
|
|
1729
1654
|
}
|
|
1730
1655
|
});
|
|
1731
1656
|
}
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1657
|
+
getInitialPullCursor(_a) {
|
|
1658
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, link }) {
|
|
1659
|
+
// Resolve the cursor from the link's durable pull checkpoint.
|
|
1660
|
+
if (!link) {
|
|
1661
|
+
return undefined;
|
|
1662
|
+
}
|
|
1663
|
+
const cursor = link.pull.contiguousAppliedToken;
|
|
1664
|
+
if (!cursor || this.isValidProgressToken(cursor)) {
|
|
1665
|
+
return cursor;
|
|
1666
|
+
}
|
|
1667
|
+
// Guard against corrupted tokens with empty fields — these would fail
|
|
1668
|
+
// MessagesSubscribe JSON schema validation (minLength: 1). Discard and
|
|
1669
|
+
// start from the beginning rather than crash the subscription.
|
|
1670
|
+
console.warn(`SyncEngineLevel: Discarding stored cursor with empty field(s) for ${did} -> ${dwnUrl}`);
|
|
1671
|
+
ReplicationLedger.resetCheckpoint(link.pull);
|
|
1672
|
+
yield this.ledger.saveLink(link);
|
|
1673
|
+
return undefined;
|
|
1674
|
+
});
|
|
1675
|
+
}
|
|
1676
|
+
isValidProgressToken(token) {
|
|
1677
|
+
return !!(token.streamId && token.messageCid && token.epoch && token.position);
|
|
1678
|
+
}
|
|
1679
|
+
createLinkStalePredicate(linkKey, capturedLink, generation) {
|
|
1680
|
+
return () => this._engineGeneration !== generation ||
|
|
1681
|
+
!this._activeLinks.has(linkKey) ||
|
|
1682
|
+
(capturedLink !== undefined && this._activeLinks.get(linkKey) !== capturedLink);
|
|
1683
|
+
}
|
|
1684
|
+
handleLivePullMessage(context, subMessage) {
|
|
1740
1685
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
// Build filters scoped to the protocol (if any).
|
|
1744
|
-
const filters = protocol ? [{ protocol }] : [];
|
|
1745
|
-
// Look up permission grant for local subscription.
|
|
1746
|
-
let permissionGrantId;
|
|
1747
|
-
if (delegateDid) {
|
|
1748
|
-
const grant = yield this._permissionsApi.getPermissionForRequest({
|
|
1749
|
-
connectedDid: did,
|
|
1750
|
-
messageType: DwnInterface.MessagesSubscribe,
|
|
1751
|
-
delegateDid,
|
|
1752
|
-
protocol,
|
|
1753
|
-
cached: true,
|
|
1754
|
-
});
|
|
1755
|
-
permissionGrantId = grant.grant.id;
|
|
1686
|
+
if (context.isStale()) {
|
|
1687
|
+
return;
|
|
1756
1688
|
}
|
|
1689
|
+
if (subMessage.type === 'eose') {
|
|
1690
|
+
yield this.handleLivePullEose(context, subMessage);
|
|
1691
|
+
return;
|
|
1692
|
+
}
|
|
1693
|
+
if (subMessage.type === 'error') {
|
|
1694
|
+
yield this.handleLivePullSubscriptionError(context, subMessage);
|
|
1695
|
+
return;
|
|
1696
|
+
}
|
|
1697
|
+
if (subMessage.type === 'event') {
|
|
1698
|
+
yield this.handleLivePullEvent(context, subMessage);
|
|
1699
|
+
}
|
|
1700
|
+
});
|
|
1701
|
+
}
|
|
1702
|
+
handleLivePullEose(_a, subMessage_1) {
|
|
1703
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, eventScope, linkKey, link, isStale }, subMessage) {
|
|
1704
|
+
if (link) {
|
|
1705
|
+
// Guard: if the link transitioned to repairing while catch-up events
|
|
1706
|
+
// were being processed, skip all mutations — repair owns the state now.
|
|
1707
|
+
if (link.status !== 'live' && link.status !== 'initializing') {
|
|
1708
|
+
return;
|
|
1709
|
+
}
|
|
1710
|
+
if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1711
|
+
console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1712
|
+
if (!isStale()) {
|
|
1713
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1714
|
+
}
|
|
1715
|
+
return;
|
|
1716
|
+
}
|
|
1717
|
+
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1718
|
+
this.drainCommittedPull(linkKey);
|
|
1719
|
+
if (isStale()) {
|
|
1720
|
+
return;
|
|
1721
|
+
}
|
|
1722
|
+
yield this.ledger.saveLink(link);
|
|
1723
|
+
}
|
|
1724
|
+
this.markPullLinkOnline({ did, dwnUrl, eventScope, linkKey, link });
|
|
1725
|
+
});
|
|
1726
|
+
}
|
|
1727
|
+
markPullLinkOnline({ did, dwnUrl, eventScope, linkKey, link }) {
|
|
1728
|
+
if (!link) {
|
|
1729
|
+
this._connectivityState = 'online';
|
|
1730
|
+
return;
|
|
1731
|
+
}
|
|
1732
|
+
const previous = link.connectivity;
|
|
1733
|
+
link.connectivity = 'online';
|
|
1734
|
+
if (previous !== 'online') {
|
|
1735
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { from: previous, to: 'online' }));
|
|
1736
|
+
}
|
|
1737
|
+
if (link.needsReconcile) {
|
|
1738
|
+
this.scheduleReconcile(linkKey, 500);
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1741
|
+
handleLivePullSubscriptionError(_a, subMessage_1) {
|
|
1742
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, linkKey, link, isStale }, subMessage) {
|
|
1743
|
+
console.warn(`SyncEngineLevel: subscription error for ${did} -> ${dwnUrl}: ${subMessage.error.code}`);
|
|
1744
|
+
if (link && !isStale()) {
|
|
1745
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1746
|
+
}
|
|
1747
|
+
});
|
|
1748
|
+
}
|
|
1749
|
+
handleLivePullEvent(context, subMessage) {
|
|
1750
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1751
|
+
const event = subMessage.event;
|
|
1752
|
+
if (yield this.shouldSkipLivePullEvent(context, subMessage)) {
|
|
1753
|
+
return;
|
|
1754
|
+
}
|
|
1755
|
+
const delivery = this.startPullDelivery(context, subMessage.cursor);
|
|
1756
|
+
try {
|
|
1757
|
+
const pulledCid = yield this.processLivePullEvent(context, event);
|
|
1758
|
+
if (!pulledCid) {
|
|
1759
|
+
return;
|
|
1760
|
+
}
|
|
1761
|
+
this.trackRecentlyPulledMessage(pulledCid, context.dwnUrl);
|
|
1762
|
+
this.clearFailedMessage(pulledCid, context.dwnUrl).catch(() => { });
|
|
1763
|
+
yield this.commitPullDelivery(context, subMessage.cursor, delivery);
|
|
1764
|
+
}
|
|
1765
|
+
catch (error) {
|
|
1766
|
+
yield this.handleLivePullProcessingError(context, event, error);
|
|
1767
|
+
}
|
|
1768
|
+
});
|
|
1769
|
+
}
|
|
1770
|
+
shouldSkipLivePullEvent(_a, subMessage_1) {
|
|
1771
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, linkKey, link, isStale }, subMessage) {
|
|
1772
|
+
// Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
|
|
1773
|
+
// skip all processing. Old subscription handlers may still fire after the
|
|
1774
|
+
// link transitions — these events should be ignored entirely, not just
|
|
1775
|
+
// skipped at the checkpoint level.
|
|
1776
|
+
if (link && link.status !== 'live' && link.status !== 'initializing') {
|
|
1777
|
+
return true;
|
|
1778
|
+
}
|
|
1779
|
+
// Domain validation: reject tokens from a different stream/epoch.
|
|
1780
|
+
if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1781
|
+
console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1782
|
+
if (!isStale()) {
|
|
1783
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1784
|
+
}
|
|
1785
|
+
return true;
|
|
1786
|
+
}
|
|
1787
|
+
if (link) {
|
|
1788
|
+
const scopeClassification = classifySyncEventScope(subMessage.event, link.scope);
|
|
1789
|
+
if (scopeClassification === 'out-of-scope') {
|
|
1790
|
+
yield this.skipOutOfScopePullEvent({ link, cursor: subMessage.cursor, isStale });
|
|
1791
|
+
return true;
|
|
1792
|
+
}
|
|
1793
|
+
if (scopeClassification === 'unknown') {
|
|
1794
|
+
console.warn(`SyncEngineLevel: Unable to classify scoped pull event for ${did} -> ${dwnUrl}, transitioning to repair`);
|
|
1795
|
+
if (!isStale()) {
|
|
1796
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1797
|
+
}
|
|
1798
|
+
return true;
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
return false;
|
|
1802
|
+
});
|
|
1803
|
+
}
|
|
1804
|
+
skipOutOfScopePullEvent(_a) {
|
|
1805
|
+
return __awaiter(this, arguments, void 0, function* ({ link, cursor, isStale }) {
|
|
1806
|
+
// Skipped events MUST advance contiguousAppliedToken — otherwise the link
|
|
1807
|
+
// would replay the same filtered-out events indefinitely after reconnect or
|
|
1808
|
+
// repair. This is safe because the event is intentionally excluded from
|
|
1809
|
+
// this scope and doesn't need processing.
|
|
1810
|
+
if (isStale()) {
|
|
1811
|
+
return;
|
|
1812
|
+
}
|
|
1813
|
+
ReplicationLedger.setReceivedToken(link.pull, cursor);
|
|
1814
|
+
ReplicationLedger.commitContiguousToken(link.pull, cursor);
|
|
1815
|
+
yield this.ledger.saveLink(link);
|
|
1816
|
+
});
|
|
1817
|
+
}
|
|
1818
|
+
startPullDelivery({ linkKey, link }, cursor) {
|
|
1819
|
+
// Assign a delivery ordinal BEFORE async processing begins. This captures
|
|
1820
|
+
// delivery order even if processing completes out of order.
|
|
1821
|
+
const runtime = link ? this.getOrCreateRuntime(linkKey) : undefined;
|
|
1822
|
+
const ordinal = runtime ? runtime.nextDeliveryOrdinal++ : -1;
|
|
1823
|
+
if (runtime) {
|
|
1824
|
+
runtime.inflight.set(ordinal, { ordinal, token: cursor, committed: false });
|
|
1825
|
+
}
|
|
1826
|
+
return { runtime, ordinal };
|
|
1827
|
+
}
|
|
1828
|
+
processLivePullEvent(context, event) {
|
|
1829
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1830
|
+
const dataStreamFactory = yield this.createLivePullDataStreamFactory(context, event);
|
|
1831
|
+
let applyStatus = yield this.applyLivePullEvent(context, event, dataStreamFactory);
|
|
1832
|
+
if (context.isStale()) {
|
|
1833
|
+
return undefined;
|
|
1834
|
+
}
|
|
1835
|
+
let applied = SyncEngineLevel.isApplySuccess(applyStatus.code);
|
|
1836
|
+
if (applied) {
|
|
1837
|
+
this.invalidateClosureCacheForMessage(context.did, event.message);
|
|
1838
|
+
}
|
|
1839
|
+
if (!(yield this.ensureClosureComplete(context, event))) {
|
|
1840
|
+
return undefined;
|
|
1841
|
+
}
|
|
1842
|
+
if (!applied) {
|
|
1843
|
+
applyStatus = yield this.applyLivePullEvent(context, event, dataStreamFactory);
|
|
1844
|
+
if (context.isStale()) {
|
|
1845
|
+
return undefined;
|
|
1846
|
+
}
|
|
1847
|
+
applied = SyncEngineLevel.isApplySuccess(applyStatus.code);
|
|
1848
|
+
if (!applied) {
|
|
1849
|
+
throw yield this.createLivePullApplyError(event, applyStatus);
|
|
1850
|
+
}
|
|
1851
|
+
this.invalidateClosureCacheForMessage(context.did, event.message);
|
|
1852
|
+
}
|
|
1853
|
+
// Squash convergence: processRawMessage triggers the DWN's built-in
|
|
1854
|
+
// squash resumable task (performRecordsSquash), so no additional
|
|
1855
|
+
// sync-engine side effect is needed here.
|
|
1856
|
+
return Message.getCid(event.message);
|
|
1857
|
+
});
|
|
1858
|
+
}
|
|
1859
|
+
applyLivePullEvent(context, event, dataStreamFactory) {
|
|
1860
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1861
|
+
const dataStream = yield dataStreamFactory();
|
|
1862
|
+
const reply = yield this.agent.dwn.processRawMessage(context.did, event.message, { dataStream });
|
|
1863
|
+
return reply.status;
|
|
1864
|
+
});
|
|
1865
|
+
}
|
|
1866
|
+
createLivePullDataStreamFactory(context, event) {
|
|
1867
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1868
|
+
if (!isRecordsWrite(event)) {
|
|
1869
|
+
return () => __awaiter(this, void 0, void 0, function* () { return undefined; });
|
|
1870
|
+
}
|
|
1871
|
+
const encodedData = event.message.encodedData;
|
|
1872
|
+
if (encodedData) {
|
|
1873
|
+
delete event.message.encodedData;
|
|
1874
|
+
const bytes = Encoder.base64UrlToBytes(encodedData);
|
|
1875
|
+
return () => __awaiter(this, void 0, void 0, function* () { return SyncEngineLevel.dataStreamFromBytes(bytes); });
|
|
1876
|
+
}
|
|
1877
|
+
const eventData = event.data;
|
|
1878
|
+
if (eventData) {
|
|
1879
|
+
const bytes = yield SyncEngineLevel.readStreamBytes(eventData);
|
|
1880
|
+
return () => __awaiter(this, void 0, void 0, function* () { return SyncEngineLevel.dataStreamFromBytes(bytes); });
|
|
1881
|
+
}
|
|
1882
|
+
if (!event.message.descriptor.dataCid) {
|
|
1883
|
+
return () => __awaiter(this, void 0, void 0, function* () { return undefined; });
|
|
1884
|
+
}
|
|
1885
|
+
// For large RecordsWrite messages (no inline data), fetch the data from
|
|
1886
|
+
// the remote DWN via MessagesRead before each store attempt. ReadableStream
|
|
1887
|
+
// instances are single-use, so a repair-triggered retry needs a fresh fetch.
|
|
1888
|
+
const { did, dwnUrl, delegateDid, permissionGrantIds } = context;
|
|
1889
|
+
const messageCid = yield Message.getCid(event.message);
|
|
1890
|
+
return () => __awaiter(this, void 0, void 0, function* () {
|
|
1891
|
+
var _a;
|
|
1892
|
+
const fetched = yield fetchRemoteMessages({
|
|
1893
|
+
did,
|
|
1894
|
+
dwnUrl,
|
|
1895
|
+
delegateDid,
|
|
1896
|
+
permissionGrantIds,
|
|
1897
|
+
messageCids: [messageCid],
|
|
1898
|
+
agent: this.agent,
|
|
1899
|
+
});
|
|
1900
|
+
return (_a = fetched[0]) === null || _a === void 0 ? void 0 : _a.dataStream;
|
|
1901
|
+
});
|
|
1902
|
+
});
|
|
1903
|
+
}
|
|
1904
|
+
createLivePullApplyError(event, status) {
|
|
1905
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1906
|
+
var _a;
|
|
1907
|
+
const cid = yield Message.getCid(event.message);
|
|
1908
|
+
return new Error(`SyncEngineLevel: live pull apply failed for ${cid}: ${status.code} ${(_a = status.detail) !== null && _a !== void 0 ? _a : ''}`.trim());
|
|
1909
|
+
});
|
|
1910
|
+
}
|
|
1911
|
+
static dataStreamFromBytes(bytes) {
|
|
1912
|
+
return new ReadableStream({
|
|
1913
|
+
start(controller) {
|
|
1914
|
+
controller.enqueue(bytes);
|
|
1915
|
+
controller.close();
|
|
1916
|
+
}
|
|
1917
|
+
});
|
|
1918
|
+
}
|
|
1919
|
+
static readStreamBytes(stream) {
|
|
1920
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1921
|
+
const reader = stream.getReader();
|
|
1922
|
+
const chunks = [];
|
|
1923
|
+
let totalSize = 0;
|
|
1924
|
+
try {
|
|
1925
|
+
for (;;) {
|
|
1926
|
+
const { done, value } = yield reader.read();
|
|
1927
|
+
if (done) {
|
|
1928
|
+
break;
|
|
1929
|
+
}
|
|
1930
|
+
chunks.push(value);
|
|
1931
|
+
totalSize += value.byteLength;
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1934
|
+
finally {
|
|
1935
|
+
reader.releaseLock();
|
|
1936
|
+
}
|
|
1937
|
+
const bytes = new Uint8Array(totalSize);
|
|
1938
|
+
let offset = 0;
|
|
1939
|
+
for (const chunk of chunks) {
|
|
1940
|
+
bytes.set(chunk, offset);
|
|
1941
|
+
offset += chunk.byteLength;
|
|
1942
|
+
}
|
|
1943
|
+
return bytes;
|
|
1944
|
+
});
|
|
1945
|
+
}
|
|
1946
|
+
static isApplySuccess(code) {
|
|
1947
|
+
return (code >= 200 && code < 300) || code === 409;
|
|
1948
|
+
}
|
|
1949
|
+
invalidateClosureCacheForMessage(did, message) {
|
|
1950
|
+
// Must run before closure validation so subsequent evaluations in the same
|
|
1951
|
+
// session see the updated local state.
|
|
1952
|
+
const closureCtx = this._closureContexts.get(did);
|
|
1953
|
+
if (closureCtx) {
|
|
1954
|
+
invalidateClosureCache(closureCtx, message);
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
ensureClosureComplete(context, event) {
|
|
1958
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1959
|
+
const { did, delegateDid, link, isStale } = context;
|
|
1960
|
+
if (!link || link.scope.kind === 'full') {
|
|
1961
|
+
return true;
|
|
1962
|
+
}
|
|
1963
|
+
let closureCtx = this._closureContexts.get(did);
|
|
1964
|
+
if (!closureCtx) {
|
|
1965
|
+
closureCtx = createClosureContext(did, undefined, {
|
|
1966
|
+
isDelegateSession: !!delegateDid,
|
|
1967
|
+
});
|
|
1968
|
+
this._closureContexts.set(did, closureCtx);
|
|
1969
|
+
}
|
|
1970
|
+
const messageStore = this.agent.dwn.node.storage.messageStore;
|
|
1971
|
+
let closureResult = yield evaluateClosure(event.message, messageStore, link.scope, closureCtx);
|
|
1972
|
+
if (isStale()) {
|
|
1973
|
+
return false;
|
|
1974
|
+
}
|
|
1975
|
+
if (closureResult.complete) {
|
|
1976
|
+
return true;
|
|
1977
|
+
}
|
|
1978
|
+
if (yield this.tryRepairMissingProtocolMetadata(context, closureCtx, closureResult)) {
|
|
1979
|
+
if (isStale()) {
|
|
1980
|
+
return false;
|
|
1981
|
+
}
|
|
1982
|
+
closureResult = yield evaluateClosure(event.message, messageStore, link.scope, closureCtx);
|
|
1983
|
+
if (isStale()) {
|
|
1984
|
+
return false;
|
|
1985
|
+
}
|
|
1986
|
+
if (closureResult.complete) {
|
|
1987
|
+
return true;
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
yield this.recordClosureFailure(context, event, closureResult.failure.code, closureResult.failure.detail);
|
|
1991
|
+
return false;
|
|
1992
|
+
});
|
|
1993
|
+
}
|
|
1994
|
+
tryRepairMissingProtocolMetadata(context, closureCtx, closureResult) {
|
|
1995
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1996
|
+
const failure = closureResult.failure;
|
|
1997
|
+
if (!SyncEngineLevel.isRepairableProtocolMetadataFailure(failure)) {
|
|
1998
|
+
return false;
|
|
1999
|
+
}
|
|
2000
|
+
const { did } = context;
|
|
2001
|
+
const repairKey = `${did}|${failure.edge.identifier}`;
|
|
2002
|
+
const activeRepair = this._protocolMetadataRepairs.get(repairKey);
|
|
2003
|
+
if (activeRepair) {
|
|
2004
|
+
return activeRepair;
|
|
2005
|
+
}
|
|
2006
|
+
const repair = this.repairMissingProtocolMetadata(context, closureCtx, failure.edge.identifier);
|
|
2007
|
+
this._protocolMetadataRepairs.set(repairKey, repair);
|
|
2008
|
+
repair.finally(() => {
|
|
2009
|
+
if (this._protocolMetadataRepairs.get(repairKey) === repair) {
|
|
2010
|
+
this._protocolMetadataRepairs.delete(repairKey);
|
|
2011
|
+
}
|
|
2012
|
+
}).catch(() => { });
|
|
2013
|
+
return repair;
|
|
2014
|
+
});
|
|
2015
|
+
}
|
|
2016
|
+
repairMissingProtocolMetadata(_a, closureCtx_1, protocol_1) {
|
|
2017
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, isStale }, closureCtx, protocol) {
|
|
2018
|
+
if (isStale()) {
|
|
2019
|
+
return false;
|
|
2020
|
+
}
|
|
2021
|
+
const configs = yield this.fetchRemoteProtocolConfigClosure({
|
|
2022
|
+
authorDid: delegateDid !== null && delegateDid !== void 0 ? delegateDid : did,
|
|
2023
|
+
delegateDid,
|
|
2024
|
+
dwnUrl,
|
|
2025
|
+
protocol,
|
|
2026
|
+
tenantDid: did,
|
|
2027
|
+
});
|
|
2028
|
+
if (isStale() || configs.length === 0) {
|
|
2029
|
+
return false;
|
|
2030
|
+
}
|
|
2031
|
+
// Live subscriptions can deliver scoped records before the local replica
|
|
2032
|
+
// has the tenant's protocol metadata. Reuse the DWN ProtocolsQuery path and
|
|
2033
|
+
// only install configs that are signed by the tenant, including composed
|
|
2034
|
+
// protocol dependencies needed to authorize the record.
|
|
2035
|
+
let repaired = false;
|
|
2036
|
+
for (const config of configs) {
|
|
2037
|
+
if (isStale()) {
|
|
2038
|
+
return repaired;
|
|
2039
|
+
}
|
|
2040
|
+
const reply = yield this.agent.dwn.processRawMessage(did, config);
|
|
2041
|
+
if (isStale()) {
|
|
2042
|
+
return repaired;
|
|
2043
|
+
}
|
|
2044
|
+
if (!SyncEngineLevel.protocolConfigApplySucceeded(reply.status.code)) {
|
|
2045
|
+
return repaired;
|
|
2046
|
+
}
|
|
2047
|
+
invalidateClosureCache(closureCtx, config);
|
|
2048
|
+
repaired = true;
|
|
2049
|
+
}
|
|
2050
|
+
return repaired;
|
|
2051
|
+
});
|
|
2052
|
+
}
|
|
2053
|
+
fetchRemoteProtocolConfigClosure(_a) {
|
|
2054
|
+
return __awaiter(this, arguments, void 0, function* ({ authorDid, delegateDid, dwnUrl, protocol, tenantDid, }) {
|
|
2055
|
+
const configsByProtocol = new Map();
|
|
2056
|
+
const visiting = new Set();
|
|
2057
|
+
const visit = (protocolUri) => __awaiter(this, void 0, void 0, function* () {
|
|
2058
|
+
if (configsByProtocol.has(protocolUri)) {
|
|
2059
|
+
return true;
|
|
2060
|
+
}
|
|
2061
|
+
if (visiting.has(protocolUri)) {
|
|
2062
|
+
return true;
|
|
2063
|
+
}
|
|
2064
|
+
visiting.add(protocolUri);
|
|
2065
|
+
const config = yield this.fetchRemoteProtocolConfig({
|
|
2066
|
+
authorDid,
|
|
2067
|
+
delegateDid,
|
|
2068
|
+
dwnUrl,
|
|
2069
|
+
protocol: protocolUri,
|
|
2070
|
+
tenantDid,
|
|
2071
|
+
});
|
|
2072
|
+
if (config === undefined) {
|
|
2073
|
+
visiting.delete(protocolUri);
|
|
2074
|
+
return false;
|
|
2075
|
+
}
|
|
2076
|
+
for (const usedProtocol of SyncEngineLevel.protocolsConfigureUses(config)) {
|
|
2077
|
+
if (!(yield visit(usedProtocol))) {
|
|
2078
|
+
visiting.delete(protocolUri);
|
|
2079
|
+
return false;
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
configsByProtocol.set(protocolUri, config);
|
|
2083
|
+
visiting.delete(protocolUri);
|
|
2084
|
+
return true;
|
|
2085
|
+
});
|
|
2086
|
+
return (yield visit(protocol)) ? [...configsByProtocol.values()] : [];
|
|
2087
|
+
});
|
|
2088
|
+
}
|
|
2089
|
+
fetchRemoteProtocolConfig(_a) {
|
|
2090
|
+
return __awaiter(this, arguments, void 0, function* ({ authorDid, delegateDid, dwnUrl, protocol, tenantDid, }) {
|
|
2091
|
+
try {
|
|
2092
|
+
const permissionGrantId = yield this.getProtocolsQueryPermissionGrantId({
|
|
2093
|
+
delegateDid,
|
|
2094
|
+
protocol,
|
|
2095
|
+
tenantDid,
|
|
2096
|
+
});
|
|
2097
|
+
const { message } = yield this.agent.processDwnRequest({
|
|
2098
|
+
author: authorDid,
|
|
2099
|
+
messageParams: Object.assign({ filter: { protocol } }, (permissionGrantId === undefined ? {} : { permissionGrantId })),
|
|
2100
|
+
messageType: DwnInterface.ProtocolsQuery,
|
|
2101
|
+
store: false,
|
|
2102
|
+
target: tenantDid,
|
|
2103
|
+
});
|
|
2104
|
+
const reply = yield this.agent.rpc.sendDwnRequest({
|
|
2105
|
+
dwnUrl,
|
|
2106
|
+
message,
|
|
2107
|
+
targetDid: tenantDid,
|
|
2108
|
+
});
|
|
2109
|
+
if (reply.status.code !== 200 || reply.entries === undefined) {
|
|
2110
|
+
return undefined;
|
|
2111
|
+
}
|
|
2112
|
+
const candidates = [];
|
|
2113
|
+
for (const entry of reply.entries) {
|
|
2114
|
+
const config = yield this.toAuthenticatedTenantProtocolConfig(tenantDid, entry);
|
|
2115
|
+
if ((config === null || config === void 0 ? void 0 : config.descriptor.definition.protocol) === protocol) {
|
|
2116
|
+
candidates.push(config);
|
|
2117
|
+
}
|
|
2118
|
+
}
|
|
2119
|
+
return SyncEngineLevel.newestProtocolConfig(candidates);
|
|
2120
|
+
}
|
|
2121
|
+
catch (_b) {
|
|
2122
|
+
return undefined;
|
|
2123
|
+
}
|
|
2124
|
+
});
|
|
2125
|
+
}
|
|
2126
|
+
getProtocolsQueryPermissionGrantId(_a) {
|
|
2127
|
+
return __awaiter(this, arguments, void 0, function* ({ delegateDid, protocol, tenantDid, }) {
|
|
2128
|
+
if (delegateDid === undefined) {
|
|
2129
|
+
return undefined;
|
|
2130
|
+
}
|
|
2131
|
+
try {
|
|
2132
|
+
const { grant } = yield this._permissionsApi.getPermissionForRequest({
|
|
2133
|
+
connectedDid: tenantDid,
|
|
2134
|
+
delegateDid,
|
|
2135
|
+
protocol,
|
|
2136
|
+
cached: true,
|
|
2137
|
+
messageType: DwnInterface.ProtocolsQuery,
|
|
2138
|
+
});
|
|
2139
|
+
return grant.id;
|
|
2140
|
+
}
|
|
2141
|
+
catch (_b) {
|
|
2142
|
+
return undefined;
|
|
2143
|
+
}
|
|
2144
|
+
});
|
|
2145
|
+
}
|
|
2146
|
+
toAuthenticatedTenantProtocolConfig(tenantDid, message) {
|
|
2147
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2148
|
+
if (!SyncEngineLevel.isProtocolsConfigureDefinitionMessage(message)) {
|
|
2149
|
+
return undefined;
|
|
2150
|
+
}
|
|
2151
|
+
try {
|
|
2152
|
+
yield ProtocolsConfigure.parse(message);
|
|
2153
|
+
if (Message.getAuthor(message) !== tenantDid) {
|
|
2154
|
+
return undefined;
|
|
2155
|
+
}
|
|
2156
|
+
yield authenticate(message.authorization, this.agent.did);
|
|
2157
|
+
return message;
|
|
2158
|
+
}
|
|
2159
|
+
catch (_a) {
|
|
2160
|
+
return undefined;
|
|
2161
|
+
}
|
|
2162
|
+
});
|
|
2163
|
+
}
|
|
2164
|
+
static newestProtocolConfig(configs) {
|
|
2165
|
+
let newest;
|
|
2166
|
+
for (const config of configs) {
|
|
2167
|
+
if (newest === undefined || SyncEngineLevel.isProtocolConfigNewer(config, newest)) {
|
|
2168
|
+
newest = config;
|
|
2169
|
+
}
|
|
2170
|
+
}
|
|
2171
|
+
return newest;
|
|
2172
|
+
}
|
|
2173
|
+
static isProtocolConfigNewer(candidate, current) {
|
|
2174
|
+
return candidate.descriptor.messageTimestamp > current.descriptor.messageTimestamp;
|
|
2175
|
+
}
|
|
2176
|
+
static protocolConfigApplySucceeded(code) {
|
|
2177
|
+
return (code >= 200 && code < 300) || code === 409;
|
|
2178
|
+
}
|
|
2179
|
+
static isRepairableProtocolMetadataFailure(failure) {
|
|
2180
|
+
return (failure === null || failure === void 0 ? void 0 : failure.edge.identifierType) === 'protocol' &&
|
|
2181
|
+
(failure.code === ClosureFailureCode.ProtocolMetadataMissing ||
|
|
2182
|
+
failure.code === ClosureFailureCode.CrossProtocolReferenceMissing ||
|
|
2183
|
+
failure.code === ClosureFailureCode.EncryptionDependencyMissing);
|
|
2184
|
+
}
|
|
2185
|
+
recordClosureFailure(_a, event_1, failureCode_1, failureDetail_1) {
|
|
2186
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, linkKey, link, isStale }, event, failureCode, failureDetail) {
|
|
2187
|
+
console.warn(`SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
|
|
2188
|
+
`${failureCode} — ${failureDetail}`);
|
|
2189
|
+
const closureCid = yield Message.getCid(event.message);
|
|
2190
|
+
void this.recordDeadLetter({
|
|
2191
|
+
messageCid: closureCid,
|
|
2192
|
+
tenantDid: did,
|
|
2193
|
+
remoteEndpoint: dwnUrl,
|
|
2194
|
+
protocol: event.message.descriptor.protocol,
|
|
2195
|
+
category: 'closure',
|
|
2196
|
+
errorCode: failureCode,
|
|
2197
|
+
errorDetail: failureDetail,
|
|
2198
|
+
});
|
|
2199
|
+
if (link && !isStale() && isTerminalClosureFailureCode(failureCode)) {
|
|
2200
|
+
yield this.transitionToTerminalIncomplete(linkKey, link);
|
|
2201
|
+
return;
|
|
2202
|
+
}
|
|
2203
|
+
if (link && !isStale()) {
|
|
2204
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
2205
|
+
}
|
|
2206
|
+
});
|
|
2207
|
+
}
|
|
2208
|
+
trackRecentlyPulledMessage(messageCid, dwnUrl) {
|
|
2209
|
+
this._recentlyPulledCids.set(`${messageCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
|
|
2210
|
+
this.evictExpiredEchoEntries();
|
|
2211
|
+
}
|
|
2212
|
+
commitPullDelivery(_a, cursor_1, delivery_1) {
|
|
2213
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, linkKey, link, isStale }, cursor, delivery) {
|
|
2214
|
+
// Guard: if the link transitioned to repairing while this handler was
|
|
2215
|
+
// in-flight, skip all state mutations — the repair process owns progression.
|
|
2216
|
+
if (!link || !delivery.runtime || link.status !== 'live' || isStale()) {
|
|
2217
|
+
return;
|
|
2218
|
+
}
|
|
2219
|
+
const entry = delivery.runtime.inflight.get(delivery.ordinal);
|
|
2220
|
+
if (entry) {
|
|
2221
|
+
entry.committed = true;
|
|
2222
|
+
}
|
|
2223
|
+
ReplicationLedger.setReceivedToken(link.pull, cursor);
|
|
2224
|
+
const drained = this.drainCommittedPull(linkKey);
|
|
2225
|
+
if (drained > 0) {
|
|
2226
|
+
yield this.ledger.saveLink(link);
|
|
2227
|
+
this.emitPullCheckpointAdvance(link);
|
|
2228
|
+
}
|
|
2229
|
+
if (delivery.runtime.inflight.size > MAX_PENDING_TOKENS) {
|
|
2230
|
+
console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
2231
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
2232
|
+
}
|
|
2233
|
+
});
|
|
2234
|
+
}
|
|
2235
|
+
emitPullCheckpointAdvance(link) {
|
|
2236
|
+
if (!link.pull.contiguousAppliedToken) {
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
// Emit after durable save — "advanced" means persisted.
|
|
2240
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'checkpoint:pull-advance', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, syncEventScope(link.scope)), { position: link.pull.contiguousAppliedToken.position, messageCid: link.pull.contiguousAppliedToken.messageCid }));
|
|
2241
|
+
}
|
|
2242
|
+
handleLivePullProcessingError(_a, event_1, error_1) {
|
|
2243
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, linkKey, link, isStale }, event, error) {
|
|
2244
|
+
console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
|
|
2245
|
+
yield this.recordPullProcessingFailure({ did, dwnUrl, event, error });
|
|
2246
|
+
// A failed processRawMessage means local state is incomplete. Transition
|
|
2247
|
+
// to repairing immediately — do NOT advance the checkpoint past this
|
|
2248
|
+
// failure or let later ordinals commit past it. SMT reconciliation will
|
|
2249
|
+
// discover and fill the gap.
|
|
2250
|
+
if (link && !isStale()) {
|
|
2251
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
2252
|
+
}
|
|
2253
|
+
});
|
|
2254
|
+
}
|
|
2255
|
+
recordPullProcessingFailure(_a) {
|
|
2256
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, event, error }) {
|
|
2257
|
+
var _b;
|
|
2258
|
+
try {
|
|
2259
|
+
const failedCid = yield Message.getCid(event.message);
|
|
2260
|
+
void this.recordDeadLetter({
|
|
2261
|
+
messageCid: failedCid,
|
|
2262
|
+
tenantDid: did,
|
|
2263
|
+
remoteEndpoint: dwnUrl,
|
|
2264
|
+
protocol: event.message.descriptor.protocol,
|
|
2265
|
+
category: 'pull-processing',
|
|
2266
|
+
errorDetail: (_b = error.message) !== null && _b !== void 0 ? _b : String(error),
|
|
2267
|
+
});
|
|
2268
|
+
}
|
|
2269
|
+
catch (_c) {
|
|
2270
|
+
// Best effort — don't let dead letter recording block repair.
|
|
2271
|
+
}
|
|
2272
|
+
});
|
|
2273
|
+
}
|
|
2274
|
+
// ---------------------------------------------------------------------------
|
|
2275
|
+
// Live push: local EventLog subscription for immediate push
|
|
2276
|
+
// ---------------------------------------------------------------------------
|
|
2277
|
+
/**
|
|
2278
|
+
* Subscribes to the local DWN's EventLog so that writes by the user are
|
|
2279
|
+
* immediately pushed to the remote DWN instead of waiting for the next poll.
|
|
2280
|
+
*/
|
|
2281
|
+
openLocalPushSubscription(target) {
|
|
2282
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2283
|
+
const { did, delegateDid, dwnUrl } = target;
|
|
2284
|
+
const protocol = singleProtocolForSyncScope(target.scope);
|
|
2285
|
+
const filters = target.scope.kind === 'protocolSet'
|
|
2286
|
+
? target.scope.protocols.map(protocol => ({ protocol }))
|
|
2287
|
+
: [];
|
|
1757
2288
|
const handlerGeneration = this._engineGeneration;
|
|
1758
2289
|
// Capture the link for identity-based staleness detection.
|
|
1759
2290
|
const capturedPushLink = this._activeLinks.get(target.linkKey);
|
|
@@ -1768,12 +2299,19 @@ export class SyncEngineLevel {
|
|
|
1768
2299
|
if (subMessage.type !== 'event') {
|
|
1769
2300
|
return;
|
|
1770
2301
|
}
|
|
1771
|
-
// Subset scope filtering: only push events that match the link
|
|
1772
|
-
//
|
|
2302
|
+
// Subset scope filtering: only push events that match the link scope.
|
|
2303
|
+
// Events outside the scope are not this link's responsibility.
|
|
1773
2304
|
const pushLinkKey = target.linkKey;
|
|
1774
2305
|
const pushLink = this._activeLinks.get(pushLinkKey);
|
|
1775
|
-
if (pushLink
|
|
1776
|
-
|
|
2306
|
+
if (pushLink) {
|
|
2307
|
+
const scopeClassification = classifySyncEventScope(subMessage.event, pushLink.scope);
|
|
2308
|
+
if (scopeClassification === 'out-of-scope') {
|
|
2309
|
+
return;
|
|
2310
|
+
}
|
|
2311
|
+
if (scopeClassification === 'unknown') {
|
|
2312
|
+
this.markLinkNeedsReconcile(pushLinkKey, pushLink, 'push-scope-unclassified');
|
|
2313
|
+
return;
|
|
2314
|
+
}
|
|
1777
2315
|
}
|
|
1778
2316
|
// Accumulate the message CID for a debounced push.
|
|
1779
2317
|
const targetKey = pushLinkKey;
|
|
@@ -1788,7 +2326,11 @@ export class SyncEngineLevel {
|
|
|
1788
2326
|
return;
|
|
1789
2327
|
}
|
|
1790
2328
|
const pushRuntime = this.getOrCreatePushRuntime(targetKey, {
|
|
1791
|
-
did,
|
|
2329
|
+
did,
|
|
2330
|
+
dwnUrl,
|
|
2331
|
+
delegateDid,
|
|
2332
|
+
protocol,
|
|
2333
|
+
permissionGrantIds: target.permissionGrantIds,
|
|
1792
2334
|
});
|
|
1793
2335
|
pushRuntime.entries.push({ cid });
|
|
1794
2336
|
// Immediate-first: if no push is in flight and no batch timer is
|
|
@@ -1806,20 +2348,20 @@ export class SyncEngineLevel {
|
|
|
1806
2348
|
target: did,
|
|
1807
2349
|
messageType: DwnInterface.MessagesSubscribe,
|
|
1808
2350
|
granteeDid: delegateDid,
|
|
1809
|
-
messageParams: { filters,
|
|
2351
|
+
messageParams: { filters, permissionGrantIds: toMessagesPermissionGrantIds(target.permissionGrantIds) },
|
|
1810
2352
|
subscriptionHandler: subscriptionHandler,
|
|
1811
2353
|
});
|
|
1812
2354
|
const reply = response.reply;
|
|
1813
2355
|
if (reply.status.code !== 200 || !reply.subscription) {
|
|
1814
2356
|
throw new Error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
|
|
1815
2357
|
}
|
|
2358
|
+
const close = () => __awaiter(this, void 0, void 0, function* () { yield reply.subscription.close(); });
|
|
1816
2359
|
this._localSubscriptions.push({
|
|
1817
|
-
linkKey:
|
|
2360
|
+
linkKey: target.linkKey,
|
|
1818
2361
|
did,
|
|
1819
2362
|
dwnUrl,
|
|
1820
2363
|
delegateDid,
|
|
1821
|
-
|
|
1822
|
-
close: () => __awaiter(this, void 0, void 0, function* () { yield reply.subscription.close(); }),
|
|
2364
|
+
close,
|
|
1823
2365
|
});
|
|
1824
2366
|
});
|
|
1825
2367
|
}
|
|
@@ -1835,108 +2377,148 @@ export class SyncEngineLevel {
|
|
|
1835
2377
|
}
|
|
1836
2378
|
flushPendingPushesForLink(linkKey) {
|
|
1837
2379
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
// debounce timer or retry callback could send pushes after the DID
|
|
1841
|
-
// was removed.
|
|
1842
|
-
if (!this._activeLinks.has(linkKey)) {
|
|
1843
|
-
return;
|
|
1844
|
-
}
|
|
1845
|
-
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
1846
|
-
if (!pushRuntime) {
|
|
1847
|
-
return;
|
|
1848
|
-
}
|
|
1849
|
-
// Capture the current active link identity so we can detect
|
|
1850
|
-
// remove+re-add during the await pushMessages() call.
|
|
1851
|
-
const flushLink = this._activeLinks.get(linkKey);
|
|
1852
|
-
const isFlushStale = () => !this._activeLinks.has(linkKey) ||
|
|
1853
|
-
(flushLink !== undefined && this._activeLinks.get(linkKey) !== flushLink);
|
|
1854
|
-
const { did, dwnUrl, delegateDid, protocol, entries: pushEntries, retryCount } = pushRuntime;
|
|
1855
|
-
pushRuntime.entries = [];
|
|
1856
|
-
if (pushEntries.length === 0) {
|
|
1857
|
-
if (!pushRuntime.timer && !pushRuntime.flushing && retryCount === 0) {
|
|
1858
|
-
this._pushRuntimes.delete(linkKey);
|
|
1859
|
-
}
|
|
2380
|
+
const batch = this.takePushFlushBatch(linkKey);
|
|
2381
|
+
if (!batch) {
|
|
1860
2382
|
return;
|
|
1861
2383
|
}
|
|
1862
|
-
const
|
|
1863
|
-
|
|
2384
|
+
const { pushRuntime, pushEntries, isStale } = batch;
|
|
2385
|
+
const { did, dwnUrl, delegateDid, protocol, permissionGrantIds, retryCount } = pushRuntime;
|
|
1864
2386
|
try {
|
|
1865
2387
|
const result = yield pushMessages({
|
|
1866
|
-
did,
|
|
1867
|
-
|
|
2388
|
+
did,
|
|
2389
|
+
dwnUrl,
|
|
2390
|
+
delegateDid,
|
|
2391
|
+
permissionGrantIds,
|
|
2392
|
+
messageCids: pushEntries.map((entry) => entry.cid),
|
|
1868
2393
|
agent: this.agent,
|
|
1869
|
-
permissionsApi: this._permissionsApi,
|
|
1870
2394
|
});
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
}
|
|
1876
|
-
// Auto-clear dead letters for CIDs that succeeded — a previously
|
|
1877
|
-
// failed message may have been repaired by reconciliation.
|
|
1878
|
-
for (const cid of result.succeeded) {
|
|
1879
|
-
this.clearFailedMessage(cid, dwnUrl).catch(() => { });
|
|
1880
|
-
}
|
|
1881
|
-
// Record permanently failed messages in the dead letter store.
|
|
1882
|
-
for (const entry of result.permanentlyFailed) {
|
|
1883
|
-
yield this.recordDeadLetter({
|
|
1884
|
-
messageCid: entry.cid,
|
|
1885
|
-
tenantDid: did,
|
|
1886
|
-
remoteEndpoint: dwnUrl,
|
|
1887
|
-
protocol,
|
|
1888
|
-
category: 'push-permanent',
|
|
1889
|
-
errorCode: String((_a = entry.statusCode) !== null && _a !== void 0 ? _a : ''),
|
|
1890
|
-
errorDetail: (_b = entry.detail) !== null && _b !== void 0 ? _b : 'permanent push failure',
|
|
1891
|
-
});
|
|
1892
|
-
}
|
|
1893
|
-
if (result.failed.length > 0) {
|
|
1894
|
-
if (isFlushStale()) {
|
|
1895
|
-
return;
|
|
1896
|
-
}
|
|
1897
|
-
const failedSet = new Set(result.failed);
|
|
1898
|
-
const failedEntries = pushEntries.filter((entry) => failedSet.has(entry.cid));
|
|
1899
|
-
this.requeueOrReconcile(linkKey, {
|
|
1900
|
-
did, dwnUrl, delegateDid, protocol,
|
|
1901
|
-
entries: failedEntries,
|
|
1902
|
-
retryCount: retryCount + 1,
|
|
1903
|
-
});
|
|
1904
|
-
}
|
|
1905
|
-
else {
|
|
1906
|
-
// Successful push — reset retry count so subsequent unrelated
|
|
1907
|
-
// batches on this link start with a fresh budget.
|
|
1908
|
-
pushRuntime.retryCount = 0;
|
|
1909
|
-
if (!pushRuntime.timer && pushRuntime.entries.length === 0) {
|
|
1910
|
-
this._pushRuntimes.delete(linkKey);
|
|
1911
|
-
}
|
|
1912
|
-
}
|
|
1913
|
-
}
|
|
1914
|
-
catch (error) {
|
|
1915
|
-
if (isFlushStale()) {
|
|
2395
|
+
yield this.handlePushBatchResult(linkKey, batch, result);
|
|
2396
|
+
}
|
|
2397
|
+
catch (error) {
|
|
2398
|
+
if (isStale()) {
|
|
1916
2399
|
return;
|
|
1917
2400
|
}
|
|
1918
2401
|
console.error(`SyncEngineLevel: Push batch failed for ${did} -> ${dwnUrl}`, error);
|
|
1919
2402
|
this.requeueOrReconcile(linkKey, {
|
|
1920
|
-
did,
|
|
2403
|
+
did,
|
|
2404
|
+
dwnUrl,
|
|
2405
|
+
delegateDid,
|
|
2406
|
+
protocol,
|
|
2407
|
+
permissionGrantIds,
|
|
1921
2408
|
entries: pushEntries,
|
|
1922
2409
|
retryCount: retryCount + 1,
|
|
1923
2410
|
});
|
|
1924
2411
|
}
|
|
1925
2412
|
finally {
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
2413
|
+
this.finishPushFlush(linkKey, pushRuntime);
|
|
2414
|
+
}
|
|
2415
|
+
});
|
|
2416
|
+
}
|
|
2417
|
+
takePushFlushBatch(linkKey) {
|
|
2418
|
+
// Guard: bail if this link was hot-removed or is no longer live. Without
|
|
2419
|
+
// this, a stale debounce timer or retry callback could send pushes after
|
|
2420
|
+
// the DID was removed or the link entered repair/terminal state.
|
|
2421
|
+
const flushLink = this._activeLinks.get(linkKey);
|
|
2422
|
+
if ((flushLink === null || flushLink === void 0 ? void 0 : flushLink.status) !== 'live') {
|
|
2423
|
+
const staleRuntime = this._pushRuntimes.get(linkKey);
|
|
2424
|
+
if (staleRuntime === null || staleRuntime === void 0 ? void 0 : staleRuntime.timer) {
|
|
2425
|
+
clearTimeout(staleRuntime.timer);
|
|
2426
|
+
}
|
|
2427
|
+
this._pushRuntimes.delete(linkKey);
|
|
2428
|
+
return undefined;
|
|
2429
|
+
}
|
|
2430
|
+
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
2431
|
+
if (!pushRuntime) {
|
|
2432
|
+
return undefined;
|
|
2433
|
+
}
|
|
2434
|
+
const { entries: pushEntries, retryCount } = pushRuntime;
|
|
2435
|
+
pushRuntime.entries = [];
|
|
2436
|
+
if (pushEntries.length === 0) {
|
|
2437
|
+
if (!pushRuntime.timer && !pushRuntime.flushing && retryCount === 0) {
|
|
2438
|
+
this._pushRuntimes.delete(linkKey);
|
|
2439
|
+
}
|
|
2440
|
+
return undefined;
|
|
2441
|
+
}
|
|
2442
|
+
// Capture the current active link identity so we can detect
|
|
2443
|
+
// remove+re-add during the await pushMessages() call.
|
|
2444
|
+
const isStale = () => !this._activeLinks.has(linkKey) ||
|
|
2445
|
+
(flushLink !== undefined && this._activeLinks.get(linkKey) !== flushLink);
|
|
2446
|
+
pushRuntime.flushing = true;
|
|
2447
|
+
return { pushRuntime, pushEntries, isStale };
|
|
2448
|
+
}
|
|
2449
|
+
handlePushBatchResult(linkKey, batch, result) {
|
|
2450
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2451
|
+
if (batch.isStale()) {
|
|
2452
|
+
return;
|
|
2453
|
+
}
|
|
2454
|
+
this.clearSucceededPushFailures(result.succeeded, batch.pushRuntime.dwnUrl);
|
|
2455
|
+
yield this.recordPermanentPushFailures(batch.pushRuntime, result.permanentlyFailed);
|
|
2456
|
+
if (result.failed.length > 0) {
|
|
2457
|
+
this.requeueFailedPushes(linkKey, batch, result.failed);
|
|
2458
|
+
return;
|
|
2459
|
+
}
|
|
2460
|
+
this.cleanupSuccessfulPushRuntime(linkKey, batch.pushRuntime);
|
|
2461
|
+
});
|
|
2462
|
+
}
|
|
2463
|
+
clearSucceededPushFailures(cids, dwnUrl) {
|
|
2464
|
+
for (const cid of cids) {
|
|
2465
|
+
this.clearFailedMessage(cid, dwnUrl).catch(() => { });
|
|
2466
|
+
}
|
|
2467
|
+
}
|
|
2468
|
+
recordPermanentPushFailures(pushRuntime, permanentlyFailed) {
|
|
2469
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2470
|
+
var _a, _b;
|
|
2471
|
+
for (const entry of permanentlyFailed) {
|
|
2472
|
+
yield this.recordDeadLetter({
|
|
2473
|
+
messageCid: entry.cid,
|
|
2474
|
+
tenantDid: pushRuntime.did,
|
|
2475
|
+
remoteEndpoint: pushRuntime.dwnUrl,
|
|
2476
|
+
protocol: pushRuntime.protocol,
|
|
2477
|
+
category: 'push-permanent',
|
|
2478
|
+
errorCode: String((_a = entry.statusCode) !== null && _a !== void 0 ? _a : ''),
|
|
2479
|
+
errorDetail: (_b = entry.detail) !== null && _b !== void 0 ? _b : 'permanent push failure',
|
|
2480
|
+
});
|
|
1937
2481
|
}
|
|
1938
2482
|
});
|
|
1939
2483
|
}
|
|
2484
|
+
requeueFailedPushes(linkKey, batch, failedCids) {
|
|
2485
|
+
if (batch.isStale()) {
|
|
2486
|
+
return;
|
|
2487
|
+
}
|
|
2488
|
+
const { did, dwnUrl, delegateDid, protocol, permissionGrantIds, retryCount } = batch.pushRuntime;
|
|
2489
|
+
const failedSet = new Set(failedCids);
|
|
2490
|
+
const failedEntries = batch.pushEntries.filter((entry) => failedSet.has(entry.cid));
|
|
2491
|
+
this.requeueOrReconcile(linkKey, {
|
|
2492
|
+
did,
|
|
2493
|
+
dwnUrl,
|
|
2494
|
+
delegateDid,
|
|
2495
|
+
protocol,
|
|
2496
|
+
permissionGrantIds,
|
|
2497
|
+
entries: failedEntries,
|
|
2498
|
+
retryCount: retryCount + 1,
|
|
2499
|
+
});
|
|
2500
|
+
}
|
|
2501
|
+
cleanupSuccessfulPushRuntime(linkKey, pushRuntime) {
|
|
2502
|
+
// Successful push — reset retry count so subsequent unrelated batches on
|
|
2503
|
+
// this link start with a fresh budget.
|
|
2504
|
+
pushRuntime.retryCount = 0;
|
|
2505
|
+
if (!pushRuntime.timer && pushRuntime.entries.length === 0) {
|
|
2506
|
+
this._pushRuntimes.delete(linkKey);
|
|
2507
|
+
}
|
|
2508
|
+
}
|
|
2509
|
+
finishPushFlush(linkKey, pushRuntime) {
|
|
2510
|
+
pushRuntime.flushing = false;
|
|
2511
|
+
// If new entries accumulated while this push was in flight, schedule a
|
|
2512
|
+
// short drain to flush them. This gives a brief batching window for burst
|
|
2513
|
+
// writes while keeping single-write latency low.
|
|
2514
|
+
const rt = this._pushRuntimes.get(linkKey);
|
|
2515
|
+
if (rt && rt.entries.length > 0 && !rt.timer) {
|
|
2516
|
+
rt.timer = setTimeout(() => {
|
|
2517
|
+
rt.timer = undefined;
|
|
2518
|
+
void this.flushPendingPushesForLink(linkKey);
|
|
2519
|
+
}, PUSH_DEBOUNCE_MS);
|
|
2520
|
+
}
|
|
2521
|
+
}
|
|
1940
2522
|
/**
|
|
1941
2523
|
* Re-queues a failed push batch for retry, or marks the link
|
|
1942
2524
|
* `needsReconcile` if retries are exhausted. Bounded to prevent
|
|
@@ -1964,34 +2546,692 @@ export class SyncEngineLevel {
|
|
|
1964
2546
|
}
|
|
1965
2547
|
this._pushRuntimes.delete(targetKey);
|
|
1966
2548
|
const link = this._activeLinks.get(targetKey);
|
|
1967
|
-
if (link
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
2549
|
+
if (link) {
|
|
2550
|
+
this.markLinkNeedsReconcile(targetKey, link, 'push-retry-exhausted');
|
|
2551
|
+
}
|
|
2552
|
+
return;
|
|
2553
|
+
}
|
|
2554
|
+
pushRuntime.entries.push(...pending.entries);
|
|
2555
|
+
pushRuntime.retryCount = pending.retryCount;
|
|
2556
|
+
const delayMs = (_a = SyncEngineLevel.PUSH_RETRY_BACKOFF_MS[pending.retryCount]) !== null && _a !== void 0 ? _a : 2000;
|
|
2557
|
+
if (pushRuntime.timer) {
|
|
2558
|
+
clearTimeout(pushRuntime.timer);
|
|
2559
|
+
}
|
|
2560
|
+
pushRuntime.timer = setTimeout(() => {
|
|
2561
|
+
pushRuntime.timer = undefined;
|
|
2562
|
+
void this.flushPendingPushesForLink(targetKey);
|
|
2563
|
+
}, delayMs);
|
|
2564
|
+
}
|
|
2565
|
+
markLinkNeedsReconcile(linkKey, link, reason) {
|
|
2566
|
+
if (link.needsReconcile) {
|
|
2567
|
+
this.scheduleReconcile(linkKey);
|
|
2568
|
+
return;
|
|
2569
|
+
}
|
|
2570
|
+
link.needsReconcile = true;
|
|
2571
|
+
void this.ledger.saveLink(link).then(() => {
|
|
2572
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'reconcile:needed', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, syncEventScope(link.scope)), { reason }));
|
|
2573
|
+
this.scheduleReconcile(linkKey);
|
|
2574
|
+
}).catch((error) => {
|
|
2575
|
+
console.error(`SyncEngineLevel: Failed to mark link for reconciliation ${link.tenantDid} -> ${link.remoteEndpoint}`, error);
|
|
2576
|
+
});
|
|
2577
|
+
}
|
|
2578
|
+
createLinkReconciler(shouldContinue) {
|
|
2579
|
+
return new SyncLinkReconciler({
|
|
2580
|
+
getLocalRoot: (did, delegateDid, protocol, permissionGrantIds) => __awaiter(this, void 0, void 0, function* () { return this.getLocalRoot(did, delegateDid, protocol, permissionGrantIds); }),
|
|
2581
|
+
getRemoteRoot: (did, dwnUrl, delegateDid, protocol, permissionGrantIds) => __awaiter(this, void 0, void 0, function* () { return this.getRemoteRoot(did, dwnUrl, delegateDid, protocol, permissionGrantIds); }),
|
|
2582
|
+
diffWithRemote: (target) => __awaiter(this, void 0, void 0, function* () { return this.diffWithRemote(target); }),
|
|
2583
|
+
pullMessages: (params) => __awaiter(this, void 0, void 0, function* () { return this.pullMessages(params); }),
|
|
2584
|
+
pushMessages: (params) => __awaiter(this, void 0, void 0, function* () { return this.pushMessages(params); }),
|
|
2585
|
+
shouldContinue,
|
|
2586
|
+
});
|
|
2587
|
+
}
|
|
2588
|
+
getReconcileProtocols(scope) {
|
|
2589
|
+
var _a;
|
|
2590
|
+
return (_a = protocolsForSyncScope(scope)) !== null && _a !== void 0 ? _a : [undefined];
|
|
2591
|
+
}
|
|
2592
|
+
getAuthorizationGrantIds(authorization) {
|
|
2593
|
+
return authorization.kind === 'delegate' ? authorization.permissionGrantIds : undefined;
|
|
2594
|
+
}
|
|
2595
|
+
reconcileProjectionTarget(target, options, shouldContinue) {
|
|
2596
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2597
|
+
if (target.scope.kind === 'recordsProjection') {
|
|
2598
|
+
return this.reconcileRecordsProjectionTarget(target, target.scope, options, shouldContinue);
|
|
2599
|
+
}
|
|
2600
|
+
if (target.scope.kind === 'protocolSet' && target.scope.protocols.length > 1) {
|
|
2601
|
+
return this.reconcileProtocolSetProjectionTarget(target, options, shouldContinue);
|
|
2602
|
+
}
|
|
2603
|
+
let converged = true;
|
|
2604
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
2605
|
+
const reconciler = this.createLinkReconciler(shouldContinue);
|
|
2606
|
+
for (const protocol of this.getReconcileProtocols(target.scope)) {
|
|
2607
|
+
const outcome = yield reconciler.reconcile({
|
|
2608
|
+
did: target.did,
|
|
2609
|
+
dwnUrl: target.dwnUrl,
|
|
2610
|
+
delegateDid: target.delegateDid,
|
|
2611
|
+
protocol,
|
|
2612
|
+
permissionGrantIds,
|
|
2613
|
+
}, options);
|
|
2614
|
+
if (outcome.aborted) {
|
|
2615
|
+
return { aborted: true };
|
|
2616
|
+
}
|
|
2617
|
+
if ((options === null || options === void 0 ? void 0 : options.verifyConvergence) === true && outcome.converged !== true) {
|
|
2618
|
+
converged = false;
|
|
2619
|
+
}
|
|
2620
|
+
}
|
|
2621
|
+
return (options === null || options === void 0 ? void 0 : options.verifyConvergence) === true ? { converged } : {};
|
|
2622
|
+
});
|
|
2623
|
+
}
|
|
2624
|
+
reconcileRecordsProjectionTarget(target, scope, options, shouldContinue) {
|
|
2625
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2626
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
2627
|
+
const localRoot = yield this.getLocalProjectedRoot(target.did, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2628
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2629
|
+
return { aborted: true };
|
|
2630
|
+
}
|
|
2631
|
+
const remoteRoot = yield this.getRemoteProjectedRoot(target.did, target.dwnUrl, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2632
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2633
|
+
return { aborted: true };
|
|
2634
|
+
}
|
|
2635
|
+
if (localRoot !== remoteRoot) {
|
|
2636
|
+
const diff = yield this.diffProjectedWithRemote({
|
|
2637
|
+
did: target.did,
|
|
2638
|
+
dwnUrl: target.dwnUrl,
|
|
2639
|
+
delegateDid: target.delegateDid,
|
|
2640
|
+
scopes: scope.scopes,
|
|
2641
|
+
permissionGrantIds,
|
|
2642
|
+
});
|
|
2643
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2644
|
+
return { aborted: true };
|
|
2645
|
+
}
|
|
2646
|
+
const aborted = yield this.applyProjectedDiff(target, scope, diff, permissionGrantIds, options, shouldContinue);
|
|
2647
|
+
if (aborted) {
|
|
2648
|
+
return { aborted: true };
|
|
2649
|
+
}
|
|
2650
|
+
}
|
|
2651
|
+
if ((options === null || options === void 0 ? void 0 : options.verifyConvergence) !== true) {
|
|
2652
|
+
return {};
|
|
2653
|
+
}
|
|
2654
|
+
const postLocalRoot = yield this.getLocalProjectedRoot(target.did, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2655
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2656
|
+
return { aborted: true };
|
|
2657
|
+
}
|
|
2658
|
+
const postRemoteRoot = yield this.getRemoteProjectedRoot(target.did, target.dwnUrl, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2659
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2660
|
+
return { aborted: true };
|
|
2661
|
+
}
|
|
2662
|
+
return { converged: postLocalRoot === postRemoteRoot };
|
|
2663
|
+
});
|
|
2664
|
+
}
|
|
2665
|
+
reconcileProtocolSetProjectionTarget(target, options, shouldContinue) {
|
|
2666
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2667
|
+
if (target.scope.kind !== 'protocolSet') {
|
|
2668
|
+
return {};
|
|
2669
|
+
}
|
|
2670
|
+
const scope = target.scope;
|
|
2671
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
2672
|
+
const diffPlan = yield this.collectProtocolSetDiffPlan(target, scope, permissionGrantIds, shouldContinue);
|
|
2673
|
+
if (!diffPlan) {
|
|
2674
|
+
return { aborted: true };
|
|
2675
|
+
}
|
|
2676
|
+
if (diffPlan.changedProtocols.length === 0) {
|
|
2677
|
+
return (options === null || options === void 0 ? void 0 : options.verifyConvergence) === true ? { converged: true } : {};
|
|
2678
|
+
}
|
|
2679
|
+
const aborted = yield this.applyProtocolSetDiffPlan(target, scope, diffPlan, permissionGrantIds, options, shouldContinue);
|
|
2680
|
+
if (aborted) {
|
|
2681
|
+
return { aborted: true };
|
|
2682
|
+
}
|
|
2683
|
+
if ((options === null || options === void 0 ? void 0 : options.verifyConvergence) !== true) {
|
|
2684
|
+
return {};
|
|
2685
|
+
}
|
|
2686
|
+
return this.verifyProtocolSetConvergence(target, diffPlan.changedProtocols, permissionGrantIds, shouldContinue);
|
|
2687
|
+
});
|
|
2688
|
+
}
|
|
2689
|
+
collectProtocolSetDiffPlan(target, scope, permissionGrantIds, shouldContinue) {
|
|
2690
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2691
|
+
const plan = { changedProtocols: [], onlyRemote: [], onlyLocal: [] };
|
|
2692
|
+
for (const protocol of scope.protocols) {
|
|
2693
|
+
const roots = yield this.getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue);
|
|
2694
|
+
if (!roots) {
|
|
2695
|
+
return undefined;
|
|
2696
|
+
}
|
|
2697
|
+
if (roots.localRoot === roots.remoteRoot) {
|
|
2698
|
+
continue;
|
|
2699
|
+
}
|
|
2700
|
+
plan.changedProtocols.push(protocol);
|
|
2701
|
+
const diff = yield this.diffWithRemote({
|
|
2702
|
+
did: target.did,
|
|
2703
|
+
dwnUrl: target.dwnUrl,
|
|
2704
|
+
delegateDid: target.delegateDid,
|
|
2705
|
+
protocol,
|
|
2706
|
+
permissionGrantIds,
|
|
2707
|
+
});
|
|
2708
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2709
|
+
return undefined;
|
|
2710
|
+
}
|
|
2711
|
+
plan.onlyRemote.push(...diff.onlyRemote);
|
|
2712
|
+
plan.onlyLocal.push(...diff.onlyLocal);
|
|
2713
|
+
}
|
|
2714
|
+
return plan;
|
|
2715
|
+
});
|
|
2716
|
+
}
|
|
2717
|
+
getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue) {
|
|
2718
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2719
|
+
const localRoot = yield this.getLocalRoot(target.did, target.delegateDid, protocol, permissionGrantIds);
|
|
2720
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2721
|
+
return undefined;
|
|
2722
|
+
}
|
|
2723
|
+
const remoteRoot = yield this.getRemoteRoot(target.did, target.dwnUrl, target.delegateDid, protocol, permissionGrantIds);
|
|
2724
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2725
|
+
return undefined;
|
|
2726
|
+
}
|
|
2727
|
+
return { localRoot, remoteRoot };
|
|
2728
|
+
});
|
|
2729
|
+
}
|
|
2730
|
+
applyProtocolSetDiffPlan(target, scope, diffPlan, permissionGrantIds, options, shouldContinue) {
|
|
2731
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2732
|
+
// Keep the remote diff combined across protocols so topologicalSort can
|
|
2733
|
+
// order composed protocol configs before records that use them. Any future
|
|
2734
|
+
// chunking for large protocol sets must preserve this global dependency
|
|
2735
|
+
// order instead of reverting to independent per-protocol chunks.
|
|
2736
|
+
if ((options === null || options === void 0 ? void 0 : options.direction) !== 'push' &&
|
|
2737
|
+
diffPlan.onlyRemote.length > 0 &&
|
|
2738
|
+
(yield this.pullRemoteDiffEntries(target, scope, diffPlan.onlyRemote, permissionGrantIds, shouldContinue))) {
|
|
2739
|
+
return true;
|
|
2740
|
+
}
|
|
2741
|
+
if ((options === null || options === void 0 ? void 0 : options.direction) === 'pull' || diffPlan.onlyLocal.length === 0) {
|
|
2742
|
+
return false;
|
|
2743
|
+
}
|
|
2744
|
+
return this.pushLocalDiffEntries(target, diffPlan.onlyLocal, permissionGrantIds, shouldContinue);
|
|
2745
|
+
});
|
|
2746
|
+
}
|
|
2747
|
+
applyProjectedDiff(target, scope, diff, permissionGrantIds, options, shouldContinue) {
|
|
2748
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2749
|
+
if (yield this.pullProjectedRemoteDiff(target, scope, diff, permissionGrantIds, options, shouldContinue)) {
|
|
2750
|
+
return true;
|
|
2751
|
+
}
|
|
2752
|
+
return this.pushProjectedLocalDiff(target, diff.onlyLocal, permissionGrantIds, options, shouldContinue);
|
|
2753
|
+
});
|
|
2754
|
+
}
|
|
2755
|
+
pullProjectedRemoteDiff(target, scope, diff, permissionGrantIds, options, shouldContinue) {
|
|
2756
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2757
|
+
var _a;
|
|
2758
|
+
if ((options === null || options === void 0 ? void 0 : options.direction) === 'push' || diff.onlyRemote.length === 0) {
|
|
2759
|
+
return false;
|
|
2760
|
+
}
|
|
2761
|
+
return this.pullRemoteDiffEntries(target, scope, diff.onlyRemote, permissionGrantIds, shouldContinue, (_a = diff.dependencies) !== null && _a !== void 0 ? _a : []);
|
|
2762
|
+
});
|
|
2763
|
+
}
|
|
2764
|
+
pushProjectedLocalDiff(target, onlyLocal, permissionGrantIds, options, shouldContinue) {
|
|
2765
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2766
|
+
if ((options === null || options === void 0 ? void 0 : options.direction) === 'pull' || onlyLocal.length === 0) {
|
|
2767
|
+
return false;
|
|
2768
|
+
}
|
|
2769
|
+
return this.pushLocalDiffEntries(target, onlyLocal, permissionGrantIds, shouldContinue);
|
|
2770
|
+
});
|
|
2771
|
+
}
|
|
2772
|
+
pullRemoteDiffEntries(target_1, scope_1, onlyRemote_1, permissionGrantIds_1, shouldContinue_1) {
|
|
2773
|
+
return __awaiter(this, arguments, void 0, function* (target, scope, onlyRemote, permissionGrantIds, shouldContinue, dependencies = []) {
|
|
2774
|
+
const primaryEntries = SyncEngineLevel.dedupeRemoteEntries(onlyRemote);
|
|
2775
|
+
try {
|
|
2776
|
+
let verifiedInitialWrites = [];
|
|
2777
|
+
if (scope.kind === 'recordsProjection') {
|
|
2778
|
+
verifiedInitialWrites = yield this.pullProjectedDependencyHints(target, scope, primaryEntries, dependencies, permissionGrantIds, shouldContinue);
|
|
2779
|
+
}
|
|
2780
|
+
const { prefetched, needsFetchCids } = partitionRemoteEntries(primaryEntries);
|
|
2781
|
+
yield this.pullMessages({
|
|
2782
|
+
did: target.did,
|
|
2783
|
+
dwnUrl: target.dwnUrl,
|
|
2784
|
+
delegateDid: target.delegateDid,
|
|
2785
|
+
scope,
|
|
2786
|
+
permissionGrantIds,
|
|
2787
|
+
messageCids: needsFetchCids,
|
|
2788
|
+
prefetched,
|
|
2789
|
+
verifiedInitialWrites,
|
|
2790
|
+
shouldContinue,
|
|
1972
2791
|
});
|
|
1973
2792
|
}
|
|
2793
|
+
catch (error) {
|
|
2794
|
+
if (error instanceof SyncPullAbortedError) {
|
|
2795
|
+
return true;
|
|
2796
|
+
}
|
|
2797
|
+
throw error;
|
|
2798
|
+
}
|
|
2799
|
+
return (shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false;
|
|
2800
|
+
});
|
|
2801
|
+
}
|
|
2802
|
+
pullProjectedDependencyHints(target, scope, primaryEntries, dependencies, permissionGrantIds, shouldContinue) {
|
|
2803
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2804
|
+
const verified = yield this.verifyProjectedDependencies(target.did, scope, primaryEntries, dependencies);
|
|
2805
|
+
if (verified.length === 0) {
|
|
2806
|
+
return [];
|
|
2807
|
+
}
|
|
2808
|
+
yield this.pullMessages({
|
|
2809
|
+
did: target.did,
|
|
2810
|
+
dwnUrl: target.dwnUrl,
|
|
2811
|
+
delegateDid: target.delegateDid,
|
|
2812
|
+
scope: SyncEngineLevel.protocolSetScopeForProjectedDependencies(verified),
|
|
2813
|
+
permissionGrantIds,
|
|
2814
|
+
messageCids: [],
|
|
2815
|
+
prefetched: verified,
|
|
2816
|
+
shouldContinue,
|
|
2817
|
+
});
|
|
2818
|
+
return SyncEngineLevel.recordsInitialWritesFromVerifiedDependencies(verified);
|
|
2819
|
+
});
|
|
2820
|
+
}
|
|
2821
|
+
verifyProjectedDependencies(tenantDid, scope, primaryEntries, dependencies) {
|
|
2822
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2823
|
+
const primaryByCid = SyncEngineLevel.indexEntriesWithMessage(primaryEntries);
|
|
2824
|
+
const initialWritesByRoot = yield this.collectProjectedRecordsInitialWriteDependencies(scope, primaryByCid, dependencies);
|
|
2825
|
+
const protocolConfigs = yield this.verifyProjectedProtocolConfigDependencies(tenantDid, scope, primaryEntries, dependencies, initialWritesByRoot);
|
|
2826
|
+
return SyncEngineLevel.dedupeDependencyEntries([
|
|
2827
|
+
...protocolConfigs,
|
|
2828
|
+
...initialWritesByRoot.values(),
|
|
2829
|
+
]);
|
|
2830
|
+
});
|
|
2831
|
+
}
|
|
2832
|
+
verifyProjectedProtocolConfigDependencies(tenantDid_1, scope_1, primaryEntries_1, dependencies_1) {
|
|
2833
|
+
return __awaiter(this, arguments, void 0, function* (tenantDid, scope, primaryEntries, dependencies, initialWritesByRoot = new Map()) {
|
|
2834
|
+
var _a;
|
|
2835
|
+
// Projected sync dependency entries are untrusted server hints. Before any
|
|
2836
|
+
// config is applied, bind it to an accepted primary record by CID,
|
|
2837
|
+
// tenant authorship, signature, timestamp, scope, and protocol closure;
|
|
2838
|
+
// malformed or unrelated hints are ignored.
|
|
2839
|
+
const primaryByCid = SyncEngineLevel.indexEntriesWithMessage(primaryEntries);
|
|
2840
|
+
const candidatesByRoot = yield this.collectProjectedProtocolConfigCandidates(tenantDid, scope, primaryByCid, dependencies, initialWritesByRoot);
|
|
2841
|
+
const verified = new Map();
|
|
2842
|
+
for (const [rootMessageCid, rootCandidates] of candidatesByRoot) {
|
|
2843
|
+
const primary = primaryByCid.get(rootMessageCid);
|
|
2844
|
+
const rootRecordsWrite = primary === undefined
|
|
2845
|
+
? undefined
|
|
2846
|
+
: SyncEngineLevel.protocolConfigRootRecordsWrite(primary.message, (_a = initialWritesByRoot.get(rootMessageCid)) === null || _a === void 0 ? void 0 : _a.message);
|
|
2847
|
+
const rootProtocol = SyncEngineLevel.recordsWriteProtocol(rootRecordsWrite);
|
|
2848
|
+
if (rootProtocol === undefined) {
|
|
2849
|
+
continue;
|
|
2850
|
+
}
|
|
2851
|
+
for (const dependency of SyncEngineLevel.filterProtocolConfigClosure(rootProtocol, rootCandidates)) {
|
|
2852
|
+
verified.set(dependency.messageCid, dependency);
|
|
2853
|
+
}
|
|
2854
|
+
}
|
|
2855
|
+
return [...verified.values()];
|
|
2856
|
+
});
|
|
2857
|
+
}
|
|
2858
|
+
static indexEntriesWithMessage(entries) {
|
|
2859
|
+
const entriesByCid = new Map();
|
|
2860
|
+
for (const entry of entries) {
|
|
2861
|
+
if (SyncEngineLevel.hasMessage(entry)) {
|
|
2862
|
+
entriesByCid.set(entry.messageCid, entry);
|
|
2863
|
+
}
|
|
2864
|
+
}
|
|
2865
|
+
return entriesByCid;
|
|
2866
|
+
}
|
|
2867
|
+
static recordsInitialWritesFromVerifiedDependencies(entries) {
|
|
2868
|
+
const initialWrites = [];
|
|
2869
|
+
for (const entry of entries) {
|
|
2870
|
+
if (SyncEngineLevel.hasMessage(entry) && SyncEngineLevel.isRecordsWriteMessage(entry.message)) {
|
|
2871
|
+
initialWrites.push(entry.message);
|
|
2872
|
+
}
|
|
2873
|
+
}
|
|
2874
|
+
return initialWrites;
|
|
2875
|
+
}
|
|
2876
|
+
collectProjectedRecordsInitialWriteDependencies(scope, primaryByCid, dependencies) {
|
|
2877
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2878
|
+
const dependenciesByRoot = new Map();
|
|
2879
|
+
for (const dependency of dependencies) {
|
|
2880
|
+
const verified = yield this.verifyRecordsInitialWriteCandidate(scope, primaryByCid, dependency);
|
|
2881
|
+
if (verified === undefined) {
|
|
2882
|
+
continue;
|
|
2883
|
+
}
|
|
2884
|
+
dependenciesByRoot.set(verified.rootMessageCid, verified.dependency);
|
|
2885
|
+
}
|
|
2886
|
+
return dependenciesByRoot;
|
|
2887
|
+
});
|
|
2888
|
+
}
|
|
2889
|
+
verifyRecordsInitialWriteCandidate(scope, primaryByCid, dependency) {
|
|
2890
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2891
|
+
if (dependency.dependencyClass !== 'recordsInitialWrite' ||
|
|
2892
|
+
!SyncEngineLevel.hasMessage(dependency) ||
|
|
2893
|
+
SyncEngineLevel.hasDependencyPayloadBytes(dependency)) {
|
|
2894
|
+
return undefined;
|
|
2895
|
+
}
|
|
2896
|
+
const primary = primaryByCid.get(dependency.rootMessageCid);
|
|
2897
|
+
if (primary === undefined ||
|
|
2898
|
+
!SyncEngineLevel.isRecordsDeleteMessage(primary.message) ||
|
|
2899
|
+
!(yield SyncEngineLevel.projectedDependencyCidsMatch({
|
|
2900
|
+
dependencyCid: dependency.messageCid,
|
|
2901
|
+
dependencyMessage: dependency.message,
|
|
2902
|
+
primaryCid: primary.messageCid,
|
|
2903
|
+
primaryMessage: primary.message,
|
|
2904
|
+
}))) {
|
|
2905
|
+
return undefined;
|
|
2906
|
+
}
|
|
2907
|
+
const initialWrite = yield this.toAuthenticatedRecordsInitialWriteDependency(dependency);
|
|
2908
|
+
if (initialWrite === undefined ||
|
|
2909
|
+
initialWrite.message.recordId !== SyncEngineLevel.recordsDeleteRecordId(primary.message) ||
|
|
2910
|
+
classifySyncMessageScope({ message: primary.message, initialWrite: initialWrite.message, scope }) !== 'in-scope') {
|
|
2911
|
+
return undefined;
|
|
2912
|
+
}
|
|
2913
|
+
return { dependency: initialWrite, rootMessageCid: dependency.rootMessageCid };
|
|
2914
|
+
});
|
|
2915
|
+
}
|
|
2916
|
+
toAuthenticatedRecordsInitialWriteDependency(dependency) {
|
|
2917
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2918
|
+
if (!SyncEngineLevel.isRecordsWriteMessage(dependency.message)) {
|
|
2919
|
+
return undefined;
|
|
2920
|
+
}
|
|
2921
|
+
try {
|
|
2922
|
+
const recordsWrite = yield RecordsWrite.parse(dependency.message);
|
|
2923
|
+
yield authenticate(recordsWrite.message.authorization, this.agent.did, recordsWrite.message.attestation);
|
|
2924
|
+
return (yield recordsWrite.isInitialWrite())
|
|
2925
|
+
? Object.assign(Object.assign({}, dependency), { message: recordsWrite.message }) : undefined;
|
|
2926
|
+
}
|
|
2927
|
+
catch (_a) {
|
|
2928
|
+
return undefined;
|
|
2929
|
+
}
|
|
2930
|
+
});
|
|
2931
|
+
}
|
|
2932
|
+
collectProjectedProtocolConfigCandidates(tenantDid, scope, primaryByCid, dependencies, initialWritesByRoot) {
|
|
2933
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2934
|
+
var _a;
|
|
2935
|
+
const candidatesByRoot = new Map();
|
|
2936
|
+
for (const dependency of dependencies) {
|
|
2937
|
+
const verified = yield this.verifyProtocolConfigCandidate(tenantDid, scope, primaryByCid, dependency, initialWritesByRoot);
|
|
2938
|
+
if (verified === undefined) {
|
|
2939
|
+
continue;
|
|
2940
|
+
}
|
|
2941
|
+
const rootCandidates = (_a = candidatesByRoot.get(verified.rootMessageCid)) !== null && _a !== void 0 ? _a : [];
|
|
2942
|
+
rootCandidates.push(verified.dependency);
|
|
2943
|
+
candidatesByRoot.set(verified.rootMessageCid, rootCandidates);
|
|
2944
|
+
}
|
|
2945
|
+
return candidatesByRoot;
|
|
2946
|
+
});
|
|
2947
|
+
}
|
|
2948
|
+
verifyProtocolConfigCandidate(tenantDid, scope, primaryByCid, dependency, initialWritesByRoot) {
|
|
2949
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2950
|
+
var _a;
|
|
2951
|
+
if (dependency.dependencyClass !== 'protocolsConfigure' || !SyncEngineLevel.hasMessage(dependency)) {
|
|
2952
|
+
return undefined;
|
|
2953
|
+
}
|
|
2954
|
+
const primary = primaryByCid.get(dependency.rootMessageCid);
|
|
2955
|
+
if (primary === undefined) {
|
|
2956
|
+
return undefined;
|
|
2957
|
+
}
|
|
2958
|
+
const verifiedDependency = yield this.verifyProtocolConfigCandidateMessage(tenantDid, scope, primary, dependency, (_a = initialWritesByRoot.get(dependency.rootMessageCid)) === null || _a === void 0 ? void 0 : _a.message);
|
|
2959
|
+
return verifiedDependency === undefined
|
|
2960
|
+
? undefined
|
|
2961
|
+
: { dependency: verifiedDependency, rootMessageCid: dependency.rootMessageCid };
|
|
2962
|
+
});
|
|
2963
|
+
}
|
|
2964
|
+
verifyProtocolConfigCandidateMessage(tenantDid, scope, primary, dependency, initialWrite) {
|
|
2965
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2966
|
+
// Protocol authorization is temporal: a record is governed by the protocol
|
|
2967
|
+
// definition active at its creation timestamp. Future configs may add
|
|
2968
|
+
// unrelated `uses` dependencies, so they must not widen this primary's
|
|
2969
|
+
// dependency closure.
|
|
2970
|
+
if (!(yield SyncEngineLevel.projectedDependencyCidsMatch({
|
|
2971
|
+
dependencyCid: dependency.messageCid,
|
|
2972
|
+
dependencyMessage: dependency.message,
|
|
2973
|
+
primaryCid: primary.messageCid,
|
|
2974
|
+
primaryMessage: primary.message,
|
|
2975
|
+
}))) {
|
|
2976
|
+
return undefined;
|
|
2977
|
+
}
|
|
2978
|
+
const authenticatedDependency = yield this.toAuthenticatedProtocolConfigDependency(tenantDid, dependency);
|
|
2979
|
+
if (authenticatedDependency === undefined) {
|
|
2980
|
+
return undefined;
|
|
2981
|
+
}
|
|
2982
|
+
const rootRecordsWrite = SyncEngineLevel.protocolConfigRootRecordsWrite(primary.message, initialWrite);
|
|
2983
|
+
const primaryIsInScope = rootRecordsWrite !== undefined &&
|
|
2984
|
+
classifySyncMessageScope({ message: primary.message, initialWrite, scope }) === 'in-scope';
|
|
2985
|
+
if (!primaryIsInScope ||
|
|
2986
|
+
!SyncEngineLevel.protocolsConfigureIsNotNewerThanRecordsWrite(authenticatedDependency.message, rootRecordsWrite)) {
|
|
2987
|
+
return undefined;
|
|
2988
|
+
}
|
|
2989
|
+
return authenticatedDependency;
|
|
2990
|
+
});
|
|
2991
|
+
}
|
|
2992
|
+
toAuthenticatedProtocolConfigDependency(tenantDid, dependency) {
|
|
2993
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2994
|
+
const config = yield this.toAuthenticatedTenantProtocolConfig(tenantDid, dependency.message);
|
|
2995
|
+
if (config === undefined) {
|
|
2996
|
+
return undefined;
|
|
2997
|
+
}
|
|
2998
|
+
return Object.assign(Object.assign({}, dependency), { message: config });
|
|
2999
|
+
});
|
|
3000
|
+
}
|
|
3001
|
+
static projectedDependencyCidsMatch(_a) {
|
|
3002
|
+
return __awaiter(this, arguments, void 0, function* ({ dependencyCid, dependencyMessage, primaryCid, primaryMessage, }) {
|
|
3003
|
+
return (yield Message.getCid(primaryMessage)) === primaryCid &&
|
|
3004
|
+
(yield Message.getCid(dependencyMessage)) === dependencyCid;
|
|
3005
|
+
});
|
|
3006
|
+
}
|
|
3007
|
+
static recordsWriteProtocol(message) {
|
|
3008
|
+
if (!SyncEngineLevel.isRecordsWriteProtocolMessage(message)) {
|
|
3009
|
+
return undefined;
|
|
3010
|
+
}
|
|
3011
|
+
const { protocol } = message.descriptor;
|
|
3012
|
+
return typeof protocol === 'string' ? protocol : undefined;
|
|
3013
|
+
}
|
|
3014
|
+
static recordsDeleteRecordId(message) {
|
|
3015
|
+
if (!SyncEngineLevel.isRecordsDeleteMessage(message)) {
|
|
3016
|
+
return undefined;
|
|
3017
|
+
}
|
|
3018
|
+
const recordId = message.descriptor.recordId;
|
|
3019
|
+
return typeof recordId === 'string' ? recordId : undefined;
|
|
3020
|
+
}
|
|
3021
|
+
static protocolConfigRootRecordsWrite(primary, initialWrite) {
|
|
3022
|
+
if (SyncEngineLevel.isRecordsWriteMessage(primary)) {
|
|
3023
|
+
return primary;
|
|
3024
|
+
}
|
|
3025
|
+
return SyncEngineLevel.isRecordsDeleteMessage(primary) ? initialWrite : undefined;
|
|
3026
|
+
}
|
|
3027
|
+
static protocolsConfigureProtocol(message) {
|
|
3028
|
+
return message.descriptor.definition.protocol;
|
|
3029
|
+
}
|
|
3030
|
+
static protocolsConfigureProtocolFromGenericMessage(message) {
|
|
3031
|
+
if (!SyncEngineLevel.isProtocolsConfigureDefinitionMessage(message)) {
|
|
3032
|
+
return undefined;
|
|
3033
|
+
}
|
|
3034
|
+
return message.descriptor.definition.protocol;
|
|
3035
|
+
}
|
|
3036
|
+
static protocolsConfigureIsNotNewerThanRecordsWrite(protocolsConfigureMessage, recordsWriteMessage) {
|
|
3037
|
+
return protocolsConfigureMessage.descriptor.messageTimestamp <= recordsWriteMessage.descriptor.messageTimestamp;
|
|
3038
|
+
}
|
|
3039
|
+
static protocolsConfigureUses(message) {
|
|
3040
|
+
var _a;
|
|
3041
|
+
const uses = (_a = message.descriptor.definition) === null || _a === void 0 ? void 0 : _a.uses;
|
|
3042
|
+
return uses === undefined
|
|
3043
|
+
? []
|
|
3044
|
+
: Object.values(uses).filter((protocol) => typeof protocol === 'string');
|
|
3045
|
+
}
|
|
3046
|
+
static hasMessage(entry) {
|
|
3047
|
+
return (entry === null || entry === void 0 ? void 0 : entry.message) !== undefined;
|
|
3048
|
+
}
|
|
3049
|
+
static isRecordsWriteProtocolMessage(message) {
|
|
3050
|
+
return (message === null || message === void 0 ? void 0 : message.descriptor.interface) === DwnInterfaceName.Records &&
|
|
3051
|
+
message.descriptor.method === DwnMethodName.Write;
|
|
3052
|
+
}
|
|
3053
|
+
static isRecordsWriteMessage(message) {
|
|
3054
|
+
return SyncEngineLevel.isRecordsWriteProtocolMessage(message) &&
|
|
3055
|
+
'recordId' in message &&
|
|
3056
|
+
typeof message.recordId === 'string' &&
|
|
3057
|
+
'contextId' in message &&
|
|
3058
|
+
typeof message.contextId === 'string';
|
|
3059
|
+
}
|
|
3060
|
+
static isRecordsDeleteMessage(message) {
|
|
3061
|
+
return message.descriptor.interface === DwnInterfaceName.Records &&
|
|
3062
|
+
message.descriptor.method === DwnMethodName.Delete;
|
|
3063
|
+
}
|
|
3064
|
+
static isProtocolsConfigureDefinitionMessage(message) {
|
|
3065
|
+
return message.descriptor.interface === DwnInterfaceName.Protocols &&
|
|
3066
|
+
message.descriptor.method === DwnMethodName.Configure &&
|
|
3067
|
+
message.authorization !== undefined &&
|
|
3068
|
+
SyncEngineLevel.hasProtocolsConfigureDefinition(message.descriptor);
|
|
3069
|
+
}
|
|
3070
|
+
static hasProtocolsConfigureDefinition(descriptor) {
|
|
3071
|
+
return SyncEngineLevel.isProtocolsConfigureDefinition(descriptor.definition);
|
|
3072
|
+
}
|
|
3073
|
+
static isProtocolsConfigureDefinition(definition) {
|
|
3074
|
+
return typeof definition === 'object' &&
|
|
3075
|
+
definition !== null &&
|
|
3076
|
+
'protocol' in definition &&
|
|
3077
|
+
typeof definition.protocol === 'string';
|
|
3078
|
+
}
|
|
3079
|
+
static filterProtocolConfigClosure(primaryProtocol, candidates) {
|
|
3080
|
+
// Start from the primary record's protocol and walk only protocols named by
|
|
3081
|
+
// accepted, signed config definitions. This keeps composed-protocol support
|
|
3082
|
+
// narrow: the governing config can admit its `uses` targets, but arbitrary
|
|
3083
|
+
// protocol config hints cannot enter the apply set.
|
|
3084
|
+
const candidatesByProtocol = SyncEngineLevel.groupGoverningProtocolConfigCandidatesByProtocol(candidates);
|
|
3085
|
+
const visitedProtocols = new Set();
|
|
3086
|
+
const pendingProtocols = [primaryProtocol];
|
|
3087
|
+
const accepted = new Map();
|
|
3088
|
+
for (let protocol = SyncEngineLevel.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols); protocol !== undefined; protocol = SyncEngineLevel.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols)) {
|
|
3089
|
+
SyncEngineLevel.acceptProtocolConfigCandidates({
|
|
3090
|
+
protocol,
|
|
3091
|
+
candidatesByProtocol,
|
|
3092
|
+
visitedProtocols,
|
|
3093
|
+
pendingProtocols,
|
|
3094
|
+
accepted,
|
|
3095
|
+
});
|
|
3096
|
+
}
|
|
3097
|
+
return [...accepted.values()];
|
|
3098
|
+
}
|
|
3099
|
+
static groupGoverningProtocolConfigCandidatesByProtocol(candidates) {
|
|
3100
|
+
const candidatesByProtocol = new Map();
|
|
3101
|
+
for (const candidate of candidates) {
|
|
3102
|
+
const protocol = SyncEngineLevel.protocolsConfigureProtocol(candidate.message);
|
|
3103
|
+
const existing = candidatesByProtocol.get(protocol);
|
|
3104
|
+
if (existing !== undefined && SyncEngineLevel.isProtocolConfigCandidateAtLeastAsNew(existing, candidate)) {
|
|
3105
|
+
continue;
|
|
3106
|
+
}
|
|
3107
|
+
candidatesByProtocol.set(protocol, candidate);
|
|
3108
|
+
}
|
|
3109
|
+
return candidatesByProtocol;
|
|
3110
|
+
}
|
|
3111
|
+
static isProtocolConfigCandidateAtLeastAsNew(existing, candidate) {
|
|
3112
|
+
const existingTimestamp = existing.message.descriptor.messageTimestamp;
|
|
3113
|
+
const candidateTimestamp = candidate.message.descriptor.messageTimestamp;
|
|
3114
|
+
if (existingTimestamp !== candidateTimestamp) {
|
|
3115
|
+
return existingTimestamp > candidateTimestamp;
|
|
3116
|
+
}
|
|
3117
|
+
return lexicographicalCompare(existing.messageCid, candidate.messageCid) >= 0;
|
|
3118
|
+
}
|
|
3119
|
+
static takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols) {
|
|
3120
|
+
while (pendingProtocols.length > 0) {
|
|
3121
|
+
const protocol = pendingProtocols.shift();
|
|
3122
|
+
if (visitedProtocols.has(protocol)) {
|
|
3123
|
+
continue;
|
|
3124
|
+
}
|
|
3125
|
+
visitedProtocols.add(protocol);
|
|
3126
|
+
return protocol;
|
|
3127
|
+
}
|
|
3128
|
+
return undefined;
|
|
3129
|
+
}
|
|
3130
|
+
static acceptProtocolConfigCandidates({ protocol, candidatesByProtocol, visitedProtocols, pendingProtocols, accepted, }) {
|
|
3131
|
+
const candidate = candidatesByProtocol.get(protocol);
|
|
3132
|
+
if (candidate === undefined) {
|
|
1974
3133
|
return;
|
|
1975
3134
|
}
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
3135
|
+
accepted.set(candidate.messageCid, candidate);
|
|
3136
|
+
SyncEngineLevel.queueUnvisitedProtocols(SyncEngineLevel.protocolsConfigureUses(candidate.message), visitedProtocols, pendingProtocols);
|
|
3137
|
+
}
|
|
3138
|
+
static queueUnvisitedProtocols(protocols, visitedProtocols, pendingProtocols) {
|
|
3139
|
+
for (const protocol of protocols) {
|
|
3140
|
+
if (!visitedProtocols.has(protocol)) {
|
|
3141
|
+
pendingProtocols.push(protocol);
|
|
3142
|
+
}
|
|
3143
|
+
}
|
|
3144
|
+
}
|
|
3145
|
+
static protocolSetScopeForProjectedDependencies(dependencies) {
|
|
3146
|
+
// Verification above is the security boundary. This protocolSet scope only
|
|
3147
|
+
// routes already-verified config dependencies through the existing
|
|
3148
|
+
// pull/apply path, which expects every prefetched message to be accepted by
|
|
3149
|
+
// the supplied sync scope before it reaches processRawMessage().
|
|
3150
|
+
const protocols = SyncEngineLevel.dedupeStrings(dependencies
|
|
3151
|
+
.flatMap(dependency => SyncEngineLevel.projectedDependencyProtocols(dependency))).sort(lexicographicalCompare);
|
|
3152
|
+
if (protocols.length === 0) {
|
|
3153
|
+
throw new Error('SyncEngineLevel: projected dependency hints contained no protocols.');
|
|
3154
|
+
}
|
|
3155
|
+
return {
|
|
3156
|
+
kind: 'protocolSet',
|
|
3157
|
+
protocols: protocols,
|
|
3158
|
+
};
|
|
3159
|
+
}
|
|
3160
|
+
static projectedDependencyProtocols(dependency) {
|
|
3161
|
+
if (dependency.message === undefined) {
|
|
3162
|
+
return [];
|
|
3163
|
+
}
|
|
3164
|
+
const protocol = dependency.dependencyClass === 'protocolsConfigure'
|
|
3165
|
+
? SyncEngineLevel.protocolsConfigureProtocolFromGenericMessage(dependency.message)
|
|
3166
|
+
: SyncEngineLevel.recordsWriteProtocol(dependency.message);
|
|
3167
|
+
return protocol === undefined ? [] : [protocol];
|
|
3168
|
+
}
|
|
3169
|
+
static hasDependencyPayloadBytes(dependency) {
|
|
3170
|
+
if (dependency.encodedData !== undefined) {
|
|
3171
|
+
return true;
|
|
3172
|
+
}
|
|
3173
|
+
const message = dependency.message;
|
|
3174
|
+
return message !== undefined && 'encodedData' in message;
|
|
3175
|
+
}
|
|
3176
|
+
static dedupeDependencyEntries(dependencies) {
|
|
3177
|
+
const deduped = new Map();
|
|
3178
|
+
for (const dependency of dependencies) {
|
|
3179
|
+
deduped.set(dependency.messageCid, dependency);
|
|
3180
|
+
}
|
|
3181
|
+
return [...deduped.values()];
|
|
3182
|
+
}
|
|
3183
|
+
pushLocalDiffEntries(target, onlyLocal, permissionGrantIds, shouldContinue) {
|
|
3184
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3185
|
+
yield this.pushMessages({
|
|
3186
|
+
did: target.did,
|
|
3187
|
+
dwnUrl: target.dwnUrl,
|
|
3188
|
+
delegateDid: target.delegateDid,
|
|
3189
|
+
permissionGrantIds,
|
|
3190
|
+
messageCids: SyncEngineLevel.dedupeStrings(onlyLocal),
|
|
3191
|
+
});
|
|
3192
|
+
return (shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false;
|
|
3193
|
+
});
|
|
3194
|
+
}
|
|
3195
|
+
verifyProtocolSetConvergence(target, changedProtocols, permissionGrantIds, shouldContinue) {
|
|
3196
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3197
|
+
for (const protocol of changedProtocols) {
|
|
3198
|
+
const roots = yield this.getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue);
|
|
3199
|
+
if (!roots) {
|
|
3200
|
+
return { aborted: true };
|
|
3201
|
+
}
|
|
3202
|
+
if (roots.localRoot !== roots.remoteRoot) {
|
|
3203
|
+
return { converged: false };
|
|
3204
|
+
}
|
|
3205
|
+
}
|
|
3206
|
+
return { converged: true };
|
|
3207
|
+
});
|
|
3208
|
+
}
|
|
3209
|
+
static dedupeRemoteEntries(entries) {
|
|
3210
|
+
const seen = new Set();
|
|
3211
|
+
const unique = [];
|
|
3212
|
+
for (const entry of entries) {
|
|
3213
|
+
if (seen.has(entry.messageCid)) {
|
|
3214
|
+
continue;
|
|
3215
|
+
}
|
|
3216
|
+
seen.add(entry.messageCid);
|
|
3217
|
+
unique.push(entry);
|
|
1981
3218
|
}
|
|
1982
|
-
|
|
1983
|
-
pushRuntime.timer = undefined;
|
|
1984
|
-
void this.flushPendingPushesForLink(targetKey);
|
|
1985
|
-
}, delayMs);
|
|
3219
|
+
return unique;
|
|
1986
3220
|
}
|
|
1987
|
-
|
|
1988
|
-
return new
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
3221
|
+
static dedupeStrings(values) {
|
|
3222
|
+
return [...new Set(values)];
|
|
3223
|
+
}
|
|
3224
|
+
clearRootConvergenceDeadLettersForScope(tenantDid, remoteEndpoint, scope) {
|
|
3225
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3226
|
+
if (scope.kind === 'recordsProjection' || (scope.kind === 'protocolSet' && scope.protocols.length > 1)) {
|
|
3227
|
+
// Batched multi-protocol and projected pulls pass the full scope to
|
|
3228
|
+
// pullMessages, so pull dead letters can be recorded without a single
|
|
3229
|
+
// protocol bucket.
|
|
3230
|
+
yield this.clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint);
|
|
3231
|
+
}
|
|
3232
|
+
for (const protocol of this.getReconcileProtocols(scope)) {
|
|
3233
|
+
yield this.clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint, protocol);
|
|
3234
|
+
}
|
|
1995
3235
|
});
|
|
1996
3236
|
}
|
|
1997
3237
|
/**
|
|
@@ -2068,18 +3308,28 @@ export class SyncEngineLevel {
|
|
|
2068
3308
|
// closure's captured `link` reference may no longer be the active
|
|
2069
3309
|
// link object. Bail before mutating the replacement's state.
|
|
2070
3310
|
const isStaleLink = () => this._activeLinks.get(linkKey) !== link;
|
|
2071
|
-
const
|
|
3311
|
+
const shouldContinue = () => this._engineGeneration === generation &&
|
|
3312
|
+
!isStaleLink() &&
|
|
3313
|
+
link.status === 'live';
|
|
3314
|
+
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, scope, authorization } = link;
|
|
3315
|
+
const eventScope = syncEventScope(scope);
|
|
2072
3316
|
try {
|
|
2073
|
-
const reconcileOutcome = yield this.
|
|
3317
|
+
const reconcileOutcome = yield this.reconcileProjectionTarget({
|
|
3318
|
+
did,
|
|
3319
|
+
dwnUrl,
|
|
3320
|
+
delegateDid,
|
|
3321
|
+
scope,
|
|
3322
|
+
authorization,
|
|
3323
|
+
}, { verifyConvergence: true }, shouldContinue);
|
|
2074
3324
|
if (reconcileOutcome.aborted || isStaleLink()) {
|
|
2075
3325
|
return;
|
|
2076
3326
|
}
|
|
2077
3327
|
if (reconcileOutcome.converged) {
|
|
2078
3328
|
yield this.ledger.clearNeedsReconcile(link);
|
|
2079
|
-
// SMT roots match
|
|
2080
|
-
//
|
|
2081
|
-
|
|
2082
|
-
this.emitEvent({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
3329
|
+
// SMT roots match, so transport/apply failures for this link may no
|
|
3330
|
+
// longer be current. Closure failures are not cleared by root equality.
|
|
3331
|
+
yield this.clearRootConvergenceDeadLettersForScope(did, dwnUrl, scope);
|
|
3332
|
+
this.emitEvent(Object.assign({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope));
|
|
2083
3333
|
}
|
|
2084
3334
|
else {
|
|
2085
3335
|
// Roots still differ — retry after a delay. This can happen when
|
|
@@ -2117,101 +3367,11 @@ export class SyncEngineLevel {
|
|
|
2117
3367
|
* Live-mode subscription methods (`openLivePullSubscription`,
|
|
2118
3368
|
* `openLocalPushSubscription`) receive `linkKey` directly and never
|
|
2119
3369
|
* call this. The remaining callers are poll-mode `sync()` and the
|
|
2120
|
-
* live-mode startup/error paths that already have
|
|
2121
|
-
*
|
|
2122
|
-
* The `undefined` fallback (which produces a legacy cursor key) exists
|
|
2123
|
-
* only for the no-protocol full-tenant targets in poll mode.
|
|
2124
|
-
*/
|
|
2125
|
-
buildLinkKey(did, dwnUrl, scopeIdOrProtocol) {
|
|
2126
|
-
return scopeIdOrProtocol ? buildLinkId(did, dwnUrl, scopeIdOrProtocol) : buildLegacyCursorKey(did, dwnUrl);
|
|
2127
|
-
}
|
|
2128
|
-
/**
|
|
2129
|
-
* @deprecated Used by poll-mode sync and one-time migration only. Live mode
|
|
2130
|
-
* uses ReplicationLedger checkpoints. Handles migration from old string cursors:
|
|
2131
|
-
* if the stored value is a bare string (pre-ProgressToken format), it is treated
|
|
2132
|
-
* as absent — the sync engine will do a full SMT reconciliation on first startup
|
|
2133
|
-
* after upgrade, which is correct and safe.
|
|
2134
|
-
*/
|
|
2135
|
-
getCursor(key) {
|
|
2136
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
2137
|
-
const cursors = this._db.sublevel('syncCursors');
|
|
2138
|
-
try {
|
|
2139
|
-
const raw = yield cursors.get(key);
|
|
2140
|
-
try {
|
|
2141
|
-
const parsed = JSON.parse(raw);
|
|
2142
|
-
if (parsed && typeof parsed === 'object' &&
|
|
2143
|
-
typeof parsed.streamId === 'string' && parsed.streamId.length > 0 &&
|
|
2144
|
-
typeof parsed.epoch === 'string' && parsed.epoch.length > 0 &&
|
|
2145
|
-
typeof parsed.position === 'string' && parsed.position.length > 0 &&
|
|
2146
|
-
typeof parsed.messageCid === 'string' && parsed.messageCid.length > 0) {
|
|
2147
|
-
return parsed;
|
|
2148
|
-
}
|
|
2149
|
-
}
|
|
2150
|
-
catch (_a) {
|
|
2151
|
-
// Not valid JSON (old string cursor) — fall through to delete.
|
|
2152
|
-
}
|
|
2153
|
-
// Entry exists but is unparseable or has invalid/empty fields. Delete it
|
|
2154
|
-
// so subsequent startups don't re-check it on every launch.
|
|
2155
|
-
yield this.deleteLegacyCursor(key);
|
|
2156
|
-
return undefined;
|
|
2157
|
-
}
|
|
2158
|
-
catch (error) {
|
|
2159
|
-
const e = error;
|
|
2160
|
-
if (e.code === 'LEVEL_NOT_FOUND') {
|
|
2161
|
-
return undefined;
|
|
2162
|
-
}
|
|
2163
|
-
throw error;
|
|
2164
|
-
}
|
|
2165
|
-
});
|
|
2166
|
-
}
|
|
2167
|
-
/**
|
|
2168
|
-
* Delete a legacy cursor from the old syncCursors sublevel.
|
|
2169
|
-
* Called as part of one-time migration to ReplicationLedger.
|
|
2170
|
-
*/
|
|
2171
|
-
deleteLegacyCursor(key) {
|
|
2172
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
2173
|
-
const cursors = this._db.sublevel('syncCursors');
|
|
2174
|
-
try {
|
|
2175
|
-
yield cursors.del(key);
|
|
2176
|
-
}
|
|
2177
|
-
catch (_a) {
|
|
2178
|
-
// Best-effort — ignore LEVEL_NOT_FOUND and transient I/O errors alike.
|
|
2179
|
-
// A failed delete leaves the bad entry for one more re-check on the
|
|
2180
|
-
// next startup, which is harmless.
|
|
2181
|
-
}
|
|
2182
|
-
});
|
|
2183
|
-
}
|
|
2184
|
-
// ---------------------------------------------------------------------------
|
|
2185
|
-
// Utility helpers
|
|
2186
|
-
// ---------------------------------------------------------------------------
|
|
2187
|
-
/**
|
|
2188
|
-
* Extracts a ReadableStream from a MessageEvent if it contains a
|
|
2189
|
-
* RecordsWrite with data — either as an inline `encodedData` field
|
|
2190
|
-
* (for records <= 30 KB) or as a pre-existing data stream.
|
|
3370
|
+
* live-mode startup/error paths that already have a projection ID and
|
|
3371
|
+
* authorization epoch.
|
|
2191
3372
|
*/
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
return undefined;
|
|
2195
|
-
}
|
|
2196
|
-
// Check for inline base64url-encoded data (small records from EventLog).
|
|
2197
|
-
// Delete the transport-level field so the DWN schema validator does not
|
|
2198
|
-
// reject the message for having unevaluated properties.
|
|
2199
|
-
const encodedData = event.message.encodedData;
|
|
2200
|
-
if (encodedData) {
|
|
2201
|
-
delete event.message.encodedData;
|
|
2202
|
-
const bytes = Encoder.base64UrlToBytes(encodedData);
|
|
2203
|
-
return new ReadableStream({
|
|
2204
|
-
start(controller) {
|
|
2205
|
-
controller.enqueue(bytes);
|
|
2206
|
-
controller.close();
|
|
2207
|
-
}
|
|
2208
|
-
});
|
|
2209
|
-
}
|
|
2210
|
-
// Check for a pre-existing data stream (e.g. from a direct message read).
|
|
2211
|
-
if (event.data) {
|
|
2212
|
-
return event.data;
|
|
2213
|
-
}
|
|
2214
|
-
return undefined;
|
|
3373
|
+
buildLinkKey(did, dwnUrl, projectionId, authorizationEpoch) {
|
|
3374
|
+
return buildLinkId(did, dwnUrl, projectionId, authorizationEpoch);
|
|
2215
3375
|
}
|
|
2216
3376
|
// ---------------------------------------------------------------------------
|
|
2217
3377
|
// Default Hash Cache
|
|
@@ -2270,7 +3430,7 @@ export class SyncEngineLevel {
|
|
|
2270
3430
|
*
|
|
2271
3431
|
* Returns a hex-encoded root hash string.
|
|
2272
3432
|
*/
|
|
2273
|
-
getLocalRoot(did, delegateDid, protocol) {
|
|
3433
|
+
getLocalRoot(did, delegateDid, protocol, permissionGrantIds) {
|
|
2274
3434
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2275
3435
|
var _a;
|
|
2276
3436
|
const si = this.stateIndex;
|
|
@@ -2281,7 +3441,6 @@ export class SyncEngineLevel {
|
|
|
2281
3441
|
return hashToHex(rootHash);
|
|
2282
3442
|
}
|
|
2283
3443
|
// Remote mode fallback: go through processRequest → RPC.
|
|
2284
|
-
const permissionGrantId = yield this.getSyncPermissionGrantId(did, delegateDid, protocol);
|
|
2285
3444
|
const response = yield this.agent.dwn.processRequest({
|
|
2286
3445
|
author: did,
|
|
2287
3446
|
target: did,
|
|
@@ -2290,7 +3449,7 @@ export class SyncEngineLevel {
|
|
|
2290
3449
|
messageParams: {
|
|
2291
3450
|
action: 'root',
|
|
2292
3451
|
protocol,
|
|
2293
|
-
|
|
3452
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds),
|
|
2294
3453
|
}
|
|
2295
3454
|
});
|
|
2296
3455
|
const reply = response.reply;
|
|
@@ -2301,10 +3460,9 @@ export class SyncEngineLevel {
|
|
|
2301
3460
|
* Get the SMT root hash from a remote DWN via a MessagesSync 'root' action.
|
|
2302
3461
|
* Returns a hex-encoded root hash string.
|
|
2303
3462
|
*/
|
|
2304
|
-
getRemoteRoot(did, dwnUrl, delegateDid, protocol) {
|
|
3463
|
+
getRemoteRoot(did, dwnUrl, delegateDid, protocol, permissionGrantIds) {
|
|
2305
3464
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2306
3465
|
var _a;
|
|
2307
|
-
const permissionGrantId = yield this.getSyncPermissionGrantId(did, delegateDid, protocol);
|
|
2308
3466
|
const syncMessage = yield this.agent.dwn.processRequest({
|
|
2309
3467
|
store: false,
|
|
2310
3468
|
author: did,
|
|
@@ -2314,7 +3472,59 @@ export class SyncEngineLevel {
|
|
|
2314
3472
|
messageParams: {
|
|
2315
3473
|
action: 'root',
|
|
2316
3474
|
protocol,
|
|
2317
|
-
|
|
3475
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds)
|
|
3476
|
+
}
|
|
3477
|
+
});
|
|
3478
|
+
const reply = yield this.agent.rpc.sendDwnRequest({
|
|
3479
|
+
dwnUrl,
|
|
3480
|
+
targetDid: did,
|
|
3481
|
+
message: syncMessage.message,
|
|
3482
|
+
});
|
|
3483
|
+
return (_a = reply.root) !== null && _a !== void 0 ? _a : '';
|
|
3484
|
+
});
|
|
3485
|
+
}
|
|
3486
|
+
getLocalProjectedRoot(did, delegateDid, scopes, permissionGrantIds) {
|
|
3487
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3488
|
+
var _a;
|
|
3489
|
+
if (this.stateIndex) {
|
|
3490
|
+
// Local projected roots use the already-derived scope directly. The
|
|
3491
|
+
// remote root/diff request still re-authorizes the invoked grant set.
|
|
3492
|
+
return RecordsProjection.getRootHex({
|
|
3493
|
+
tenant: did,
|
|
3494
|
+
messageStore: this.agent.dwn.node.storage.messageStore,
|
|
3495
|
+
scopes,
|
|
3496
|
+
});
|
|
3497
|
+
}
|
|
3498
|
+
const response = yield this.agent.dwn.processRequest({
|
|
3499
|
+
author: did,
|
|
3500
|
+
target: did,
|
|
3501
|
+
messageType: DwnInterface.MessagesSync,
|
|
3502
|
+
granteeDid: delegateDid,
|
|
3503
|
+
messageParams: {
|
|
3504
|
+
action: 'root',
|
|
3505
|
+
projectionRootVersion: RECORDS_PROJECTION_ROOT_VERSION,
|
|
3506
|
+
projectionScopes: [...scopes],
|
|
3507
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds),
|
|
3508
|
+
}
|
|
3509
|
+
});
|
|
3510
|
+
const reply = response.reply;
|
|
3511
|
+
return (_a = reply.root) !== null && _a !== void 0 ? _a : '';
|
|
3512
|
+
});
|
|
3513
|
+
}
|
|
3514
|
+
getRemoteProjectedRoot(did, dwnUrl, delegateDid, scopes, permissionGrantIds) {
|
|
3515
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3516
|
+
var _a;
|
|
3517
|
+
const syncMessage = yield this.agent.dwn.processRequest({
|
|
3518
|
+
store: false,
|
|
3519
|
+
author: did,
|
|
3520
|
+
target: did,
|
|
3521
|
+
messageType: DwnInterface.MessagesSync,
|
|
3522
|
+
granteeDid: delegateDid,
|
|
3523
|
+
messageParams: {
|
|
3524
|
+
action: 'root',
|
|
3525
|
+
projectionRootVersion: RECORDS_PROJECTION_ROOT_VERSION,
|
|
3526
|
+
projectionScopes: [...scopes],
|
|
3527
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds),
|
|
2318
3528
|
}
|
|
2319
3529
|
});
|
|
2320
3530
|
const reply = yield this.agent.rpc.sendDwnRequest({
|
|
@@ -2343,43 +3553,62 @@ export class SyncEngineLevel {
|
|
|
2343
3553
|
* This replaces `walkTreeDiff()` which required one HTTP call per tree node.
|
|
2344
3554
|
*/
|
|
2345
3555
|
diffWithRemote(_a) {
|
|
2346
|
-
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, protocol }) {
|
|
2347
|
-
var _b, _c;
|
|
3556
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, protocol, permissionGrantIds }) {
|
|
2348
3557
|
// Step 1: Collect local subtree hashes at BATCHED_DIFF_DEPTH directly from StateIndex.
|
|
2349
|
-
const localHashes = yield this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH);
|
|
3558
|
+
const localHashes = yield this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH, permissionGrantIds);
|
|
2350
3559
|
// Step 2: Send a single 'diff' request to the remote with our hashes.
|
|
2351
|
-
const
|
|
3560
|
+
const messageParams = {
|
|
3561
|
+
action: 'diff',
|
|
3562
|
+
protocol,
|
|
3563
|
+
hashes: localHashes,
|
|
3564
|
+
depth: BATCHED_DIFF_DEPTH,
|
|
3565
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds),
|
|
3566
|
+
};
|
|
3567
|
+
// Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
|
|
3568
|
+
// Reuse the same grant set from step 2.
|
|
3569
|
+
return this.diffRemoteMessages({ did, dwnUrl, delegateDid }, messageParams, prefix => this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantIds), 'diff');
|
|
3570
|
+
});
|
|
3571
|
+
}
|
|
3572
|
+
diffProjectedWithRemote(_a) {
|
|
3573
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, scopes, permissionGrantIds }) {
|
|
3574
|
+
const localHashes = yield this.collectLocalProjectedSubtreeHashes(did, scopes, BATCHED_DIFF_DEPTH, delegateDid, permissionGrantIds);
|
|
3575
|
+
const messageParams = {
|
|
3576
|
+
action: 'diff',
|
|
3577
|
+
projectionRootVersion: RECORDS_PROJECTION_ROOT_VERSION,
|
|
3578
|
+
projectionScopes: [...scopes],
|
|
3579
|
+
hashes: localHashes,
|
|
3580
|
+
depth: BATCHED_DIFF_DEPTH,
|
|
3581
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds),
|
|
3582
|
+
};
|
|
3583
|
+
return this.diffRemoteMessages({ did, dwnUrl, delegateDid }, messageParams, prefix => this.getLocalProjectedLeaves(did, prefix, delegateDid, scopes, permissionGrantIds), 'projected diff');
|
|
3584
|
+
});
|
|
3585
|
+
}
|
|
3586
|
+
diffRemoteMessages(target, messageParams, getLocalLeavesForPrefix, operationName) {
|
|
3587
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3588
|
+
var _a, _b, _c;
|
|
2352
3589
|
const syncMessage = yield this.agent.dwn.processRequest({
|
|
2353
3590
|
store: false,
|
|
2354
|
-
author: did,
|
|
2355
|
-
target: did,
|
|
3591
|
+
author: target.did,
|
|
3592
|
+
target: target.did,
|
|
2356
3593
|
messageType: DwnInterface.MessagesSync,
|
|
2357
|
-
granteeDid: delegateDid,
|
|
2358
|
-
messageParams
|
|
2359
|
-
action: 'diff',
|
|
2360
|
-
protocol,
|
|
2361
|
-
hashes: localHashes,
|
|
2362
|
-
depth: BATCHED_DIFF_DEPTH,
|
|
2363
|
-
permissionGrantId,
|
|
2364
|
-
}
|
|
3594
|
+
granteeDid: target.delegateDid,
|
|
3595
|
+
messageParams,
|
|
2365
3596
|
});
|
|
2366
3597
|
const reply = yield this.agent.rpc.sendDwnRequest({
|
|
2367
|
-
dwnUrl,
|
|
2368
|
-
targetDid: did,
|
|
3598
|
+
dwnUrl: target.dwnUrl,
|
|
3599
|
+
targetDid: target.did,
|
|
2369
3600
|
message: syncMessage.message,
|
|
2370
3601
|
});
|
|
2371
3602
|
if (reply.status.code !== 200) {
|
|
2372
|
-
throw new Error(`SyncEngineLevel:
|
|
3603
|
+
throw new Error(`SyncEngineLevel: ${operationName} failed with ${reply.status.code}: ${reply.status.detail}`);
|
|
2373
3604
|
}
|
|
2374
|
-
// Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
|
|
2375
|
-
// Reuse the same grant ID from step 2 (avoids redundant lookup).
|
|
2376
|
-
const permissionGrantIdForLeaves = permissionGrantId;
|
|
2377
3605
|
const onlyLocalCids = [];
|
|
2378
|
-
for (const prefix of (
|
|
2379
|
-
const leaves = yield
|
|
3606
|
+
for (const prefix of (_a = reply.onlyLocal) !== null && _a !== void 0 ? _a : []) {
|
|
3607
|
+
const leaves = yield getLocalLeavesForPrefix(prefix);
|
|
2380
3608
|
onlyLocalCids.push(...leaves);
|
|
2381
3609
|
}
|
|
2382
3610
|
return {
|
|
3611
|
+
dependencies: (_b = reply.dependencies) !== null && _b !== void 0 ? _b : [],
|
|
2383
3612
|
onlyRemote: (_c = reply.onlyRemote) !== null && _c !== void 0 ? _c : [],
|
|
2384
3613
|
onlyLocal: onlyLocalCids,
|
|
2385
3614
|
};
|
|
@@ -2393,7 +3622,7 @@ export class SyncEngineLevel {
|
|
|
2393
3622
|
* Uses direct StateIndex access in local mode. In remote mode, falls back
|
|
2394
3623
|
* to `getLocalSubtreeHash` which routes through RPC.
|
|
2395
3624
|
*/
|
|
2396
|
-
collectLocalSubtreeHashes(did, protocol, depth) {
|
|
3625
|
+
collectLocalSubtreeHashes(did, protocol, depth, permissionGrantIds) {
|
|
2397
3626
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2398
3627
|
const result = {};
|
|
2399
3628
|
const defaultHash = yield this.getDefaultHashHex(depth);
|
|
@@ -2410,7 +3639,7 @@ export class SyncEngineLevel {
|
|
|
2410
3639
|
}
|
|
2411
3640
|
else {
|
|
2412
3641
|
// Remote mode fallback.
|
|
2413
|
-
hexHash = yield this.getLocalSubtreeHash(did, prefix, undefined, protocol);
|
|
3642
|
+
hexHash = yield this.getLocalSubtreeHash(did, prefix, undefined, protocol, permissionGrantIds);
|
|
2414
3643
|
}
|
|
2415
3644
|
if (hexHash === defaultHash) {
|
|
2416
3645
|
// Empty subtree — omit from the map.
|
|
@@ -2430,13 +3659,50 @@ export class SyncEngineLevel {
|
|
|
2430
3659
|
return result;
|
|
2431
3660
|
});
|
|
2432
3661
|
}
|
|
3662
|
+
collectLocalProjectedSubtreeHashes(did, scopes, depth, delegateDid, permissionGrantIds) {
|
|
3663
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3664
|
+
const result = {};
|
|
3665
|
+
const snapshot = this.stateIndex
|
|
3666
|
+
? yield RecordsProjection.createSnapshot({
|
|
3667
|
+
tenant: did,
|
|
3668
|
+
messageStore: this.agent.dwn.node.storage.messageStore,
|
|
3669
|
+
scopes,
|
|
3670
|
+
})
|
|
3671
|
+
: undefined;
|
|
3672
|
+
try {
|
|
3673
|
+
const walk = (prefix, currentDepth) => __awaiter(this, void 0, void 0, function* () {
|
|
3674
|
+
const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
|
|
3675
|
+
const hexHash = snapshot
|
|
3676
|
+
? hashToHex(yield snapshot.getSubtreeHash(bitPath))
|
|
3677
|
+
: yield this.getLocalProjectedSubtreeHash(did, prefix, delegateDid, scopes, permissionGrantIds);
|
|
3678
|
+
const defaultHash = yield this.getDefaultHashHex(currentDepth);
|
|
3679
|
+
if (hexHash === defaultHash) {
|
|
3680
|
+
return;
|
|
3681
|
+
}
|
|
3682
|
+
if (currentDepth >= depth) {
|
|
3683
|
+
result[prefix] = hexHash;
|
|
3684
|
+
return;
|
|
3685
|
+
}
|
|
3686
|
+
yield Promise.all([
|
|
3687
|
+
walk(prefix + '0', currentDepth + 1),
|
|
3688
|
+
walk(prefix + '1', currentDepth + 1),
|
|
3689
|
+
]);
|
|
3690
|
+
});
|
|
3691
|
+
yield walk('', 0);
|
|
3692
|
+
return result;
|
|
3693
|
+
}
|
|
3694
|
+
finally {
|
|
3695
|
+
yield (snapshot === null || snapshot === void 0 ? void 0 : snapshot.close());
|
|
3696
|
+
}
|
|
3697
|
+
});
|
|
3698
|
+
}
|
|
2433
3699
|
/**
|
|
2434
3700
|
* Get the subtree hash at a given bit prefix from the local DWN.
|
|
2435
3701
|
*
|
|
2436
3702
|
* In local mode: queries the StateIndex directly.
|
|
2437
3703
|
* In remote mode: constructs a signed MessagesSync message and routes through RPC.
|
|
2438
3704
|
*/
|
|
2439
|
-
getLocalSubtreeHash(did, prefix, delegateDid, protocol,
|
|
3705
|
+
getLocalSubtreeHash(did, prefix, delegateDid, protocol, permissionGrantIds) {
|
|
2440
3706
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2441
3707
|
var _a;
|
|
2442
3708
|
const si = this.stateIndex;
|
|
@@ -2457,7 +3723,27 @@ export class SyncEngineLevel {
|
|
|
2457
3723
|
action: 'subtree',
|
|
2458
3724
|
prefix,
|
|
2459
3725
|
protocol,
|
|
2460
|
-
|
|
3726
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds)
|
|
3727
|
+
}
|
|
3728
|
+
});
|
|
3729
|
+
const reply = response.reply;
|
|
3730
|
+
return (_a = reply.hash) !== null && _a !== void 0 ? _a : '';
|
|
3731
|
+
});
|
|
3732
|
+
}
|
|
3733
|
+
getLocalProjectedSubtreeHash(did, prefix, delegateDid, scopes, permissionGrantIds) {
|
|
3734
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3735
|
+
var _a;
|
|
3736
|
+
const response = yield this.agent.dwn.processRequest({
|
|
3737
|
+
author: did,
|
|
3738
|
+
target: did,
|
|
3739
|
+
messageType: DwnInterface.MessagesSync,
|
|
3740
|
+
granteeDid: delegateDid,
|
|
3741
|
+
messageParams: {
|
|
3742
|
+
action: 'subtree',
|
|
3743
|
+
prefix,
|
|
3744
|
+
projectionRootVersion: RECORDS_PROJECTION_ROOT_VERSION,
|
|
3745
|
+
projectionScopes: [...scopes],
|
|
3746
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds)
|
|
2461
3747
|
}
|
|
2462
3748
|
});
|
|
2463
3749
|
const reply = response.reply;
|
|
@@ -2470,7 +3756,7 @@ export class SyncEngineLevel {
|
|
|
2470
3756
|
* In local mode: queries the StateIndex directly.
|
|
2471
3757
|
* In remote mode: constructs a signed MessagesSync message and routes through RPC.
|
|
2472
3758
|
*/
|
|
2473
|
-
getLocalLeaves(did, prefix, delegateDid, protocol,
|
|
3759
|
+
getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantIds) {
|
|
2474
3760
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2475
3761
|
var _a;
|
|
2476
3762
|
const si = this.stateIndex;
|
|
@@ -2490,7 +3776,35 @@ export class SyncEngineLevel {
|
|
|
2490
3776
|
action: 'leaves',
|
|
2491
3777
|
prefix,
|
|
2492
3778
|
protocol,
|
|
2493
|
-
|
|
3779
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds)
|
|
3780
|
+
}
|
|
3781
|
+
});
|
|
3782
|
+
const reply = response.reply;
|
|
3783
|
+
return (_a = reply.entries) !== null && _a !== void 0 ? _a : [];
|
|
3784
|
+
});
|
|
3785
|
+
}
|
|
3786
|
+
getLocalProjectedLeaves(did, prefix, delegateDid, scopes, permissionGrantIds) {
|
|
3787
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3788
|
+
var _a;
|
|
3789
|
+
if (this.stateIndex) {
|
|
3790
|
+
return RecordsProjection.getLeaves({
|
|
3791
|
+
tenant: did,
|
|
3792
|
+
messageStore: this.agent.dwn.node.storage.messageStore,
|
|
3793
|
+
scopes,
|
|
3794
|
+
prefix: SyncEngineLevel.parseBitPrefix(prefix),
|
|
3795
|
+
});
|
|
3796
|
+
}
|
|
3797
|
+
const response = yield this.agent.dwn.processRequest({
|
|
3798
|
+
author: did,
|
|
3799
|
+
target: did,
|
|
3800
|
+
messageType: DwnInterface.MessagesSync,
|
|
3801
|
+
granteeDid: delegateDid,
|
|
3802
|
+
messageParams: {
|
|
3803
|
+
action: 'leaves',
|
|
3804
|
+
prefix,
|
|
3805
|
+
projectionRootVersion: RECORDS_PROJECTION_ROOT_VERSION,
|
|
3806
|
+
projectionScopes: [...scopes],
|
|
3807
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds)
|
|
2494
3808
|
}
|
|
2495
3809
|
});
|
|
2496
3810
|
const reply = response.reply;
|
|
@@ -2509,25 +3823,109 @@ export class SyncEngineLevel {
|
|
|
2509
3823
|
* Only `messageCids` that were NOT prefetched are fetched individually.
|
|
2510
3824
|
*/
|
|
2511
3825
|
pullMessages(_a) {
|
|
2512
|
-
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, protocol, messageCids, prefetched }) {
|
|
3826
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, protocol, scope, permissionGrantIds, messageCids, prefetched, verifiedInitialWrites, shouldContinue, }) {
|
|
3827
|
+
const acceptanceScope = scope !== null && scope !== void 0 ? scope : (protocol === undefined
|
|
3828
|
+
? { kind: 'full' }
|
|
3829
|
+
: { kind: 'protocolSet', protocols: [protocol] });
|
|
3830
|
+
const rejectedPullEntries = new Map();
|
|
2513
3831
|
const failedCids = yield pullMessages({
|
|
2514
|
-
did,
|
|
3832
|
+
did,
|
|
3833
|
+
dwnUrl,
|
|
3834
|
+
delegateDid,
|
|
3835
|
+
permissionGrantIds,
|
|
3836
|
+
messageCids,
|
|
3837
|
+
prefetched,
|
|
3838
|
+
shouldContinue,
|
|
2515
3839
|
agent: this.agent,
|
|
2516
|
-
|
|
3840
|
+
acceptEntry: (entry, entries) => __awaiter(this, void 0, void 0, function* () {
|
|
3841
|
+
const result = yield this.acceptPulledSyncEntry(did, acceptanceScope, entry, entries, verifiedInitialWrites);
|
|
3842
|
+
if (!result.accepted) {
|
|
3843
|
+
rejectedPullEntries.set(yield getMessageCid(entry.message), result);
|
|
3844
|
+
}
|
|
3845
|
+
return result.accepted;
|
|
3846
|
+
}),
|
|
2517
3847
|
});
|
|
2518
3848
|
// Record permanently failed pull entries in the dead letter store.
|
|
2519
3849
|
for (const cid of failedCids) {
|
|
3850
|
+
const rejection = rejectedPullEntries.get(cid);
|
|
2520
3851
|
yield this.recordDeadLetter({
|
|
2521
3852
|
messageCid: cid,
|
|
2522
3853
|
tenantDid: did,
|
|
2523
3854
|
remoteEndpoint: dwnUrl,
|
|
2524
3855
|
protocol,
|
|
2525
|
-
category: 'pull-processing',
|
|
2526
|
-
|
|
3856
|
+
category: rejection ? 'pull-scope-rejected' : 'pull-processing',
|
|
3857
|
+
errorCode: rejection === null || rejection === void 0 ? void 0 : rejection.classification,
|
|
3858
|
+
errorDetail: rejection
|
|
3859
|
+
? `pulled message rejected by ${rejection.classification} sync scope gate`
|
|
3860
|
+
: 'pull processing failed after retry passes exhausted',
|
|
2527
3861
|
});
|
|
2528
3862
|
}
|
|
2529
3863
|
});
|
|
2530
3864
|
}
|
|
3865
|
+
acceptPulledSyncEntry(did_1, scope_1, entry_1, entries_1) {
|
|
3866
|
+
return __awaiter(this, arguments, void 0, function* (did, scope, entry, entries, verifiedInitialWrites = []) {
|
|
3867
|
+
if (scope.kind === 'full') {
|
|
3868
|
+
return { accepted: true };
|
|
3869
|
+
}
|
|
3870
|
+
const initialWrite = yield this.resolvePulledDeleteInitialWrite(did, entry.message, entries, verifiedInitialWrites);
|
|
3871
|
+
const classification = classifySyncMessageScope({
|
|
3872
|
+
message: entry.message,
|
|
3873
|
+
initialWrite,
|
|
3874
|
+
scope,
|
|
3875
|
+
});
|
|
3876
|
+
if (classification === 'in-scope') {
|
|
3877
|
+
return { accepted: true };
|
|
3878
|
+
}
|
|
3879
|
+
const messageCid = yield getMessageCid(entry.message);
|
|
3880
|
+
console.warn(`SyncEngineLevel: refusing to apply ${classification} pulled message ${messageCid}`);
|
|
3881
|
+
return { accepted: false, classification };
|
|
3882
|
+
});
|
|
3883
|
+
}
|
|
3884
|
+
resolvePulledDeleteInitialWrite(did_1, message_1, entries_1) {
|
|
3885
|
+
return __awaiter(this, arguments, void 0, function* (did, message, entries, verifiedInitialWrites = []) {
|
|
3886
|
+
const descriptor = message.descriptor;
|
|
3887
|
+
if (descriptor.interface !== DwnInterfaceName.Records ||
|
|
3888
|
+
descriptor.method !== DwnMethodName.Delete ||
|
|
3889
|
+
typeof descriptor.recordId !== 'string') {
|
|
3890
|
+
return undefined;
|
|
3891
|
+
}
|
|
3892
|
+
if (!this.agent.dwn.isRemoteMode) {
|
|
3893
|
+
const localInitialWrite = yield RecordsWrite.fetchInitialRecordsWriteMessage(this.agent.dwn.node.storage.messageStore, did, descriptor.recordId);
|
|
3894
|
+
if (localInitialWrite) {
|
|
3895
|
+
return localInitialWrite;
|
|
3896
|
+
}
|
|
3897
|
+
}
|
|
3898
|
+
const verifiedInitialWrite = SyncEngineLevel.findInitialWriteByRecordId(descriptor.recordId, verifiedInitialWrites);
|
|
3899
|
+
if (verifiedInitialWrite !== undefined) {
|
|
3900
|
+
return verifiedInitialWrite;
|
|
3901
|
+
}
|
|
3902
|
+
// Batch entries are only used when the initial write has not been applied
|
|
3903
|
+
// locally yet. Verified dependency hints cover the projected remote-mode
|
|
3904
|
+
// path where the initial write was applied in the previous pull batch and
|
|
3905
|
+
// no embedded local message store is available. Batch entries are still
|
|
3906
|
+
// parsed as initial RecordsWrite messages, and processRawMessage
|
|
3907
|
+
// authenticates the delete before any local mutation occurs.
|
|
3908
|
+
return this.findInitialWriteInPullBatch(descriptor.recordId, entries);
|
|
3909
|
+
});
|
|
3910
|
+
}
|
|
3911
|
+
static findInitialWriteByRecordId(recordId, initialWrites) {
|
|
3912
|
+
return initialWrites.find(initialWrite => initialWrite.recordId === recordId);
|
|
3913
|
+
}
|
|
3914
|
+
findInitialWriteInPullBatch(recordId, entries) {
|
|
3915
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3916
|
+
for (const entry of entries) {
|
|
3917
|
+
if (entry.message.descriptor.interface !== DwnInterfaceName.Records ||
|
|
3918
|
+
entry.message.descriptor.method !== DwnMethodName.Write) {
|
|
3919
|
+
continue;
|
|
3920
|
+
}
|
|
3921
|
+
const candidate = entry.message;
|
|
3922
|
+
if (candidate.recordId === recordId && (yield RecordsWrite.isInitialWrite(candidate))) {
|
|
3923
|
+
return candidate;
|
|
3924
|
+
}
|
|
3925
|
+
}
|
|
3926
|
+
return undefined;
|
|
3927
|
+
});
|
|
3928
|
+
}
|
|
2531
3929
|
// ---------------------------------------------------------------------------
|
|
2532
3930
|
// Echo-loop suppression
|
|
2533
3931
|
// ---------------------------------------------------------------------------
|
|
@@ -2578,11 +3976,10 @@ export class SyncEngineLevel {
|
|
|
2578
3976
|
* in dependency order (topological sort).
|
|
2579
3977
|
*/
|
|
2580
3978
|
pushMessages(_a) {
|
|
2581
|
-
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid,
|
|
3979
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, permissionGrantIds, messageCids }) {
|
|
2582
3980
|
return pushMessages({
|
|
2583
|
-
did, dwnUrl, delegateDid,
|
|
3981
|
+
did, dwnUrl, delegateDid, permissionGrantIds, messageCids,
|
|
2584
3982
|
agent: this.agent,
|
|
2585
|
-
permissionsApi: this._permissionsApi,
|
|
2586
3983
|
});
|
|
2587
3984
|
});
|
|
2588
3985
|
}
|
|
@@ -2606,8 +4003,8 @@ export class SyncEngineLevel {
|
|
|
2606
4003
|
* When `protocol` is undefined (full-tenant link), clears entries that
|
|
2607
4004
|
* also have no protocol.
|
|
2608
4005
|
*/
|
|
2609
|
-
clearDeadLettersForLink(
|
|
2610
|
-
return __awaiter(this,
|
|
4006
|
+
clearDeadLettersForLink(tenantDid_1, remoteEndpoint_1, protocol_1) {
|
|
4007
|
+
return __awaiter(this, arguments, void 0, function* (tenantDid, remoteEndpoint, protocol, options = {}) {
|
|
2611
4008
|
var _a, e_1, _b, _c;
|
|
2612
4009
|
const batch = [];
|
|
2613
4010
|
try {
|
|
@@ -2619,7 +4016,8 @@ export class SyncEngineLevel {
|
|
|
2619
4016
|
const entry = JSON.parse(value);
|
|
2620
4017
|
if (entry.tenantDid === tenantDid &&
|
|
2621
4018
|
entry.remoteEndpoint === remoteEndpoint &&
|
|
2622
|
-
entry.protocol === protocol
|
|
4019
|
+
entry.protocol === protocol &&
|
|
4020
|
+
(options.categories === undefined || options.categories.has(entry.category))) {
|
|
2623
4021
|
batch.push({ type: 'del', key });
|
|
2624
4022
|
}
|
|
2625
4023
|
}
|
|
@@ -2643,6 +4041,18 @@ export class SyncEngineLevel {
|
|
|
2643
4041
|
}
|
|
2644
4042
|
});
|
|
2645
4043
|
}
|
|
4044
|
+
clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint, protocol) {
|
|
4045
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
4046
|
+
try {
|
|
4047
|
+
yield this.clearDeadLettersForLink(tenantDid, remoteEndpoint, protocol, {
|
|
4048
|
+
categories: SyncEngineLevel.ROOT_CONVERGENCE_CLEARABLE_DEAD_LETTER_CATEGORIES,
|
|
4049
|
+
});
|
|
4050
|
+
}
|
|
4051
|
+
catch (error) {
|
|
4052
|
+
console.warn(`SyncEngineLevel: Failed to clear root-convergence dead letters for ${tenantDid} -> ${remoteEndpoint}`, error);
|
|
4053
|
+
}
|
|
4054
|
+
});
|
|
4055
|
+
}
|
|
2646
4056
|
/**
|
|
2647
4057
|
* Build a compound dead letter key. Different remotes can fail the same CID
|
|
2648
4058
|
* for different reasons, so the key includes the remote endpoint.
|
|
@@ -2689,7 +4099,7 @@ export class SyncEngineLevel {
|
|
|
2689
4099
|
finally { if (e_2) throw e_2.error; }
|
|
2690
4100
|
}
|
|
2691
4101
|
// Deterministic ordering: newest first so apps see the most recent failures.
|
|
2692
|
-
entries.sort((a, b) => b.failedAt
|
|
4102
|
+
entries.sort((a, b) => lexicographicalCompare(b.failedAt, a.failedAt));
|
|
2693
4103
|
return entries;
|
|
2694
4104
|
});
|
|
2695
4105
|
}
|
|
@@ -2775,12 +4185,17 @@ export class SyncEngineLevel {
|
|
|
2775
4185
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2776
4186
|
var _a, e_5, _b, _c;
|
|
2777
4187
|
let failedMessageCount = 0;
|
|
4188
|
+
let closureFailureCount = 0;
|
|
2778
4189
|
try {
|
|
2779
4190
|
for (var _d = true, _e = __asyncValues(this._deadLetters.iterator()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
|
|
2780
4191
|
_c = _f.value;
|
|
2781
4192
|
_d = false;
|
|
2782
|
-
const
|
|
4193
|
+
const [, value] = _c;
|
|
2783
4194
|
failedMessageCount++;
|
|
4195
|
+
const entry = JSON.parse(value);
|
|
4196
|
+
if (entry.category === 'closure') {
|
|
4197
|
+
closureFailureCount++;
|
|
4198
|
+
}
|
|
2784
4199
|
}
|
|
2785
4200
|
}
|
|
2786
4201
|
catch (e_5_1) { e_5 = { error: e_5_1 }; }
|
|
@@ -2790,35 +4205,90 @@ export class SyncEngineLevel {
|
|
|
2790
4205
|
}
|
|
2791
4206
|
finally { if (e_5) throw e_5.error; }
|
|
2792
4207
|
}
|
|
2793
|
-
//
|
|
2794
|
-
//
|
|
2795
|
-
//
|
|
4208
|
+
// Superseded authorization epochs can leave durable link state behind. Only
|
|
4209
|
+
// links that still belong to the current registered projection/epoch should
|
|
4210
|
+
// affect health. Endpoint-level orphan cleanup is a separate GC concern.
|
|
4211
|
+
const currentLinkIdentityKeys = yield this.getCurrentDurableLinkIdentityKeys();
|
|
2796
4212
|
let degradedLinkCount = 0;
|
|
2797
4213
|
const allLinks = yield this.ledger.getAllLinks();
|
|
2798
4214
|
for (const link of allLinks) {
|
|
2799
|
-
|
|
4215
|
+
const isCurrentLink = currentLinkIdentityKeys === undefined || currentLinkIdentityKeys.has(this.getDurableLinkIdentityKey(link));
|
|
4216
|
+
if (isCurrentLink && SyncEngineLevel.isUnhealthyLinkStatus(link.status)) {
|
|
2800
4217
|
degradedLinkCount++;
|
|
2801
4218
|
}
|
|
2802
4219
|
}
|
|
2803
4220
|
return {
|
|
2804
4221
|
connectivity: this.connectivityState,
|
|
2805
|
-
failedMessageCount,
|
|
2806
|
-
|
|
4222
|
+
failedMessageCount: failedMessageCount,
|
|
4223
|
+
closureFailureCount: closureFailureCount,
|
|
4224
|
+
degradedLinkCount: degradedLinkCount,
|
|
4225
|
+
syncHealthy: failedMessageCount === 0 && degradedLinkCount === 0,
|
|
2807
4226
|
};
|
|
2808
4227
|
});
|
|
2809
4228
|
}
|
|
4229
|
+
getCurrentDurableLinkIdentityKeys() {
|
|
4230
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
4231
|
+
var _a, e_6, _b, _c;
|
|
4232
|
+
try {
|
|
4233
|
+
const identityKeys = new Set();
|
|
4234
|
+
try {
|
|
4235
|
+
for (var _d = true, _e = __asyncValues(this._db.sublevel('registeredIdentities').iterator()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
|
|
4236
|
+
_c = _f.value;
|
|
4237
|
+
_d = false;
|
|
4238
|
+
const [did, options] = _c;
|
|
4239
|
+
let parsed;
|
|
4240
|
+
try {
|
|
4241
|
+
parsed = JSON.parse(options);
|
|
4242
|
+
}
|
|
4243
|
+
catch (error) {
|
|
4244
|
+
console.warn(`SyncEngineLevel: Corrupt sync options for ${did}, skipping health target:`, error);
|
|
4245
|
+
continue;
|
|
4246
|
+
}
|
|
4247
|
+
const scope = syncScopeFromProtocols(parsed.protocols);
|
|
4248
|
+
const resolutions = yield this.buildSyncTargetResolutions(did, scope, parsed);
|
|
4249
|
+
for (const resolution of resolutions) {
|
|
4250
|
+
const projectionId = yield computeProjectionId(did, resolution.scope);
|
|
4251
|
+
identityKeys.add(SyncEngineLevel.durableLinkIdentityKey(did, projectionId, resolution.authorizationEpoch));
|
|
4252
|
+
}
|
|
4253
|
+
}
|
|
4254
|
+
}
|
|
4255
|
+
catch (e_6_1) { e_6 = { error: e_6_1 }; }
|
|
4256
|
+
finally {
|
|
4257
|
+
try {
|
|
4258
|
+
if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
|
|
4259
|
+
}
|
|
4260
|
+
finally { if (e_6) throw e_6.error; }
|
|
4261
|
+
}
|
|
4262
|
+
return identityKeys;
|
|
4263
|
+
}
|
|
4264
|
+
catch (error) {
|
|
4265
|
+
console.warn('SyncEngineLevel: Failed to resolve current durable link identity keys for health; falling back to all durable links', error);
|
|
4266
|
+
return undefined;
|
|
4267
|
+
}
|
|
4268
|
+
});
|
|
4269
|
+
}
|
|
4270
|
+
getDurableLinkIdentityKey(link) {
|
|
4271
|
+
return SyncEngineLevel.durableLinkIdentityKey(link.tenantDid, link.projectionId, link.authorizationEpoch);
|
|
4272
|
+
}
|
|
4273
|
+
static durableLinkIdentityKey(tenantDid, projectionId, authorizationEpoch) {
|
|
4274
|
+
return `${tenantDid}^${projectionId}^${authorizationEpoch}`;
|
|
4275
|
+
}
|
|
4276
|
+
static isUnhealthyLinkStatus(status) {
|
|
4277
|
+
return status === 'repairing' || status === 'degraded_poll' || status === 'terminal_incomplete';
|
|
4278
|
+
}
|
|
2810
4279
|
// ---------------------------------------------------------------------------
|
|
2811
4280
|
// Sync targets
|
|
2812
4281
|
// ---------------------------------------------------------------------------
|
|
2813
4282
|
/**
|
|
2814
|
-
* Returns the list of sync targets:
|
|
4283
|
+
* Returns the list of sync targets: one canonical projection target per
|
|
4284
|
+
* registered DID and resolved DWN endpoint.
|
|
2815
4285
|
* Results are cached for up to 30 seconds to avoid redundant DID resolution
|
|
2816
4286
|
* on every sync tick. The cache is invalidated when identities are registered,
|
|
2817
4287
|
* unregistered, or updated.
|
|
2818
4288
|
*/
|
|
2819
4289
|
getSyncTargets() {
|
|
2820
4290
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2821
|
-
var _a,
|
|
4291
|
+
var _a, e_7, _b, _c;
|
|
2822
4292
|
// Return cached targets if still valid.
|
|
2823
4293
|
if (this._syncTargetsCache
|
|
2824
4294
|
&& (Date.now() - this._syncTargetsCache.timestamp) < SyncEngineLevel.SYNC_TARGETS_CACHE_TTL_MS) {
|
|
@@ -2845,31 +4315,22 @@ export class SyncEngineLevel {
|
|
|
2845
4315
|
console.warn(`SyncEngineLevel: Corrupt sync options for ${did}, skipping identity:`, error);
|
|
2846
4316
|
continue;
|
|
2847
4317
|
}
|
|
2848
|
-
const { protocols, delegateDid } = parsed;
|
|
2849
4318
|
const dwnEndpointUrls = yield this.agent.dwn.getDwnEndpointUrlsForTarget(did);
|
|
2850
4319
|
if (dwnEndpointUrls.length === 0) {
|
|
2851
4320
|
anyEndpointMissing = true;
|
|
2852
4321
|
continue;
|
|
2853
4322
|
}
|
|
2854
4323
|
for (const dwnUrl of dwnEndpointUrls) {
|
|
2855
|
-
|
|
2856
|
-
// Sync all protocols (global tree).
|
|
2857
|
-
targets.push({ did, delegateDid, dwnUrl });
|
|
2858
|
-
}
|
|
2859
|
-
else {
|
|
2860
|
-
for (const protocol of protocols) {
|
|
2861
|
-
targets.push({ did, delegateDid, dwnUrl, protocol });
|
|
2862
|
-
}
|
|
2863
|
-
}
|
|
4324
|
+
targets.push(...yield this.buildSyncTargetsForEndpoint(did, dwnUrl, parsed));
|
|
2864
4325
|
}
|
|
2865
4326
|
}
|
|
2866
4327
|
}
|
|
2867
|
-
catch (
|
|
4328
|
+
catch (e_7_1) { e_7 = { error: e_7_1 }; }
|
|
2868
4329
|
finally {
|
|
2869
4330
|
try {
|
|
2870
4331
|
if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
|
|
2871
4332
|
}
|
|
2872
|
-
finally { if (
|
|
4333
|
+
finally { if (e_7) throw e_7.error; }
|
|
2873
4334
|
}
|
|
2874
4335
|
// Only cache when:
|
|
2875
4336
|
// - The result is non-empty (empty = transient resolution failure).
|
|
@@ -2885,25 +4346,6 @@ export class SyncEngineLevel {
|
|
|
2885
4346
|
return targets;
|
|
2886
4347
|
});
|
|
2887
4348
|
}
|
|
2888
|
-
/**
|
|
2889
|
-
* Gets the permission grant ID for MessagesSync if a delegateDid is provided.
|
|
2890
|
-
* Returns undefined if no delegate is in use (owner access).
|
|
2891
|
-
*/
|
|
2892
|
-
getSyncPermissionGrantId(did, delegateDid, protocol) {
|
|
2893
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
2894
|
-
if (!delegateDid) {
|
|
2895
|
-
return undefined;
|
|
2896
|
-
}
|
|
2897
|
-
const messagesSyncGrant = yield this._permissionsApi.getPermissionForRequest({
|
|
2898
|
-
connectedDid: did,
|
|
2899
|
-
messageType: DwnInterface.MessagesSync,
|
|
2900
|
-
delegateDid,
|
|
2901
|
-
protocol,
|
|
2902
|
-
cached: true
|
|
2903
|
-
});
|
|
2904
|
-
return messagesSyncGrant.grant.id;
|
|
2905
|
-
});
|
|
2906
|
-
}
|
|
2907
4349
|
}
|
|
2908
4350
|
/** TTL for echo-loop suppression entries (60 seconds). */
|
|
2909
4351
|
SyncEngineLevel.ECHO_SUPPRESS_TTL_MS = 60000;
|
|
@@ -2911,12 +4353,14 @@ SyncEngineLevel.ECHO_SUPPRESS_TTL_MS = 60000;
|
|
|
2911
4353
|
SyncEngineLevel.ECHO_SUPPRESS_MAX_ENTRIES = 10000;
|
|
2912
4354
|
/** TTL for the sync targets cache (30 seconds). */
|
|
2913
4355
|
SyncEngineLevel.SYNC_TARGETS_CACHE_TTL_MS = 30000;
|
|
4356
|
+
/** Backoff schedule for recently published did:dht records. */
|
|
4357
|
+
SyncEngineLevel.DID_RESOLUTION_RETRY_BACKOFF_MS = [2000, 4000, 8000];
|
|
2914
4358
|
/** Maximum consecutive failures before entering backoff. */
|
|
2915
4359
|
SyncEngineLevel.MAX_CONSECUTIVE_FAILURES = 5;
|
|
2916
4360
|
/** Backoff multiplier for consecutive failures (caps at 4x the configured interval). */
|
|
2917
4361
|
SyncEngineLevel.MAX_BACKOFF_MULTIPLIER = 4;
|
|
2918
4362
|
// ---------------------------------------------------------------------------
|
|
2919
|
-
// Per-link repair and degraded-poll orchestration
|
|
4363
|
+
// Per-link repair and degraded-poll orchestration
|
|
2920
4364
|
// ---------------------------------------------------------------------------
|
|
2921
4365
|
/** Maximum consecutive repair attempts before falling back to degraded_poll. */
|
|
2922
4366
|
SyncEngineLevel.MAX_REPAIR_ATTEMPTS = 3;
|
|
@@ -2924,4 +4368,5 @@ SyncEngineLevel.MAX_REPAIR_ATTEMPTS = 3;
|
|
|
2924
4368
|
SyncEngineLevel.REPAIR_BACKOFF_MS = [1000, 3000, 10000];
|
|
2925
4369
|
/** Push retry backoff schedule: immediate, 250ms, 1s, 2s, then give up. */
|
|
2926
4370
|
SyncEngineLevel.PUSH_RETRY_BACKOFF_MS = [0, 250, 1000, 2000];
|
|
4371
|
+
SyncEngineLevel.ROOT_CONVERGENCE_CLEARABLE_DEAD_LETTER_CATEGORIES = new Set(['push-permanent', 'push-exhausted', 'pull-processing', 'pull-scope-rejected']);
|
|
2927
4372
|
//# sourceMappingURL=sync-engine-level.js.map
|