@enbox/agent 0.7.7 → 0.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.mjs +9 -9
- package/dist/browser.mjs.map +4 -4
- package/dist/esm/dwn-api.js +3 -2
- package/dist/esm/dwn-api.js.map +1 -1
- package/dist/esm/enbox-connect-protocol.js +5 -5
- package/dist/esm/enbox-connect-protocol.js.map +1 -1
- package/dist/esm/index.js +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/permissions-api.js +7 -34
- package/dist/esm/permissions-api.js.map +1 -1
- package/dist/esm/sync-closure-resolver.js +229 -110
- package/dist/esm/sync-closure-resolver.js.map +1 -1
- package/dist/esm/sync-closure-types.js +24 -7
- package/dist/esm/sync-closure-types.js.map +1 -1
- package/dist/esm/sync-engine-level.js +1961 -764
- package/dist/esm/sync-engine-level.js.map +1 -1
- package/dist/esm/sync-link-id.js +4 -13
- package/dist/esm/sync-link-id.js.map +1 -1
- package/dist/esm/sync-link-reconciler.js +26 -8
- package/dist/esm/sync-link-reconciler.js.map +1 -1
- package/dist/esm/sync-messages.js +218 -154
- package/dist/esm/sync-messages.js.map +1 -1
- package/dist/esm/sync-permission-grants.js +208 -0
- package/dist/esm/sync-permission-grants.js.map +1 -0
- package/dist/esm/sync-replication-ledger.js +23 -40
- package/dist/esm/sync-replication-ledger.js.map +1 -1
- package/dist/esm/sync-scope-acceptance.js +126 -0
- package/dist/esm/sync-scope-acceptance.js.map +1 -0
- package/dist/esm/sync-topological-sort.js +57 -15
- package/dist/esm/sync-topological-sort.js.map +1 -1
- package/dist/esm/types/sync.js +130 -22
- package/dist/esm/types/sync.js.map +1 -1
- package/dist/types/dwn-api.d.ts.map +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/permissions-api.d.ts +1 -2
- package/dist/types/permissions-api.d.ts.map +1 -1
- package/dist/types/sync-closure-resolver.d.ts.map +1 -1
- package/dist/types/sync-closure-types.d.ts +14 -3
- package/dist/types/sync-closure-types.d.ts.map +1 -1
- package/dist/types/sync-engine-level.d.ts +127 -25
- package/dist/types/sync-engine-level.d.ts.map +1 -1
- package/dist/types/sync-link-id.d.ts +3 -9
- package/dist/types/sync-link-id.d.ts.map +1 -1
- package/dist/types/sync-link-reconciler.d.ts +12 -2
- package/dist/types/sync-link-reconciler.d.ts.map +1 -1
- package/dist/types/sync-messages.d.ts +16 -13
- package/dist/types/sync-messages.d.ts.map +1 -1
- package/dist/types/sync-permission-grants.d.ts +52 -0
- package/dist/types/sync-permission-grants.d.ts.map +1 -0
- package/dist/types/sync-replication-ledger.d.ts +5 -13
- package/dist/types/sync-replication-ledger.d.ts.map +1 -1
- package/dist/types/sync-scope-acceptance.d.ts +28 -0
- package/dist/types/sync-scope-acceptance.d.ts.map +1 -0
- package/dist/types/sync-topological-sort.d.ts +2 -1
- package/dist/types/sync-topological-sort.d.ts.map +1 -1
- package/dist/types/types/permissions.d.ts +2 -0
- package/dist/types/types/permissions.d.ts.map +1 -1
- package/dist/types/types/sync.d.ts +137 -75
- package/dist/types/types/sync.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/dwn-api.ts +3 -2
- package/src/enbox-connect-protocol.ts +5 -5
- package/src/index.ts +10 -1
- package/src/permissions-api.ts +11 -42
- package/src/sync-closure-resolver.ts +306 -126
- package/src/sync-closure-types.ts +38 -9
- package/src/sync-engine-level.ts +2560 -797
- package/src/sync-link-id.ts +9 -14
- package/src/sync-link-reconciler.ts +43 -10
- package/src/sync-messages.ts +263 -159
- package/src/sync-permission-grants.ts +297 -0
- package/src/sync-replication-ledger.ts +55 -50
- package/src/sync-scope-acceptance.ts +186 -0
- package/src/sync-topological-sort.ts +89 -21
- package/src/types/permissions.ts +2 -0
- package/src/types/sync.ts +235 -62
|
@@ -17,20 +17,22 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
|
17
17
|
import ms from 'ms';
|
|
18
18
|
import { Level } from 'level';
|
|
19
19
|
import { sleep } from '@enbox/common';
|
|
20
|
-
import { Encoder, hashToHex, initDefaultHashes, Message } from '@enbox/dwn-sdk-js';
|
|
21
|
-
import { evaluateClosure } from './sync-closure-resolver.js';
|
|
22
|
-
import { MAX_PENDING_TOKENS } from './types/sync.js';
|
|
23
|
-
import { ReplicationLedger } from './sync-replication-ledger.js';
|
|
24
|
-
import { createClosureContext, invalidateClosureCache } from './sync-closure-types.js';
|
|
20
|
+
import { authenticate, DwnInterfaceName, DwnMethodName, Encoder, hashToHex, initDefaultHashes, Message, ProtocolsConfigure, RECORDS_PROJECTION_ROOT_VERSION, RecordsProjection, RecordsWrite } from '@enbox/dwn-sdk-js';
|
|
25
21
|
import { AgentPermissionsApi } from './permissions-api.js';
|
|
22
|
+
import { buildLinkId } from './sync-link-id.js';
|
|
26
23
|
import { DwnInterface } from './types/dwn.js';
|
|
24
|
+
import { evaluateClosure } from './sync-closure-resolver.js';
|
|
27
25
|
import { isRecordsWrite } from './utils.js';
|
|
28
|
-
import {
|
|
26
|
+
import { ReplicationLedger } from './sync-replication-ledger.js';
|
|
29
27
|
import { topologicalSort } from './sync-topological-sort.js';
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
28
|
+
import { classifySyncEventScope, classifySyncMessageScope } from './sync-scope-acceptance.js';
|
|
29
|
+
import { computeAuthorizationEpoch, computeProjectionId, lexicographicalCompare, MAX_PENDING_TOKENS, protocolsForSyncScope, singleProtocolForSyncScope, syncScopeFromProtocols } from './types/sync.js';
|
|
30
|
+
import { createClosureContext, invalidateClosureCache, isTerminalClosureFailureCode } from './sync-closure-types.js';
|
|
31
|
+
import { fetchRemoteMessages, getMessageCid, pullMessages, pushMessages, SyncPullAbortedError } from './sync-messages.js';
|
|
32
|
+
import { partitionRemoteEntries, SyncLinkReconciler } from './sync-link-reconciler.js';
|
|
33
|
+
import { permissionGrantIdsFromEntries, resolveMessagesSyncScopes, toMessagesPermissionGrantIds, toSyncAuthorizationGrants } from './sync-permission-grants.js';
|
|
32
34
|
/**
|
|
33
|
-
* Maximum bit prefix depth for the per-node tree walk
|
|
35
|
+
* Maximum bit prefix depth for the per-node tree walk fallback.
|
|
34
36
|
* At depth 16, each subtree covers ~1/65536 of the key space.
|
|
35
37
|
*/
|
|
36
38
|
const MAX_DIFF_DEPTH = 16;
|
|
@@ -49,50 +51,29 @@ const BATCHED_DIFF_DEPTH = 8;
|
|
|
49
51
|
* once the in-flight push completes.
|
|
50
52
|
*/
|
|
51
53
|
const PUSH_DEBOUNCE_MS = 100;
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
if (scope.kind === 'full') {
|
|
67
|
-
return true;
|
|
68
|
-
}
|
|
69
|
-
if (!scope.protocolPathPrefixes && !scope.contextIdPrefixes) {
|
|
70
|
-
return true;
|
|
71
|
-
}
|
|
72
|
-
const desc = message.descriptor;
|
|
73
|
-
// Check protocolPath prefix.
|
|
74
|
-
if (scope.protocolPathPrefixes && scope.protocolPathPrefixes.length > 0) {
|
|
75
|
-
const protocolPath = desc.protocolPath;
|
|
76
|
-
if (!protocolPath) {
|
|
77
|
-
return false;
|
|
78
|
-
}
|
|
79
|
-
const matches = scope.protocolPathPrefixes.some(prefix => protocolPath === prefix || protocolPath.startsWith(prefix + '/'));
|
|
80
|
-
if (!matches) {
|
|
81
|
-
return false;
|
|
82
|
-
}
|
|
54
|
+
var LinkSubscriptionOpenResult;
|
|
55
|
+
(function (LinkSubscriptionOpenResult) {
|
|
56
|
+
LinkSubscriptionOpenResult["ReadyForLive"] = "readyForLive";
|
|
57
|
+
LinkSubscriptionOpenResult["Polling"] = "polling";
|
|
58
|
+
LinkSubscriptionOpenResult["Repairing"] = "repairing";
|
|
59
|
+
})(LinkSubscriptionOpenResult || (LinkSubscriptionOpenResult = {}));
|
|
60
|
+
var LinkInitializationStatus;
|
|
61
|
+
(function (LinkInitializationStatus) {
|
|
62
|
+
LinkInitializationStatus["Active"] = "active";
|
|
63
|
+
LinkInitializationStatus["Failed"] = "failed";
|
|
64
|
+
})(LinkInitializationStatus || (LinkInitializationStatus = {}));
|
|
65
|
+
function syncEventScope(scope) {
|
|
66
|
+
if (scope === undefined) {
|
|
67
|
+
return {};
|
|
83
68
|
}
|
|
84
|
-
|
|
85
|
-
if (
|
|
86
|
-
|
|
87
|
-
if (!contextId) {
|
|
88
|
-
return false;
|
|
89
|
-
}
|
|
90
|
-
const matches = scope.contextIdPrefixes.some(prefix => contextId === prefix || contextId.startsWith(prefix + '/'));
|
|
91
|
-
if (!matches) {
|
|
92
|
-
return false;
|
|
93
|
-
}
|
|
69
|
+
const coveredProtocols = protocolsForSyncScope(scope);
|
|
70
|
+
if (coveredProtocols === undefined) {
|
|
71
|
+
return {};
|
|
94
72
|
}
|
|
95
|
-
|
|
73
|
+
const protocols = [...coveredProtocols];
|
|
74
|
+
return protocols.length === 1
|
|
75
|
+
? { protocol: protocols[0], protocols }
|
|
76
|
+
: { protocols };
|
|
96
77
|
}
|
|
97
78
|
export class SyncEngineLevel {
|
|
98
79
|
/** Validate `SyncIdentityOptions` for `registerIdentity` and `updateIdentityOptions`. */
|
|
@@ -107,6 +88,54 @@ export class SyncEngineLevel {
|
|
|
107
88
|
throw new Error('SyncEngineLevel: protocols must be \'all\' or a non-empty array of protocol URIs. An empty array is ambiguous.');
|
|
108
89
|
}
|
|
109
90
|
}
|
|
91
|
+
buildSyncTargetsForEndpoint(did, dwnUrl, options) {
|
|
92
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
93
|
+
const requestedScope = syncScopeFromProtocols(options.protocols);
|
|
94
|
+
const resolutions = yield this.buildSyncTargetResolutions(did, requestedScope, options);
|
|
95
|
+
return resolutions.map(resolution => (Object.assign({ did,
|
|
96
|
+
dwnUrl }, resolution)));
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
buildSyncTargetResolutions(did, requestedScope, options) {
|
|
100
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
101
|
+
const { delegateDid } = options;
|
|
102
|
+
if (delegateDid === undefined) {
|
|
103
|
+
return [{
|
|
104
|
+
scope: requestedScope,
|
|
105
|
+
authorization: { kind: 'owner' },
|
|
106
|
+
authorizationEpoch: yield computeAuthorizationEpoch({ kind: 'owner' }),
|
|
107
|
+
}];
|
|
108
|
+
}
|
|
109
|
+
const resolvedScopes = yield resolveMessagesSyncScopes({
|
|
110
|
+
did,
|
|
111
|
+
delegateDid,
|
|
112
|
+
requestedScope,
|
|
113
|
+
messageType: DwnInterface.MessagesSync,
|
|
114
|
+
permissionsApi: this._permissionsApi,
|
|
115
|
+
});
|
|
116
|
+
return Promise.all(resolvedScopes.map((_a) => __awaiter(this, [_a], void 0, function* ({ scope, permissionGrants }) {
|
|
117
|
+
const permissionGrantIds = permissionGrantIdsFromEntries(permissionGrants);
|
|
118
|
+
if (permissionGrantIds === undefined) {
|
|
119
|
+
throw new Error(`SyncEngineLevel: delegate ${delegateDid} has no active sync grants for ${did}.`);
|
|
120
|
+
}
|
|
121
|
+
return {
|
|
122
|
+
scope,
|
|
123
|
+
delegateDid,
|
|
124
|
+
authorization: {
|
|
125
|
+
kind: 'delegate',
|
|
126
|
+
delegateDid,
|
|
127
|
+
permissionGrantIds,
|
|
128
|
+
},
|
|
129
|
+
authorizationEpoch: yield computeAuthorizationEpoch({
|
|
130
|
+
kind: 'delegate',
|
|
131
|
+
delegateDid,
|
|
132
|
+
grants: toSyncAuthorizationGrants(permissionGrants),
|
|
133
|
+
}),
|
|
134
|
+
permissionGrantIds,
|
|
135
|
+
};
|
|
136
|
+
})));
|
|
137
|
+
});
|
|
138
|
+
}
|
|
110
139
|
constructor({ agent, dataPath, db }) {
|
|
111
140
|
this._syncLock = false;
|
|
112
141
|
/**
|
|
@@ -300,7 +329,13 @@ export class SyncEngineLevel {
|
|
|
300
329
|
this._syncTargetsCacheGeneration++;
|
|
301
330
|
// If live sync is active, hot-add subscriptions for this identity.
|
|
302
331
|
if (this._syncMode === 'live') {
|
|
303
|
-
yield this.addIdentityToLiveSync(did, options);
|
|
332
|
+
const currentIdentityKeys = yield this.addIdentityToLiveSync(did, options);
|
|
333
|
+
if (currentIdentityKeys.size > 0) {
|
|
334
|
+
yield this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
else {
|
|
338
|
+
yield this.tryPruneSupersededDurableLinksForRegisteredIdentity(did, options);
|
|
304
339
|
}
|
|
305
340
|
});
|
|
306
341
|
}
|
|
@@ -318,6 +353,7 @@ export class SyncEngineLevel {
|
|
|
318
353
|
yield registeredIdentities.del(did);
|
|
319
354
|
this._syncTargetsCache = undefined;
|
|
320
355
|
this._syncTargetsCacheGeneration++;
|
|
356
|
+
yield this.pruneSupersededDurableLinksForIdentity(did, new Set());
|
|
321
357
|
});
|
|
322
358
|
}
|
|
323
359
|
getIdentityOptions(did) {
|
|
@@ -352,18 +388,18 @@ export class SyncEngineLevel {
|
|
|
352
388
|
yield registeredIdentities.put(did, JSON.stringify(options));
|
|
353
389
|
this._syncTargetsCache = undefined;
|
|
354
390
|
this._syncTargetsCacheGeneration++;
|
|
355
|
-
// Always persist the new delegate to durable links, regardless of
|
|
356
|
-
// sync mode. If sync is stopped or polling, existing persisted links
|
|
357
|
-
// would otherwise keep the old delegateDid. When live sync starts
|
|
358
|
-
// later, initializeLinkTarget() loads the link from LevelDB without
|
|
359
|
-
// normalizing delegateDid, so repair/reconcile paths could use stale
|
|
360
|
-
// delegate data.
|
|
361
|
-
yield this.ledger.updateDelegateDid(did, options.delegateDid);
|
|
362
391
|
// If live sync is active, tear down and rebuild subscriptions with
|
|
363
|
-
// the new options.
|
|
392
|
+
// the new options. Delegate/scope changes derive a new authorization
|
|
393
|
+
// epoch, so existing durable links are not mutated in place.
|
|
364
394
|
if (this._syncMode === 'live' && this.hasActiveLinksForDid(did)) {
|
|
365
395
|
yield this.removeIdentityFromLiveSync(did);
|
|
366
|
-
yield this.addIdentityToLiveSync(did, options);
|
|
396
|
+
const currentIdentityKeys = yield this.addIdentityToLiveSync(did, options);
|
|
397
|
+
if (currentIdentityKeys.size > 0) {
|
|
398
|
+
yield this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
else {
|
|
402
|
+
yield this.tryPruneSupersededDurableLinksForRegisteredIdentity(did, options);
|
|
367
403
|
}
|
|
368
404
|
});
|
|
369
405
|
}
|
|
@@ -394,16 +430,13 @@ export class SyncEngineLevel {
|
|
|
394
430
|
let groupsFailed = 0;
|
|
395
431
|
const results = yield Promise.allSettled([...byUrl.entries()].map((_a) => __awaiter(this, [_a], void 0, function* ([dwnUrl, targets]) {
|
|
396
432
|
for (const target of targets) {
|
|
397
|
-
const { did, delegateDid, protocol } = target;
|
|
398
433
|
try {
|
|
399
|
-
yield this.
|
|
400
|
-
did, dwnUrl, delegateDid, protocol,
|
|
401
|
-
}, { direction });
|
|
434
|
+
yield this.reconcileProjectionTarget(target, { direction });
|
|
402
435
|
}
|
|
403
436
|
catch (error) {
|
|
404
437
|
// Skip remaining targets for this DWN endpoint.
|
|
405
438
|
groupsFailed++;
|
|
406
|
-
console.error(`SyncEngineLevel: Error syncing ${did} with ${dwnUrl}`, error);
|
|
439
|
+
console.error(`SyncEngineLevel: Error syncing ${target.did} with ${dwnUrl}`, error);
|
|
407
440
|
return;
|
|
408
441
|
}
|
|
409
442
|
}
|
|
@@ -502,7 +535,7 @@ export class SyncEngineLevel {
|
|
|
502
535
|
});
|
|
503
536
|
}
|
|
504
537
|
// ---------------------------------------------------------------------------
|
|
505
|
-
// Poll-mode sync
|
|
538
|
+
// Poll-mode sync
|
|
506
539
|
// ---------------------------------------------------------------------------
|
|
507
540
|
startPollSync(intervalMilliseconds) {
|
|
508
541
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -635,30 +668,75 @@ export class SyncEngineLevel {
|
|
|
635
668
|
*/
|
|
636
669
|
transitionToRepairing(linkKey, link, options) {
|
|
637
670
|
return __awaiter(this, void 0, void 0, function* () {
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
link.connectivity = 'offline';
|
|
641
|
-
yield this.ledger.setStatus(link, 'repairing');
|
|
642
|
-
this.emitEvent({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevStatus, to: 'repairing' });
|
|
643
|
-
if (prevConnectivity !== 'offline') {
|
|
644
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint, protocol: link.protocol, from: prevConnectivity, to: 'offline' });
|
|
671
|
+
if (link.status === 'terminal_incomplete') {
|
|
672
|
+
return;
|
|
645
673
|
}
|
|
674
|
+
yield this.setLinkOfflineStatus(link, 'repairing');
|
|
646
675
|
if (options === null || options === void 0 ? void 0 : options.resumeToken) {
|
|
647
676
|
this._repairContext.set(linkKey, { resumeToken: options.resumeToken });
|
|
648
677
|
}
|
|
649
678
|
// Clear runtime ordinals immediately — stale state must not linger
|
|
650
679
|
// across repair attempts.
|
|
651
|
-
|
|
652
|
-
if (rt) {
|
|
653
|
-
rt.inflight.clear();
|
|
654
|
-
rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
|
|
655
|
-
}
|
|
680
|
+
this.clearLinkRuntimeInflight(linkKey);
|
|
656
681
|
// Kick off repair with retry scheduling on failure.
|
|
657
682
|
void this.repairLink(linkKey).catch(() => {
|
|
658
683
|
this.scheduleRepairRetry(linkKey);
|
|
659
684
|
});
|
|
660
685
|
});
|
|
661
686
|
}
|
|
687
|
+
transitionToTerminalIncomplete(linkKey, link) {
|
|
688
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
689
|
+
if (link.status === 'terminal_incomplete') {
|
|
690
|
+
return;
|
|
691
|
+
}
|
|
692
|
+
yield this.setLinkOfflineStatus(link, 'terminal_incomplete');
|
|
693
|
+
yield this.closeLinkSubscriptions(link);
|
|
694
|
+
this.clearLinkRuntimeInflight(linkKey);
|
|
695
|
+
const retryTimer = this._repairRetryTimers.get(linkKey);
|
|
696
|
+
if (retryTimer) {
|
|
697
|
+
clearTimeout(retryTimer);
|
|
698
|
+
this._repairRetryTimers.delete(linkKey);
|
|
699
|
+
}
|
|
700
|
+
const degradedTimer = this._degradedPollTimers.get(linkKey);
|
|
701
|
+
if (degradedTimer) {
|
|
702
|
+
clearInterval(degradedTimer);
|
|
703
|
+
this._degradedPollTimers.delete(linkKey);
|
|
704
|
+
}
|
|
705
|
+
const reconcileTimer = this._reconcileTimers.get(linkKey);
|
|
706
|
+
if (reconcileTimer) {
|
|
707
|
+
clearTimeout(reconcileTimer);
|
|
708
|
+
this._reconcileTimers.delete(linkKey);
|
|
709
|
+
}
|
|
710
|
+
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
711
|
+
if (pushRuntime === null || pushRuntime === void 0 ? void 0 : pushRuntime.timer) {
|
|
712
|
+
clearTimeout(pushRuntime.timer);
|
|
713
|
+
}
|
|
714
|
+
this._pushRuntimes.delete(linkKey);
|
|
715
|
+
this._repairAttempts.delete(linkKey);
|
|
716
|
+
this._repairContext.delete(linkKey);
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
setLinkOfflineStatus(link, status) {
|
|
720
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
721
|
+
const prevStatus = link.status;
|
|
722
|
+
const prevConnectivity = link.connectivity;
|
|
723
|
+
link.connectivity = 'offline';
|
|
724
|
+
yield this.ledger.setStatus(link, status);
|
|
725
|
+
const eventScope = syncEventScope(link.scope);
|
|
726
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, eventScope), { from: prevStatus, to: status }));
|
|
727
|
+
if (prevConnectivity !== 'offline') {
|
|
728
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, eventScope), { from: prevConnectivity, to: 'offline' }));
|
|
729
|
+
}
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
clearLinkRuntimeInflight(linkKey) {
|
|
733
|
+
const rt = this._linkRuntimes.get(linkKey);
|
|
734
|
+
if (!rt) {
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
rt.inflight.clear();
|
|
738
|
+
rt.nextCommitOrdinal = rt.nextDeliveryOrdinal;
|
|
739
|
+
}
|
|
662
740
|
/**
|
|
663
741
|
* Schedule a retry for a failed repair. Uses exponential backoff.
|
|
664
742
|
* No-op if the link is already in `degraded_poll` (timer loop owns retries)
|
|
@@ -748,8 +826,9 @@ export class SyncEngineLevel {
|
|
|
748
826
|
// `_activeLinks` may contain a *different* link object for the same key.
|
|
749
827
|
// The old repair closure must not mutate the replacement link's state.
|
|
750
828
|
const isStaleLink = () => this._activeLinks.get(linkKey) !== link;
|
|
751
|
-
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid,
|
|
752
|
-
|
|
829
|
+
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, scope, authorization } = link;
|
|
830
|
+
const eventScope = syncEventScope(scope);
|
|
831
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'repair:started', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { attempt: ((_a = this._repairAttempts.get(linkKey)) !== null && _a !== void 0 ? _a : 0) + 1 }));
|
|
753
832
|
const attempts = ((_b = this._repairAttempts.get(linkKey)) !== null && _b !== void 0 ? _b : 0) + 1;
|
|
754
833
|
this._repairAttempts.set(linkKey, attempts);
|
|
755
834
|
// Step 1: Close existing subscriptions FIRST to stop old events from
|
|
@@ -766,7 +845,13 @@ export class SyncEngineLevel {
|
|
|
766
845
|
rt.nextCommitOrdinal = 0;
|
|
767
846
|
try {
|
|
768
847
|
// Step 3: Run SMT reconciliation for this link.
|
|
769
|
-
const reconcileOutcome = yield this.
|
|
848
|
+
const reconcileOutcome = yield this.reconcileProjectionTarget({
|
|
849
|
+
did,
|
|
850
|
+
dwnUrl,
|
|
851
|
+
delegateDid,
|
|
852
|
+
scope,
|
|
853
|
+
authorization,
|
|
854
|
+
}, undefined, () => this._engineGeneration === generation && !isStaleLink());
|
|
770
855
|
if (reconcileOutcome.aborted) {
|
|
771
856
|
return;
|
|
772
857
|
}
|
|
@@ -794,7 +879,16 @@ export class SyncEngineLevel {
|
|
|
794
879
|
if (this._engineGeneration !== generation || isStaleLink()) {
|
|
795
880
|
return;
|
|
796
881
|
}
|
|
797
|
-
const target = {
|
|
882
|
+
const target = {
|
|
883
|
+
did,
|
|
884
|
+
dwnUrl,
|
|
885
|
+
delegateDid,
|
|
886
|
+
scope,
|
|
887
|
+
authorization,
|
|
888
|
+
authorizationEpoch: link.authorizationEpoch,
|
|
889
|
+
permissionGrantIds: this.getAuthorizationGrantIds(authorization),
|
|
890
|
+
linkKey,
|
|
891
|
+
};
|
|
798
892
|
try {
|
|
799
893
|
yield this.openLivePullSubscription(target);
|
|
800
894
|
}
|
|
@@ -847,16 +941,15 @@ export class SyncEngineLevel {
|
|
|
847
941
|
const prevRepairConnectivity = link.connectivity;
|
|
848
942
|
link.connectivity = 'online';
|
|
849
943
|
yield this.ledger.setStatus(link, 'live');
|
|
850
|
-
//
|
|
851
|
-
//
|
|
852
|
-
//
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
this.emitEvent({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl, protocol });
|
|
944
|
+
// Root convergence proves primary CID membership matches, but it does
|
|
945
|
+
// not prove dependencies are usable. Keep closure failures until a later
|
|
946
|
+
// successful apply/closure pass clears the specific CID.
|
|
947
|
+
yield this.clearRootConvergenceDeadLettersForScope(did, dwnUrl, scope);
|
|
948
|
+
this.emitEvent(Object.assign({ type: 'repair:completed', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope));
|
|
856
949
|
if (prevRepairConnectivity !== 'online') {
|
|
857
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
950
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { from: prevRepairConnectivity, to: 'online' }));
|
|
858
951
|
}
|
|
859
|
-
this.emitEvent({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
952
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:status-change', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { from: 'repairing', to: 'live' }));
|
|
860
953
|
}
|
|
861
954
|
catch (error) {
|
|
862
955
|
// If teardown occurred during repair or the link was replaced by a
|
|
@@ -865,7 +958,7 @@ export class SyncEngineLevel {
|
|
|
865
958
|
return;
|
|
866
959
|
}
|
|
867
960
|
console.error(`SyncEngineLevel: Repair failed for ${did} -> ${dwnUrl} (attempt ${attempts})`, error);
|
|
868
|
-
this.emitEvent({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
961
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'repair:failed', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { attempt: attempts, error: String((_d = error.message) !== null && _d !== void 0 ? _d : error) }));
|
|
869
962
|
if (attempts >= SyncEngineLevel.MAX_REPAIR_ATTEMPTS) {
|
|
870
963
|
console.warn(`SyncEngineLevel: Max repair attempts reached for ${did} -> ${dwnUrl}, entering degraded_poll`);
|
|
871
964
|
yield this.enterDegradedPoll(linkKey);
|
|
@@ -882,25 +975,35 @@ export class SyncEngineLevel {
|
|
|
882
975
|
closeLinkSubscriptions(link) {
|
|
883
976
|
return __awaiter(this, void 0, void 0, function* () {
|
|
884
977
|
const { tenantDid: did, remoteEndpoint: dwnUrl } = link;
|
|
885
|
-
const linkKey = this.buildLinkKey(did, dwnUrl, link.
|
|
886
|
-
|
|
978
|
+
const linkKey = this.buildLinkKey(did, dwnUrl, link.projectionId, link.authorizationEpoch);
|
|
979
|
+
yield this.closeLiveSubscription(linkKey);
|
|
980
|
+
yield this.closeLocalSubscription(linkKey);
|
|
981
|
+
});
|
|
982
|
+
}
|
|
983
|
+
closeLiveSubscription(linkKey) {
|
|
984
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
887
985
|
const pullSub = this._liveSubscriptions.find((s) => s.linkKey === linkKey);
|
|
888
|
-
if (pullSub) {
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
|
|
986
|
+
if (!pullSub) {
|
|
987
|
+
return;
|
|
988
|
+
}
|
|
989
|
+
try {
|
|
990
|
+
yield pullSub.close();
|
|
894
991
|
}
|
|
895
|
-
|
|
992
|
+
catch ( /* best effort */_a) { /* best effort */ }
|
|
993
|
+
this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
closeLocalSubscription(linkKey) {
|
|
997
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
896
998
|
const pushSub = this._localSubscriptions.find((s) => s.linkKey === linkKey);
|
|
897
|
-
if (pushSub) {
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
this._localSubscriptions = this._localSubscriptions.filter(s => s !== pushSub);
|
|
999
|
+
if (!pushSub) {
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
try {
|
|
1003
|
+
yield pushSub.close();
|
|
903
1004
|
}
|
|
1005
|
+
catch ( /* best effort */_a) { /* best effort */ }
|
|
1006
|
+
this._localSubscriptions = this._localSubscriptions.filter(s => s !== pushSub);
|
|
904
1007
|
});
|
|
905
1008
|
}
|
|
906
1009
|
/**
|
|
@@ -918,8 +1021,9 @@ export class SyncEngineLevel {
|
|
|
918
1021
|
const prevDegradedStatus = link.status;
|
|
919
1022
|
yield this.ledger.setStatus(link, 'degraded_poll');
|
|
920
1023
|
this._repairAttempts.delete(linkKey);
|
|
921
|
-
|
|
922
|
-
this.emitEvent({ type: '
|
|
1024
|
+
const eventScope = syncEventScope(link.scope);
|
|
1025
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:status-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, eventScope), { from: prevDegradedStatus, to: 'degraded_poll' }));
|
|
1026
|
+
this.emitEvent(Object.assign({ type: 'degraded-poll:entered', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, eventScope));
|
|
923
1027
|
// Clear any existing timer for this link.
|
|
924
1028
|
const existing = this._degradedPollTimers.get(linkKey);
|
|
925
1029
|
if (existing) {
|
|
@@ -1025,14 +1129,7 @@ export class SyncEngineLevel {
|
|
|
1025
1129
|
const prev = link.connectivity;
|
|
1026
1130
|
if (prev !== 'offline') {
|
|
1027
1131
|
link.connectivity = 'offline';
|
|
1028
|
-
this.emitEvent({
|
|
1029
|
-
type: 'link:connectivity-change',
|
|
1030
|
-
tenantDid: link.tenantDid,
|
|
1031
|
-
remoteEndpoint: link.remoteEndpoint,
|
|
1032
|
-
protocol: link.protocol,
|
|
1033
|
-
from: prev,
|
|
1034
|
-
to: 'offline',
|
|
1035
|
-
});
|
|
1132
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:connectivity-change', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, syncEventScope(link.scope)), { from: prev, to: 'offline' }));
|
|
1036
1133
|
}
|
|
1037
1134
|
}
|
|
1038
1135
|
};
|
|
@@ -1143,78 +1240,123 @@ export class SyncEngineLevel {
|
|
|
1143
1240
|
// ---------------------------------------------------------------------------
|
|
1144
1241
|
/**
|
|
1145
1242
|
* Initialize a single replication link target: create or resume the durable
|
|
1146
|
-
* link,
|
|
1147
|
-
* transition the link to `'live'`.
|
|
1243
|
+
* link, open pull + push subscriptions, and transition the link to `'live'`.
|
|
1148
1244
|
*/
|
|
1149
1245
|
initializeLinkTarget(target) {
|
|
1150
1246
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1151
|
-
var _a;
|
|
1152
1247
|
let link;
|
|
1153
1248
|
try {
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
: { kind: 'full' };
|
|
1157
|
-
link = yield this.ledger.getOrCreateLink({
|
|
1158
|
-
tenantDid: target.did,
|
|
1159
|
-
remoteEndpoint: target.dwnUrl,
|
|
1160
|
-
scope: linkScope,
|
|
1161
|
-
delegateDid: target.delegateDid,
|
|
1162
|
-
protocol: target.protocol,
|
|
1163
|
-
});
|
|
1164
|
-
const linkKey = this.buildLinkKey(target.did, target.dwnUrl, link.scopeId);
|
|
1165
|
-
if (!link.pull.contiguousAppliedToken) {
|
|
1166
|
-
const legacyKey = buildLegacyCursorKey(target.did, target.dwnUrl, target.protocol);
|
|
1167
|
-
const legacyCursor = yield this.getCursor(legacyKey);
|
|
1168
|
-
if (legacyCursor) {
|
|
1169
|
-
ReplicationLedger.resetCheckpoint(link.pull, legacyCursor);
|
|
1170
|
-
yield this.ledger.saveLink(link);
|
|
1171
|
-
yield this.deleteLegacyCursor(legacyKey);
|
|
1172
|
-
}
|
|
1173
|
-
}
|
|
1249
|
+
link = yield this.getOrCreateReplicationLink(target);
|
|
1250
|
+
const linkKey = this.getReplicationLinkKey(target, link);
|
|
1174
1251
|
this._activeLinks.set(linkKey, link);
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
try {
|
|
1178
|
-
yield this.openLocalPushSubscription(targetWithKey);
|
|
1252
|
+
if (link.status === 'terminal_incomplete') {
|
|
1253
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1179
1254
|
}
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
try {
|
|
1184
|
-
yield pullSub.close();
|
|
1185
|
-
}
|
|
1186
|
-
catch ( /* best effort */_b) { /* best effort */ }
|
|
1187
|
-
this._liveSubscriptions = this._liveSubscriptions.filter(s => s !== pullSub);
|
|
1188
|
-
}
|
|
1189
|
-
throw pushError;
|
|
1255
|
+
const subscriptionResult = yield this.openLinkSubscriptions(Object.assign(Object.assign({}, target), { linkKey }));
|
|
1256
|
+
if (subscriptionResult === LinkSubscriptionOpenResult.ReadyForLive) {
|
|
1257
|
+
yield this.markLinkLive(target, link, linkKey);
|
|
1190
1258
|
}
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
if (link.needsReconcile) {
|
|
1194
|
-
this.scheduleReconcile(linkKey, 1000);
|
|
1259
|
+
else if (subscriptionResult === LinkSubscriptionOpenResult.Polling) {
|
|
1260
|
+
yield this.markLinkPolling(target, link);
|
|
1195
1261
|
}
|
|
1262
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1196
1263
|
}
|
|
1197
1264
|
catch (error) {
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1265
|
+
return this.handleInitializeLinkTargetError(target, link, error);
|
|
1266
|
+
}
|
|
1267
|
+
});
|
|
1268
|
+
}
|
|
1269
|
+
getOrCreateReplicationLink(target) {
|
|
1270
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1271
|
+
return this.ledger.getOrCreateLink({
|
|
1272
|
+
tenantDid: target.did,
|
|
1273
|
+
remoteEndpoint: target.dwnUrl,
|
|
1274
|
+
scope: target.scope,
|
|
1275
|
+
authorization: target.authorization,
|
|
1276
|
+
authorizationEpoch: target.authorizationEpoch,
|
|
1277
|
+
delegateDid: target.delegateDid,
|
|
1278
|
+
});
|
|
1279
|
+
});
|
|
1280
|
+
}
|
|
1281
|
+
getReplicationLinkKey(target, link) {
|
|
1282
|
+
return this.buildLinkKey(target.did, target.dwnUrl, link.projectionId, link.authorizationEpoch);
|
|
1283
|
+
}
|
|
1284
|
+
openLinkSubscriptions(target) {
|
|
1285
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1286
|
+
if (!SyncEngineLevel.supportsLiveSubscriptions(target.scope)) {
|
|
1287
|
+
return LinkSubscriptionOpenResult.Polling;
|
|
1288
|
+
}
|
|
1289
|
+
yield this.openLivePullSubscription(target);
|
|
1290
|
+
const link = this._activeLinks.get(target.linkKey);
|
|
1291
|
+
if ((link === null || link === void 0 ? void 0 : link.status) === 'repairing') {
|
|
1292
|
+
yield this.closeLiveSubscription(target.linkKey);
|
|
1293
|
+
return LinkSubscriptionOpenResult.Repairing;
|
|
1294
|
+
}
|
|
1295
|
+
try {
|
|
1296
|
+
yield this.openLocalPushSubscription(target);
|
|
1297
|
+
}
|
|
1298
|
+
catch (error) {
|
|
1299
|
+
yield this.closeLiveSubscription(target.linkKey);
|
|
1300
|
+
throw error;
|
|
1301
|
+
}
|
|
1302
|
+
return LinkSubscriptionOpenResult.ReadyForLive;
|
|
1303
|
+
});
|
|
1304
|
+
}
|
|
1305
|
+
static supportsLiveSubscriptions(scope) {
|
|
1306
|
+
// Records-primary projected links reconcile by root/diff polling until the
|
|
1307
|
+
// DWN has explicit path/context live subscription semantics.
|
|
1308
|
+
return scope.kind !== 'recordsProjection';
|
|
1309
|
+
}
|
|
1310
|
+
markLinkLive(target, link, linkKey) {
|
|
1311
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1312
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:status-change', tenantDid: target.did, remoteEndpoint: target.dwnUrl }, syncEventScope(target.scope)), { from: 'initializing', to: 'live' }));
|
|
1313
|
+
yield this.ledger.setStatus(link, 'live');
|
|
1314
|
+
if (link.needsReconcile) {
|
|
1315
|
+
this.scheduleReconcile(linkKey, 1000);
|
|
1215
1316
|
}
|
|
1216
1317
|
});
|
|
1217
1318
|
}
|
|
1319
|
+
markLinkPolling(target, link) {
|
|
1320
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1321
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:status-change', tenantDid: target.did, remoteEndpoint: target.dwnUrl }, syncEventScope(target.scope)), { from: 'initializing', to: 'polling' }));
|
|
1322
|
+
yield this.ledger.setStatus(link, 'polling');
|
|
1323
|
+
});
|
|
1324
|
+
}
|
|
1325
|
+
handleInitializeLinkTargetError(target, link, error) {
|
|
1326
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1327
|
+
var _a;
|
|
1328
|
+
if (error.isProgressGap && link) {
|
|
1329
|
+
const linkKey = this.getReplicationLinkKey(target, link);
|
|
1330
|
+
console.warn(`SyncEngineLevel: ProgressGap detected for ${target.did} -> ${target.dwnUrl}, initiating repair`);
|
|
1331
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'gap:detected', tenantDid: target.did, remoteEndpoint: target.dwnUrl }, syncEventScope(target.scope)), { reason: 'ProgressGap' }));
|
|
1332
|
+
yield this.transitionToRepairing(linkKey, link, {
|
|
1333
|
+
resumeToken: (_a = error.gapInfo) === null || _a === void 0 ? void 0 : _a.latestAvailable,
|
|
1334
|
+
});
|
|
1335
|
+
return this.createActiveLinkInitializationResult(link);
|
|
1336
|
+
}
|
|
1337
|
+
console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
|
|
1338
|
+
if (link) {
|
|
1339
|
+
this.cleanupFailedLinkInitialization(this.getReplicationLinkKey(target, link));
|
|
1340
|
+
}
|
|
1341
|
+
if (this.isDidResolutionFailure(error)) {
|
|
1342
|
+
throw error;
|
|
1343
|
+
}
|
|
1344
|
+
return { status: LinkInitializationStatus.Failed };
|
|
1345
|
+
});
|
|
1346
|
+
}
|
|
1347
|
+
createActiveLinkInitializationResult(link) {
|
|
1348
|
+
return {
|
|
1349
|
+
status: LinkInitializationStatus.Active,
|
|
1350
|
+
durableLinkIdentityKey: this.getDurableLinkIdentityKey(link),
|
|
1351
|
+
};
|
|
1352
|
+
}
|
|
1353
|
+
cleanupFailedLinkInitialization(linkKey) {
|
|
1354
|
+
this._activeLinks.delete(linkKey);
|
|
1355
|
+
this._linkRuntimes.delete(linkKey);
|
|
1356
|
+
if (this._liveSubscriptions.length === 0) {
|
|
1357
|
+
this._connectivityState = 'unknown';
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1218
1360
|
/**
|
|
1219
1361
|
* Wrapper around {@link initializeLinkTarget} that retries on DID
|
|
1220
1362
|
* resolution failures. Newly published `did:dht` DIDs take a few
|
|
@@ -1225,32 +1367,33 @@ export class SyncEngineLevel {
|
|
|
1225
1367
|
*/
|
|
1226
1368
|
initializeLinkTargetWithRetry(target) {
|
|
1227
1369
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1228
|
-
var _a;
|
|
1229
1370
|
try {
|
|
1230
|
-
yield this.initializeLinkTarget(target);
|
|
1371
|
+
return yield this.initializeLinkTarget(target);
|
|
1231
1372
|
}
|
|
1232
1373
|
catch (error) {
|
|
1233
|
-
|
|
1234
|
-
const isDidResolutionFailure = msg.includes('GetPublicKeyNotFound') || msg.includes('notFound');
|
|
1235
|
-
if (!isDidResolutionFailure) {
|
|
1374
|
+
if (!this.isDidResolutionFailure(error)) {
|
|
1236
1375
|
throw error;
|
|
1237
1376
|
}
|
|
1238
|
-
const
|
|
1239
|
-
for (const delay of delays) {
|
|
1377
|
+
for (const delay of SyncEngineLevel.DID_RESOLUTION_RETRY_BACKOFF_MS) {
|
|
1240
1378
|
yield sleep(delay);
|
|
1241
1379
|
try {
|
|
1242
|
-
yield this.initializeLinkTarget(target);
|
|
1243
|
-
return;
|
|
1380
|
+
return yield this.initializeLinkTarget(target);
|
|
1244
1381
|
}
|
|
1245
|
-
catch (
|
|
1382
|
+
catch (_a) {
|
|
1246
1383
|
// Continue to next attempt.
|
|
1247
1384
|
}
|
|
1248
1385
|
}
|
|
1249
1386
|
// All retries exhausted — the original error was already logged
|
|
1250
1387
|
// by initializeLinkTarget's catch block.
|
|
1388
|
+
return { status: LinkInitializationStatus.Failed };
|
|
1251
1389
|
}
|
|
1252
1390
|
});
|
|
1253
1391
|
}
|
|
1392
|
+
isDidResolutionFailure(error) {
|
|
1393
|
+
var _a;
|
|
1394
|
+
const message = (_a = error.message) !== null && _a !== void 0 ? _a : '';
|
|
1395
|
+
return message.includes('GetPublicKeyNotFound');
|
|
1396
|
+
}
|
|
1254
1397
|
// ---------------------------------------------------------------------------
|
|
1255
1398
|
// Hot-add / hot-remove: per-identity live sync management
|
|
1256
1399
|
// ---------------------------------------------------------------------------
|
|
@@ -1270,23 +1413,22 @@ export class SyncEngineLevel {
|
|
|
1270
1413
|
/** Hot-add a single identity to the active live sync session. */
|
|
1271
1414
|
addIdentityToLiveSync(did, options) {
|
|
1272
1415
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1273
|
-
const { protocols, delegateDid } = options;
|
|
1274
1416
|
const dwnEndpointUrls = yield this.agent.dwn.getDwnEndpointUrlsForTarget(did);
|
|
1275
1417
|
if (dwnEndpointUrls.length === 0) {
|
|
1276
|
-
return;
|
|
1418
|
+
return new Set();
|
|
1277
1419
|
}
|
|
1278
1420
|
const targets = [];
|
|
1279
1421
|
for (const dwnUrl of dwnEndpointUrls) {
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1422
|
+
targets.push(...yield this.buildSyncTargetsForEndpoint(did, dwnUrl, options));
|
|
1423
|
+
}
|
|
1424
|
+
const results = yield Promise.allSettled(targets.map(t => this.initializeLinkTargetWithRetry(t)));
|
|
1425
|
+
const currentIdentityKeys = new Set();
|
|
1426
|
+
for (const result of results) {
|
|
1427
|
+
if (result.status === 'fulfilled' && result.value.status === LinkInitializationStatus.Active) {
|
|
1428
|
+
currentIdentityKeys.add(result.value.durableLinkIdentityKey);
|
|
1287
1429
|
}
|
|
1288
1430
|
}
|
|
1289
|
-
|
|
1431
|
+
return currentIdentityKeys;
|
|
1290
1432
|
});
|
|
1291
1433
|
}
|
|
1292
1434
|
/** Hot-remove a single identity from the active live sync session. */
|
|
@@ -1361,6 +1503,40 @@ export class SyncEngineLevel {
|
|
|
1361
1503
|
this._closureContexts.delete(did);
|
|
1362
1504
|
});
|
|
1363
1505
|
}
|
|
1506
|
+
tryPruneSupersededDurableLinksForRegisteredIdentity(did, options) {
|
|
1507
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1508
|
+
try {
|
|
1509
|
+
const currentIdentityKeys = yield this.getDurableLinkIdentityKeysForRegisteredIdentity(did, options);
|
|
1510
|
+
yield this.pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys);
|
|
1511
|
+
}
|
|
1512
|
+
catch (error) {
|
|
1513
|
+
console.warn(`SyncEngineLevel: Failed to prune superseded durable links for ${did}`, error);
|
|
1514
|
+
}
|
|
1515
|
+
});
|
|
1516
|
+
}
|
|
1517
|
+
getDurableLinkIdentityKeysForRegisteredIdentity(did, options) {
|
|
1518
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1519
|
+
const scope = syncScopeFromProtocols(options.protocols);
|
|
1520
|
+
const resolutions = yield this.buildSyncTargetResolutions(did, scope, options);
|
|
1521
|
+
const keys = new Set();
|
|
1522
|
+
for (const resolution of resolutions) {
|
|
1523
|
+
const projectionId = yield computeProjectionId(did, resolution.scope);
|
|
1524
|
+
keys.add(SyncEngineLevel.durableLinkIdentityKey(did, projectionId, resolution.authorizationEpoch));
|
|
1525
|
+
}
|
|
1526
|
+
return keys;
|
|
1527
|
+
});
|
|
1528
|
+
}
|
|
1529
|
+
pruneSupersededDurableLinksForIdentity(did, currentIdentityKeys) {
|
|
1530
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1531
|
+
const links = yield this.ledger.getLinksForTenant(did);
|
|
1532
|
+
yield Promise.all(links.map((link) => __awaiter(this, void 0, void 0, function* () {
|
|
1533
|
+
if (currentIdentityKeys.has(this.getDurableLinkIdentityKey(link))) {
|
|
1534
|
+
return;
|
|
1535
|
+
}
|
|
1536
|
+
yield this.ledger.deleteLink(link.tenantDid, link.remoteEndpoint, link.projectionId, link.authorizationEpoch);
|
|
1537
|
+
})));
|
|
1538
|
+
});
|
|
1539
|
+
}
|
|
1364
1540
|
// ---------------------------------------------------------------------------
|
|
1365
1541
|
// Live pull: MessagesSubscribe to remote DWN
|
|
1366
1542
|
// ---------------------------------------------------------------------------
|
|
@@ -1370,52 +1546,14 @@ export class SyncEngineLevel {
|
|
|
1370
1546
|
*/
|
|
1371
1547
|
openLivePullSubscription(target) {
|
|
1372
1548
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1373
|
-
|
|
1374
|
-
const
|
|
1375
|
-
// Resolve the cursor from the link's durable pull checkpoint.
|
|
1376
|
-
// Legacy syncCursors migration happens at link load time in startLiveSync().
|
|
1549
|
+
const { did, delegateDid, dwnUrl } = target;
|
|
1550
|
+
const eventScope = syncEventScope(target.scope);
|
|
1377
1551
|
const cursorKey = target.linkKey;
|
|
1378
1552
|
const link = this._activeLinks.get(cursorKey);
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
// start from the beginning rather than crash the subscription.
|
|
1383
|
-
if (cursor && (!cursor.streamId || !cursor.messageCid || !cursor.epoch || !cursor.position)) {
|
|
1384
|
-
console.warn(`SyncEngineLevel: Discarding stored cursor with empty field(s) for ${did} -> ${dwnUrl}`);
|
|
1385
|
-
cursor = undefined;
|
|
1386
|
-
if (link) {
|
|
1387
|
-
ReplicationLedger.resetCheckpoint(link.pull);
|
|
1388
|
-
yield this.ledger.saveLink(link);
|
|
1389
|
-
}
|
|
1390
|
-
}
|
|
1391
|
-
// Build the MessagesSubscribe filters.
|
|
1392
|
-
// When the link has protocolPathPrefixes, include them in the filter so the
|
|
1393
|
-
// EventLog delivers only matching events (server-side filtering). This replaces
|
|
1394
|
-
// the less efficient agent-side isEventInScope filtering for the pull path.
|
|
1395
|
-
// Note: only the first prefix is used as the MessagesFilter field because
|
|
1396
|
-
// MessagesFilter.protocolPathPrefix is a single string. Multiple prefixes
|
|
1397
|
-
// would need multiple filters (OR semantics) — for now we use the first one.
|
|
1398
|
-
const protocolPathPrefix = (link === null || link === void 0 ? void 0 : link.scope.kind) === 'protocol'
|
|
1399
|
-
? (_a = link.scope.protocolPathPrefixes) === null || _a === void 0 ? void 0 : _a[0]
|
|
1400
|
-
: undefined;
|
|
1401
|
-
const filters = protocol
|
|
1402
|
-
? [Object.assign({ protocol }, (protocolPathPrefix ? { protocolPathPrefix } : {}))]
|
|
1553
|
+
const cursor = yield this.getInitialPullCursor({ did, dwnUrl, link });
|
|
1554
|
+
const filters = target.scope.kind === 'protocolSet'
|
|
1555
|
+
? target.scope.protocols.map(protocol => ({ protocol }))
|
|
1403
1556
|
: [];
|
|
1404
|
-
// Look up permission grant for MessagesSubscribe if using a delegate.
|
|
1405
|
-
// The unified scope matching in AgentPermissionsApi accepts a
|
|
1406
|
-
// Messages.Read grant for MessagesSubscribe requests, so a single
|
|
1407
|
-
// lookup is sufficient.
|
|
1408
|
-
let permissionGrantId;
|
|
1409
|
-
if (delegateDid) {
|
|
1410
|
-
const grant = yield this._permissionsApi.getPermissionForRequest({
|
|
1411
|
-
connectedDid: did,
|
|
1412
|
-
messageType: DwnInterface.MessagesSubscribe,
|
|
1413
|
-
delegateDid,
|
|
1414
|
-
protocol,
|
|
1415
|
-
cached: true
|
|
1416
|
-
});
|
|
1417
|
-
permissionGrantId = grant.grant.id;
|
|
1418
|
-
}
|
|
1419
1557
|
const handlerGeneration = this._engineGeneration;
|
|
1420
1558
|
// Define the subscription handler that processes incoming events.
|
|
1421
1559
|
// NOTE: The WebSocket client fires handlers without awaiting (fire-and-forget),
|
|
@@ -1423,236 +1561,19 @@ export class SyncEngineLevel {
|
|
|
1423
1561
|
// ensures the checkpoint advances only when all earlier deliveries are committed.
|
|
1424
1562
|
// Capture the link reference at subscription-open time so we can
|
|
1425
1563
|
// detect remove+re-add via object identity, not just key existence.
|
|
1426
|
-
const
|
|
1427
|
-
const
|
|
1428
|
-
|
|
1429
|
-
|
|
1564
|
+
const isStale = this.createLinkStalePredicate(cursorKey, link, handlerGeneration);
|
|
1565
|
+
const pullContext = {
|
|
1566
|
+
did,
|
|
1567
|
+
dwnUrl,
|
|
1568
|
+
delegateDid,
|
|
1569
|
+
eventScope,
|
|
1570
|
+
linkKey: cursorKey,
|
|
1571
|
+
link,
|
|
1572
|
+
permissionGrantIds: target.permissionGrantIds,
|
|
1573
|
+
isStale,
|
|
1574
|
+
};
|
|
1430
1575
|
const subscriptionHandler = (subMessage) => __awaiter(this, void 0, void 0, function* () {
|
|
1431
|
-
|
|
1432
|
-
if (isStale()) {
|
|
1433
|
-
return;
|
|
1434
|
-
}
|
|
1435
|
-
if (subMessage.type === 'eose') {
|
|
1436
|
-
// End-of-stored-events — catch-up complete.
|
|
1437
|
-
if (link) {
|
|
1438
|
-
// Guard: if the link transitioned to repairing while catch-up events
|
|
1439
|
-
// were being processed, skip all mutations — repair owns the state now.
|
|
1440
|
-
if (link.status !== 'live' && link.status !== 'initializing') {
|
|
1441
|
-
return;
|
|
1442
|
-
}
|
|
1443
|
-
if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1444
|
-
console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1445
|
-
if (!isStale()) {
|
|
1446
|
-
yield this.transitionToRepairing(cursorKey, link);
|
|
1447
|
-
}
|
|
1448
|
-
return;
|
|
1449
|
-
}
|
|
1450
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1451
|
-
this.drainCommittedPull(cursorKey);
|
|
1452
|
-
if (isStale()) {
|
|
1453
|
-
return;
|
|
1454
|
-
}
|
|
1455
|
-
yield this.ledger.saveLink(link);
|
|
1456
|
-
}
|
|
1457
|
-
// Transport is reachable — set connectivity to online.
|
|
1458
|
-
if (link) {
|
|
1459
|
-
const prevEoseConnectivity = link.connectivity;
|
|
1460
|
-
link.connectivity = 'online';
|
|
1461
|
-
if (prevEoseConnectivity !== 'online') {
|
|
1462
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl, protocol, from: prevEoseConnectivity, to: 'online' });
|
|
1463
|
-
}
|
|
1464
|
-
// If the link was marked dirty, schedule reconciliation now that it's healthy.
|
|
1465
|
-
if (link.needsReconcile) {
|
|
1466
|
-
this.scheduleReconcile(cursorKey, 500);
|
|
1467
|
-
}
|
|
1468
|
-
}
|
|
1469
|
-
else {
|
|
1470
|
-
this._connectivityState = 'online';
|
|
1471
|
-
}
|
|
1472
|
-
return;
|
|
1473
|
-
}
|
|
1474
|
-
if (subMessage.type === 'event') {
|
|
1475
|
-
const event = subMessage.event;
|
|
1476
|
-
// Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
|
|
1477
|
-
// skip all processing. Old subscription handlers may still fire after the
|
|
1478
|
-
// link transitions — these events should be ignored entirely, not just
|
|
1479
|
-
// skipped at the checkpoint level.
|
|
1480
|
-
if (link && link.status !== 'live' && link.status !== 'initializing') {
|
|
1481
|
-
return;
|
|
1482
|
-
}
|
|
1483
|
-
// Domain validation: reject tokens from a different stream/epoch.
|
|
1484
|
-
if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1485
|
-
console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1486
|
-
if (!isStale()) {
|
|
1487
|
-
yield this.transitionToRepairing(cursorKey, link);
|
|
1488
|
-
}
|
|
1489
|
-
return;
|
|
1490
|
-
}
|
|
1491
|
-
// Subset scope filtering: if the link has protocolPath/contextId prefixes,
|
|
1492
|
-
// skip events that don't match. This is agent-side filtering because
|
|
1493
|
-
// MessagesSubscribe only supports protocol-level filtering today.
|
|
1494
|
-
//
|
|
1495
|
-
// Skipped events MUST advance contiguousAppliedToken — otherwise the
|
|
1496
|
-
// link would replay the same filtered-out events indefinitely after
|
|
1497
|
-
// reconnect/repair. This is safe because the event is intentionally
|
|
1498
|
-
// excluded from this scope and doesn't need processing.
|
|
1499
|
-
if (link && !isEventInScope(event.message, link.scope)) {
|
|
1500
|
-
if (!isStale()) {
|
|
1501
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1502
|
-
ReplicationLedger.commitContiguousToken(link.pull, subMessage.cursor);
|
|
1503
|
-
yield this.ledger.saveLink(link);
|
|
1504
|
-
}
|
|
1505
|
-
return;
|
|
1506
|
-
}
|
|
1507
|
-
// Assign a delivery ordinal BEFORE async processing begins.
|
|
1508
|
-
// This captures the delivery order even if processing completes out of order.
|
|
1509
|
-
const rt = link ? this.getOrCreateRuntime(cursorKey) : undefined;
|
|
1510
|
-
const ordinal = rt ? rt.nextDeliveryOrdinal++ : -1;
|
|
1511
|
-
if (rt) {
|
|
1512
|
-
rt.inflight.set(ordinal, { ordinal, token: subMessage.cursor, committed: false });
|
|
1513
|
-
}
|
|
1514
|
-
try {
|
|
1515
|
-
// Extract inline data from the event (available for records <= 30 KB).
|
|
1516
|
-
let dataStream = this.extractDataStream(event);
|
|
1517
|
-
// For large RecordsWrite messages (no inline data), fetch the data
|
|
1518
|
-
// from the remote DWN via MessagesRead before storing locally.
|
|
1519
|
-
if (!dataStream && isRecordsWrite(event) && event.message.descriptor.dataCid) {
|
|
1520
|
-
const messageCid = yield Message.getCid(event.message);
|
|
1521
|
-
const fetched = yield fetchRemoteMessages({
|
|
1522
|
-
did, dwnUrl, delegateDid, protocol,
|
|
1523
|
-
messageCids: [messageCid],
|
|
1524
|
-
agent: this.agent,
|
|
1525
|
-
permissionsApi: this._permissionsApi,
|
|
1526
|
-
});
|
|
1527
|
-
if (fetched.length > 0 && fetched[0].dataStream) {
|
|
1528
|
-
dataStream = fetched[0].dataStream;
|
|
1529
|
-
}
|
|
1530
|
-
}
|
|
1531
|
-
yield this.agent.dwn.processRawMessage(did, event.message, { dataStream });
|
|
1532
|
-
if (isStale()) {
|
|
1533
|
-
return;
|
|
1534
|
-
}
|
|
1535
|
-
// Invalidate closure cache entries that may be affected by this message.
|
|
1536
|
-
// Must run before closure validation so subsequent evaluations in the
|
|
1537
|
-
// same session see the updated local state.
|
|
1538
|
-
const closureCtxForInvalidation = this._closureContexts.get(did);
|
|
1539
|
-
if (closureCtxForInvalidation) {
|
|
1540
|
-
invalidateClosureCache(closureCtxForInvalidation, event.message);
|
|
1541
|
-
}
|
|
1542
|
-
// Closure validation for scoped subset sync (Phase 3).
|
|
1543
|
-
// For protocol-scoped links, verify that all hard dependencies for
|
|
1544
|
-
// this operation are locally present before considering it committed.
|
|
1545
|
-
// Full-tenant scope bypasses this entirely (returns complete with 0 queries).
|
|
1546
|
-
if ((link === null || link === void 0 ? void 0 : link.scope.kind) === 'protocol') {
|
|
1547
|
-
const messageStore = this.agent.dwn.node.storage.messageStore;
|
|
1548
|
-
let closureCtx = this._closureContexts.get(did);
|
|
1549
|
-
if (!closureCtx) {
|
|
1550
|
-
closureCtx = createClosureContext(did, undefined, {
|
|
1551
|
-
isDelegateSession: !!delegateDid,
|
|
1552
|
-
});
|
|
1553
|
-
this._closureContexts.set(did, closureCtx);
|
|
1554
|
-
}
|
|
1555
|
-
const closureResult = yield evaluateClosure(event.message, messageStore, link.scope, closureCtx);
|
|
1556
|
-
if (isStale()) {
|
|
1557
|
-
return;
|
|
1558
|
-
}
|
|
1559
|
-
if (!closureResult.complete) {
|
|
1560
|
-
const failureCode = closureResult.failure.code;
|
|
1561
|
-
const failureDetail = closureResult.failure.detail;
|
|
1562
|
-
console.warn(`SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
|
|
1563
|
-
`${failureCode} — ${failureDetail}`);
|
|
1564
|
-
// Record the message that triggered the closure failure.
|
|
1565
|
-
const closureCid = yield Message.getCid(event.message);
|
|
1566
|
-
void this.recordDeadLetter({
|
|
1567
|
-
messageCid: closureCid,
|
|
1568
|
-
tenantDid: did,
|
|
1569
|
-
remoteEndpoint: dwnUrl,
|
|
1570
|
-
protocol,
|
|
1571
|
-
category: 'closure',
|
|
1572
|
-
errorCode: failureCode,
|
|
1573
|
-
errorDetail: failureDetail,
|
|
1574
|
-
});
|
|
1575
|
-
if (!isStale()) {
|
|
1576
|
-
yield this.transitionToRepairing(cursorKey, link);
|
|
1577
|
-
}
|
|
1578
|
-
return;
|
|
1579
|
-
}
|
|
1580
|
-
}
|
|
1581
|
-
// Squash convergence: processRawMessage triggers the DWN's built-in
|
|
1582
|
-
// squash resumable task (performRecordsSquash) which runs inline and
|
|
1583
|
-
// handles subset consumers correctly:
|
|
1584
|
-
// - If older siblings are locally present → purges them
|
|
1585
|
-
// - If squash arrives before older siblings → backstop rejects them (409)
|
|
1586
|
-
// - If no older siblings are local → no-op (correct)
|
|
1587
|
-
// Both sync orderings (squash-first or siblings-first) converge to
|
|
1588
|
-
// the same final state. No additional sync-engine side-effect is needed.
|
|
1589
|
-
// Track this CID for echo-loop suppression, scoped to the source endpoint.
|
|
1590
|
-
const pulledCid = yield Message.getCid(event.message);
|
|
1591
|
-
this._recentlyPulledCids.set(`${pulledCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
|
|
1592
|
-
this.evictExpiredEchoEntries();
|
|
1593
|
-
// Auto-clear any dead letter for this CID — it was processed
|
|
1594
|
-
// successfully, so a previous failure has been self-healed.
|
|
1595
|
-
this.clearFailedMessage(pulledCid, dwnUrl).catch(() => { });
|
|
1596
|
-
// Mark this ordinal as committed and drain the checkpoint.
|
|
1597
|
-
// Guard: if the link transitioned to repairing while this handler was
|
|
1598
|
-
// in-flight (e.g., an earlier ordinal's handler failed concurrently),
|
|
1599
|
-
// skip all state mutations — the repair process owns progression now.
|
|
1600
|
-
if (link && rt && link.status === 'live' && !isStale()) {
|
|
1601
|
-
const entry = rt.inflight.get(ordinal);
|
|
1602
|
-
if (entry) {
|
|
1603
|
-
entry.committed = true;
|
|
1604
|
-
}
|
|
1605
|
-
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1606
|
-
const drained = this.drainCommittedPull(cursorKey);
|
|
1607
|
-
if (drained > 0) {
|
|
1608
|
-
yield this.ledger.saveLink(link);
|
|
1609
|
-
// Emit after durable save — "advanced" means persisted.
|
|
1610
|
-
if (link.pull.contiguousAppliedToken) {
|
|
1611
|
-
this.emitEvent({
|
|
1612
|
-
type: 'checkpoint:pull-advance',
|
|
1613
|
-
tenantDid: link.tenantDid,
|
|
1614
|
-
remoteEndpoint: link.remoteEndpoint,
|
|
1615
|
-
protocol: link.protocol,
|
|
1616
|
-
position: link.pull.contiguousAppliedToken.position,
|
|
1617
|
-
messageCid: link.pull.contiguousAppliedToken.messageCid,
|
|
1618
|
-
});
|
|
1619
|
-
}
|
|
1620
|
-
}
|
|
1621
|
-
// Overflow: too many in-flight ordinals without draining.
|
|
1622
|
-
if (rt.inflight.size > MAX_PENDING_TOKENS) {
|
|
1623
|
-
console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1624
|
-
yield this.transitionToRepairing(cursorKey, link);
|
|
1625
|
-
}
|
|
1626
|
-
}
|
|
1627
|
-
}
|
|
1628
|
-
catch (error) {
|
|
1629
|
-
console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
|
|
1630
|
-
// Record the failing message in the dead letter store before
|
|
1631
|
-
// transitioning to repair. The CID identifies which specific
|
|
1632
|
-
// message caused the transition.
|
|
1633
|
-
try {
|
|
1634
|
-
const failedCid = yield Message.getCid(event.message);
|
|
1635
|
-
void this.recordDeadLetter({
|
|
1636
|
-
messageCid: failedCid,
|
|
1637
|
-
tenantDid: did,
|
|
1638
|
-
remoteEndpoint: dwnUrl,
|
|
1639
|
-
protocol,
|
|
1640
|
-
category: 'pull-processing',
|
|
1641
|
-
errorDetail: (_a = error.message) !== null && _a !== void 0 ? _a : String(error),
|
|
1642
|
-
});
|
|
1643
|
-
}
|
|
1644
|
-
catch (_b) {
|
|
1645
|
-
// Best effort — don't let dead letter recording block repair.
|
|
1646
|
-
}
|
|
1647
|
-
// A failed processRawMessage means local state is incomplete.
|
|
1648
|
-
// Transition to repairing immediately — do NOT advance the checkpoint
|
|
1649
|
-
// past this failure or let later ordinals commit past it. SMT
|
|
1650
|
-
// reconciliation will discover and fill the gap.
|
|
1651
|
-
if (link && !isStale()) {
|
|
1652
|
-
yield this.transitionToRepairing(cursorKey, link);
|
|
1653
|
-
}
|
|
1654
|
-
}
|
|
1655
|
-
}
|
|
1576
|
+
yield this.handleLivePullMessage(pullContext, subMessage);
|
|
1656
1577
|
});
|
|
1657
1578
|
// Construct the subscribe message and send it directly to the specific
|
|
1658
1579
|
// dwnUrl via WebSocket. We do NOT use agent.dwn.sendRequest() because
|
|
@@ -1664,7 +1585,7 @@ export class SyncEngineLevel {
|
|
|
1664
1585
|
target: did,
|
|
1665
1586
|
messageType: DwnInterface.MessagesSubscribe,
|
|
1666
1587
|
granteeDid: delegateDid,
|
|
1667
|
-
messageParams: { filters, cursor,
|
|
1588
|
+
messageParams: { filters, cursor, permissionGrantIds: toMessagesPermissionGrantIds(target.permissionGrantIds) },
|
|
1668
1589
|
};
|
|
1669
1590
|
const { message } = yield this.agent.dwn.processRequest(subscribeRequest);
|
|
1670
1591
|
if (!message) {
|
|
@@ -1710,13 +1631,14 @@ export class SyncEngineLevel {
|
|
|
1710
1631
|
if (reply.status.code !== 200 || !reply.subscription) {
|
|
1711
1632
|
throw new Error(`SyncEngineLevel: MessagesSubscribe failed for ${did} -> ${dwnUrl}: ${reply.status.code} ${reply.status.detail}`);
|
|
1712
1633
|
}
|
|
1634
|
+
const linkKey = cursorKey;
|
|
1635
|
+
const close = () => __awaiter(this, void 0, void 0, function* () { yield reply.subscription.close(); });
|
|
1713
1636
|
this._liveSubscriptions.push({
|
|
1714
|
-
linkKey
|
|
1637
|
+
linkKey,
|
|
1715
1638
|
did,
|
|
1716
1639
|
dwnUrl,
|
|
1717
1640
|
delegateDid,
|
|
1718
|
-
|
|
1719
|
-
close: () => __awaiter(this, void 0, void 0, function* () { yield reply.subscription.close(); }),
|
|
1641
|
+
close,
|
|
1720
1642
|
});
|
|
1721
1643
|
// Set per-link connectivity to online after successful subscription setup.
|
|
1722
1644
|
const pullLink = this._activeLinks.get(cursorKey);
|
|
@@ -1724,37 +1646,357 @@ export class SyncEngineLevel {
|
|
|
1724
1646
|
const prevPullConnectivity = pullLink.connectivity;
|
|
1725
1647
|
pullLink.connectivity = 'online';
|
|
1726
1648
|
if (prevPullConnectivity !== 'online') {
|
|
1727
|
-
this.emitEvent({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
1649
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { from: prevPullConnectivity, to: 'online' }));
|
|
1728
1650
|
}
|
|
1729
1651
|
}
|
|
1730
1652
|
});
|
|
1731
1653
|
}
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1654
|
+
getInitialPullCursor(_a) {
|
|
1655
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, link }) {
|
|
1656
|
+
// Resolve the cursor from the link's durable pull checkpoint.
|
|
1657
|
+
if (!link) {
|
|
1658
|
+
return undefined;
|
|
1659
|
+
}
|
|
1660
|
+
const cursor = link.pull.contiguousAppliedToken;
|
|
1661
|
+
if (!cursor || this.isValidProgressToken(cursor)) {
|
|
1662
|
+
return cursor;
|
|
1663
|
+
}
|
|
1664
|
+
// Guard against corrupted tokens with empty fields — these would fail
|
|
1665
|
+
// MessagesSubscribe JSON schema validation (minLength: 1). Discard and
|
|
1666
|
+
// start from the beginning rather than crash the subscription.
|
|
1667
|
+
console.warn(`SyncEngineLevel: Discarding stored cursor with empty field(s) for ${did} -> ${dwnUrl}`);
|
|
1668
|
+
ReplicationLedger.resetCheckpoint(link.pull);
|
|
1669
|
+
yield this.ledger.saveLink(link);
|
|
1670
|
+
return undefined;
|
|
1671
|
+
});
|
|
1672
|
+
}
|
|
1673
|
+
isValidProgressToken(token) {
|
|
1674
|
+
return !!(token.streamId && token.messageCid && token.epoch && token.position);
|
|
1675
|
+
}
|
|
1676
|
+
createLinkStalePredicate(linkKey, capturedLink, generation) {
|
|
1677
|
+
return () => this._engineGeneration !== generation ||
|
|
1678
|
+
!this._activeLinks.has(linkKey) ||
|
|
1679
|
+
(capturedLink !== undefined && this._activeLinks.get(linkKey) !== capturedLink);
|
|
1680
|
+
}
|
|
1681
|
+
handleLivePullMessage(context, subMessage) {
|
|
1740
1682
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
// Build filters scoped to the protocol (if any).
|
|
1744
|
-
const filters = protocol ? [{ protocol }] : [];
|
|
1745
|
-
// Look up permission grant for local subscription.
|
|
1746
|
-
let permissionGrantId;
|
|
1747
|
-
if (delegateDid) {
|
|
1748
|
-
const grant = yield this._permissionsApi.getPermissionForRequest({
|
|
1749
|
-
connectedDid: did,
|
|
1750
|
-
messageType: DwnInterface.MessagesSubscribe,
|
|
1751
|
-
delegateDid,
|
|
1752
|
-
protocol,
|
|
1753
|
-
cached: true,
|
|
1754
|
-
});
|
|
1755
|
-
permissionGrantId = grant.grant.id;
|
|
1683
|
+
if (context.isStale()) {
|
|
1684
|
+
return;
|
|
1756
1685
|
}
|
|
1757
|
-
|
|
1686
|
+
if (subMessage.type === 'eose') {
|
|
1687
|
+
yield this.handleLivePullEose(context, subMessage);
|
|
1688
|
+
return;
|
|
1689
|
+
}
|
|
1690
|
+
if (subMessage.type === 'error') {
|
|
1691
|
+
yield this.handleLivePullSubscriptionError(context, subMessage);
|
|
1692
|
+
return;
|
|
1693
|
+
}
|
|
1694
|
+
if (subMessage.type === 'event') {
|
|
1695
|
+
yield this.handleLivePullEvent(context, subMessage);
|
|
1696
|
+
}
|
|
1697
|
+
});
|
|
1698
|
+
}
|
|
1699
|
+
handleLivePullEose(_a, subMessage_1) {
|
|
1700
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, eventScope, linkKey, link, isStale }, subMessage) {
|
|
1701
|
+
if (link) {
|
|
1702
|
+
// Guard: if the link transitioned to repairing while catch-up events
|
|
1703
|
+
// were being processed, skip all mutations — repair owns the state now.
|
|
1704
|
+
if (link.status !== 'live' && link.status !== 'initializing') {
|
|
1705
|
+
return;
|
|
1706
|
+
}
|
|
1707
|
+
if (!ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1708
|
+
console.warn(`SyncEngineLevel: Token domain mismatch on EOSE for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1709
|
+
if (!isStale()) {
|
|
1710
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1711
|
+
}
|
|
1712
|
+
return;
|
|
1713
|
+
}
|
|
1714
|
+
ReplicationLedger.setReceivedToken(link.pull, subMessage.cursor);
|
|
1715
|
+
this.drainCommittedPull(linkKey);
|
|
1716
|
+
if (isStale()) {
|
|
1717
|
+
return;
|
|
1718
|
+
}
|
|
1719
|
+
yield this.ledger.saveLink(link);
|
|
1720
|
+
}
|
|
1721
|
+
this.markPullLinkOnline({ did, dwnUrl, eventScope, linkKey, link });
|
|
1722
|
+
});
|
|
1723
|
+
}
|
|
1724
|
+
markPullLinkOnline({ did, dwnUrl, eventScope, linkKey, link }) {
|
|
1725
|
+
if (!link) {
|
|
1726
|
+
this._connectivityState = 'online';
|
|
1727
|
+
return;
|
|
1728
|
+
}
|
|
1729
|
+
const previous = link.connectivity;
|
|
1730
|
+
link.connectivity = 'online';
|
|
1731
|
+
if (previous !== 'online') {
|
|
1732
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'link:connectivity-change', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope), { from: previous, to: 'online' }));
|
|
1733
|
+
}
|
|
1734
|
+
if (link.needsReconcile) {
|
|
1735
|
+
this.scheduleReconcile(linkKey, 500);
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
handleLivePullSubscriptionError(_a, subMessage_1) {
|
|
1739
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, linkKey, link, isStale }, subMessage) {
|
|
1740
|
+
console.warn(`SyncEngineLevel: subscription error for ${did} -> ${dwnUrl}: ${subMessage.error.code}`);
|
|
1741
|
+
if (link && !isStale()) {
|
|
1742
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1743
|
+
}
|
|
1744
|
+
});
|
|
1745
|
+
}
|
|
1746
|
+
handleLivePullEvent(context, subMessage) {
|
|
1747
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1748
|
+
const event = subMessage.event;
|
|
1749
|
+
if (yield this.shouldSkipLivePullEvent(context, subMessage)) {
|
|
1750
|
+
return;
|
|
1751
|
+
}
|
|
1752
|
+
const delivery = this.startPullDelivery(context, subMessage.cursor);
|
|
1753
|
+
try {
|
|
1754
|
+
const pulledCid = yield this.processLivePullEvent(context, event);
|
|
1755
|
+
if (!pulledCid) {
|
|
1756
|
+
return;
|
|
1757
|
+
}
|
|
1758
|
+
this.trackRecentlyPulledMessage(pulledCid, context.dwnUrl);
|
|
1759
|
+
this.clearFailedMessage(pulledCid, context.dwnUrl).catch(() => { });
|
|
1760
|
+
yield this.commitPullDelivery(context, subMessage.cursor, delivery);
|
|
1761
|
+
}
|
|
1762
|
+
catch (error) {
|
|
1763
|
+
yield this.handleLivePullProcessingError(context, event, error);
|
|
1764
|
+
}
|
|
1765
|
+
});
|
|
1766
|
+
}
|
|
1767
|
+
shouldSkipLivePullEvent(_a, subMessage_1) {
|
|
1768
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, linkKey, link, isStale }, subMessage) {
|
|
1769
|
+
// Guard: if the link is not live (e.g., repairing, degraded_poll, paused),
|
|
1770
|
+
// skip all processing. Old subscription handlers may still fire after the
|
|
1771
|
+
// link transitions — these events should be ignored entirely, not just
|
|
1772
|
+
// skipped at the checkpoint level.
|
|
1773
|
+
if (link && link.status !== 'live' && link.status !== 'initializing') {
|
|
1774
|
+
return true;
|
|
1775
|
+
}
|
|
1776
|
+
// Domain validation: reject tokens from a different stream/epoch.
|
|
1777
|
+
if (link && !ReplicationLedger.validateTokenDomain(link.pull, subMessage.cursor)) {
|
|
1778
|
+
console.warn(`SyncEngineLevel: Token domain mismatch for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1779
|
+
if (!isStale()) {
|
|
1780
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1781
|
+
}
|
|
1782
|
+
return true;
|
|
1783
|
+
}
|
|
1784
|
+
if (link) {
|
|
1785
|
+
const scopeClassification = classifySyncEventScope(subMessage.event, link.scope);
|
|
1786
|
+
if (scopeClassification === 'out-of-scope') {
|
|
1787
|
+
yield this.skipOutOfScopePullEvent({ link, cursor: subMessage.cursor, isStale });
|
|
1788
|
+
return true;
|
|
1789
|
+
}
|
|
1790
|
+
if (scopeClassification === 'unknown') {
|
|
1791
|
+
console.warn(`SyncEngineLevel: Unable to classify scoped pull event for ${did} -> ${dwnUrl}, transitioning to repair`);
|
|
1792
|
+
if (!isStale()) {
|
|
1793
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1794
|
+
}
|
|
1795
|
+
return true;
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
return false;
|
|
1799
|
+
});
|
|
1800
|
+
}
|
|
1801
|
+
skipOutOfScopePullEvent(_a) {
|
|
1802
|
+
return __awaiter(this, arguments, void 0, function* ({ link, cursor, isStale }) {
|
|
1803
|
+
// Skipped events MUST advance contiguousAppliedToken — otherwise the link
|
|
1804
|
+
// would replay the same filtered-out events indefinitely after reconnect or
|
|
1805
|
+
// repair. This is safe because the event is intentionally excluded from
|
|
1806
|
+
// this scope and doesn't need processing.
|
|
1807
|
+
if (isStale()) {
|
|
1808
|
+
return;
|
|
1809
|
+
}
|
|
1810
|
+
ReplicationLedger.setReceivedToken(link.pull, cursor);
|
|
1811
|
+
ReplicationLedger.commitContiguousToken(link.pull, cursor);
|
|
1812
|
+
yield this.ledger.saveLink(link);
|
|
1813
|
+
});
|
|
1814
|
+
}
|
|
1815
|
+
startPullDelivery({ linkKey, link }, cursor) {
|
|
1816
|
+
// Assign a delivery ordinal BEFORE async processing begins. This captures
|
|
1817
|
+
// delivery order even if processing completes out of order.
|
|
1818
|
+
const runtime = link ? this.getOrCreateRuntime(linkKey) : undefined;
|
|
1819
|
+
const ordinal = runtime ? runtime.nextDeliveryOrdinal++ : -1;
|
|
1820
|
+
if (runtime) {
|
|
1821
|
+
runtime.inflight.set(ordinal, { ordinal, token: cursor, committed: false });
|
|
1822
|
+
}
|
|
1823
|
+
return { runtime, ordinal };
|
|
1824
|
+
}
|
|
1825
|
+
processLivePullEvent(context, event) {
|
|
1826
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1827
|
+
const dataStream = yield this.getLivePullDataStream(context, event);
|
|
1828
|
+
yield this.agent.dwn.processRawMessage(context.did, event.message, { dataStream });
|
|
1829
|
+
if (context.isStale()) {
|
|
1830
|
+
return undefined;
|
|
1831
|
+
}
|
|
1832
|
+
this.invalidateClosureCacheForMessage(context.did, event.message);
|
|
1833
|
+
if (!(yield this.ensureClosureComplete(context, event))) {
|
|
1834
|
+
return undefined;
|
|
1835
|
+
}
|
|
1836
|
+
// Squash convergence: processRawMessage triggers the DWN's built-in
|
|
1837
|
+
// squash resumable task (performRecordsSquash), so no additional
|
|
1838
|
+
// sync-engine side effect is needed here.
|
|
1839
|
+
return Message.getCid(event.message);
|
|
1840
|
+
});
|
|
1841
|
+
}
|
|
1842
|
+
getLivePullDataStream(_a, event_1) {
|
|
1843
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, permissionGrantIds }, event) {
|
|
1844
|
+
var _b;
|
|
1845
|
+
const inlineData = this.extractDataStream(event);
|
|
1846
|
+
if (inlineData || !isRecordsWrite(event) || !event.message.descriptor.dataCid) {
|
|
1847
|
+
return inlineData;
|
|
1848
|
+
}
|
|
1849
|
+
// For large RecordsWrite messages (no inline data), fetch the data from
|
|
1850
|
+
// the remote DWN via MessagesRead before storing locally.
|
|
1851
|
+
const messageCid = yield Message.getCid(event.message);
|
|
1852
|
+
const fetched = yield fetchRemoteMessages({
|
|
1853
|
+
did,
|
|
1854
|
+
dwnUrl,
|
|
1855
|
+
delegateDid,
|
|
1856
|
+
permissionGrantIds,
|
|
1857
|
+
messageCids: [messageCid],
|
|
1858
|
+
agent: this.agent,
|
|
1859
|
+
});
|
|
1860
|
+
return (_b = fetched[0]) === null || _b === void 0 ? void 0 : _b.dataStream;
|
|
1861
|
+
});
|
|
1862
|
+
}
|
|
1863
|
+
invalidateClosureCacheForMessage(did, message) {
|
|
1864
|
+
// Must run before closure validation so subsequent evaluations in the same
|
|
1865
|
+
// session see the updated local state.
|
|
1866
|
+
const closureCtx = this._closureContexts.get(did);
|
|
1867
|
+
if (closureCtx) {
|
|
1868
|
+
invalidateClosureCache(closureCtx, message);
|
|
1869
|
+
}
|
|
1870
|
+
}
|
|
1871
|
+
ensureClosureComplete(context, event) {
|
|
1872
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1873
|
+
const { did, delegateDid, link, isStale } = context;
|
|
1874
|
+
if (!link || link.scope.kind === 'full') {
|
|
1875
|
+
return true;
|
|
1876
|
+
}
|
|
1877
|
+
let closureCtx = this._closureContexts.get(did);
|
|
1878
|
+
if (!closureCtx) {
|
|
1879
|
+
closureCtx = createClosureContext(did, undefined, {
|
|
1880
|
+
isDelegateSession: !!delegateDid,
|
|
1881
|
+
});
|
|
1882
|
+
this._closureContexts.set(did, closureCtx);
|
|
1883
|
+
}
|
|
1884
|
+
const messageStore = this.agent.dwn.node.storage.messageStore;
|
|
1885
|
+
const closureResult = yield evaluateClosure(event.message, messageStore, link.scope, closureCtx);
|
|
1886
|
+
if (isStale()) {
|
|
1887
|
+
return false;
|
|
1888
|
+
}
|
|
1889
|
+
if (closureResult.complete) {
|
|
1890
|
+
return true;
|
|
1891
|
+
}
|
|
1892
|
+
yield this.recordClosureFailure(context, event, closureResult.failure.code, closureResult.failure.detail);
|
|
1893
|
+
return false;
|
|
1894
|
+
});
|
|
1895
|
+
}
|
|
1896
|
+
recordClosureFailure(_a, event_1, failureCode_1, failureDetail_1) {
|
|
1897
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, linkKey, link, isStale }, event, failureCode, failureDetail) {
|
|
1898
|
+
console.warn(`SyncEngineLevel: Closure incomplete for ${did} -> ${dwnUrl}: ` +
|
|
1899
|
+
`${failureCode} — ${failureDetail}`);
|
|
1900
|
+
const closureCid = yield Message.getCid(event.message);
|
|
1901
|
+
void this.recordDeadLetter({
|
|
1902
|
+
messageCid: closureCid,
|
|
1903
|
+
tenantDid: did,
|
|
1904
|
+
remoteEndpoint: dwnUrl,
|
|
1905
|
+
protocol: event.message.descriptor.protocol,
|
|
1906
|
+
category: 'closure',
|
|
1907
|
+
errorCode: failureCode,
|
|
1908
|
+
errorDetail: failureDetail,
|
|
1909
|
+
});
|
|
1910
|
+
if (link && !isStale() && isTerminalClosureFailureCode(failureCode)) {
|
|
1911
|
+
yield this.transitionToTerminalIncomplete(linkKey, link);
|
|
1912
|
+
return;
|
|
1913
|
+
}
|
|
1914
|
+
if (link && !isStale()) {
|
|
1915
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1916
|
+
}
|
|
1917
|
+
});
|
|
1918
|
+
}
|
|
1919
|
+
trackRecentlyPulledMessage(messageCid, dwnUrl) {
|
|
1920
|
+
this._recentlyPulledCids.set(`${messageCid}|${dwnUrl}`, Date.now() + SyncEngineLevel.ECHO_SUPPRESS_TTL_MS);
|
|
1921
|
+
this.evictExpiredEchoEntries();
|
|
1922
|
+
}
|
|
1923
|
+
commitPullDelivery(_a, cursor_1, delivery_1) {
|
|
1924
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, linkKey, link, isStale }, cursor, delivery) {
|
|
1925
|
+
// Guard: if the link transitioned to repairing while this handler was
|
|
1926
|
+
// in-flight, skip all state mutations — the repair process owns progression.
|
|
1927
|
+
if (!link || !delivery.runtime || link.status !== 'live' || isStale()) {
|
|
1928
|
+
return;
|
|
1929
|
+
}
|
|
1930
|
+
const entry = delivery.runtime.inflight.get(delivery.ordinal);
|
|
1931
|
+
if (entry) {
|
|
1932
|
+
entry.committed = true;
|
|
1933
|
+
}
|
|
1934
|
+
ReplicationLedger.setReceivedToken(link.pull, cursor);
|
|
1935
|
+
const drained = this.drainCommittedPull(linkKey);
|
|
1936
|
+
if (drained > 0) {
|
|
1937
|
+
yield this.ledger.saveLink(link);
|
|
1938
|
+
this.emitPullCheckpointAdvance(link);
|
|
1939
|
+
}
|
|
1940
|
+
if (delivery.runtime.inflight.size > MAX_PENDING_TOKENS) {
|
|
1941
|
+
console.warn(`SyncEngineLevel: Pull in-flight overflow for ${did} -> ${dwnUrl}, transitioning to repairing`);
|
|
1942
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1943
|
+
}
|
|
1944
|
+
});
|
|
1945
|
+
}
|
|
1946
|
+
emitPullCheckpointAdvance(link) {
|
|
1947
|
+
if (!link.pull.contiguousAppliedToken) {
|
|
1948
|
+
return;
|
|
1949
|
+
}
|
|
1950
|
+
// Emit after durable save — "advanced" means persisted.
|
|
1951
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'checkpoint:pull-advance', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, syncEventScope(link.scope)), { position: link.pull.contiguousAppliedToken.position, messageCid: link.pull.contiguousAppliedToken.messageCid }));
|
|
1952
|
+
}
|
|
1953
|
+
handleLivePullProcessingError(_a, event_1, error_1) {
|
|
1954
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, linkKey, link, isStale }, event, error) {
|
|
1955
|
+
console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
|
|
1956
|
+
yield this.recordPullProcessingFailure({ did, dwnUrl, event, error });
|
|
1957
|
+
// A failed processRawMessage means local state is incomplete. Transition
|
|
1958
|
+
// to repairing immediately — do NOT advance the checkpoint past this
|
|
1959
|
+
// failure or let later ordinals commit past it. SMT reconciliation will
|
|
1960
|
+
// discover and fill the gap.
|
|
1961
|
+
if (link && !isStale()) {
|
|
1962
|
+
yield this.transitionToRepairing(linkKey, link);
|
|
1963
|
+
}
|
|
1964
|
+
});
|
|
1965
|
+
}
|
|
1966
|
+
recordPullProcessingFailure(_a) {
|
|
1967
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, event, error }) {
|
|
1968
|
+
var _b;
|
|
1969
|
+
try {
|
|
1970
|
+
const failedCid = yield Message.getCid(event.message);
|
|
1971
|
+
void this.recordDeadLetter({
|
|
1972
|
+
messageCid: failedCid,
|
|
1973
|
+
tenantDid: did,
|
|
1974
|
+
remoteEndpoint: dwnUrl,
|
|
1975
|
+
protocol: event.message.descriptor.protocol,
|
|
1976
|
+
category: 'pull-processing',
|
|
1977
|
+
errorDetail: (_b = error.message) !== null && _b !== void 0 ? _b : String(error),
|
|
1978
|
+
});
|
|
1979
|
+
}
|
|
1980
|
+
catch (_c) {
|
|
1981
|
+
// Best effort — don't let dead letter recording block repair.
|
|
1982
|
+
}
|
|
1983
|
+
});
|
|
1984
|
+
}
|
|
1985
|
+
// ---------------------------------------------------------------------------
|
|
1986
|
+
// Live push: local EventLog subscription for immediate push
|
|
1987
|
+
// ---------------------------------------------------------------------------
|
|
1988
|
+
/**
|
|
1989
|
+
* Subscribes to the local DWN's EventLog so that writes by the user are
|
|
1990
|
+
* immediately pushed to the remote DWN instead of waiting for the next poll.
|
|
1991
|
+
*/
|
|
1992
|
+
openLocalPushSubscription(target) {
|
|
1993
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1994
|
+
const { did, delegateDid, dwnUrl } = target;
|
|
1995
|
+
const protocol = singleProtocolForSyncScope(target.scope);
|
|
1996
|
+
const filters = target.scope.kind === 'protocolSet'
|
|
1997
|
+
? target.scope.protocols.map(protocol => ({ protocol }))
|
|
1998
|
+
: [];
|
|
1999
|
+
const handlerGeneration = this._engineGeneration;
|
|
1758
2000
|
// Capture the link for identity-based staleness detection.
|
|
1759
2001
|
const capturedPushLink = this._activeLinks.get(target.linkKey);
|
|
1760
2002
|
const isPushStale = () => this._engineGeneration !== handlerGeneration ||
|
|
@@ -1768,12 +2010,19 @@ export class SyncEngineLevel {
|
|
|
1768
2010
|
if (subMessage.type !== 'event') {
|
|
1769
2011
|
return;
|
|
1770
2012
|
}
|
|
1771
|
-
// Subset scope filtering: only push events that match the link
|
|
1772
|
-
//
|
|
2013
|
+
// Subset scope filtering: only push events that match the link scope.
|
|
2014
|
+
// Events outside the scope are not this link's responsibility.
|
|
1773
2015
|
const pushLinkKey = target.linkKey;
|
|
1774
2016
|
const pushLink = this._activeLinks.get(pushLinkKey);
|
|
1775
|
-
if (pushLink
|
|
1776
|
-
|
|
2017
|
+
if (pushLink) {
|
|
2018
|
+
const scopeClassification = classifySyncEventScope(subMessage.event, pushLink.scope);
|
|
2019
|
+
if (scopeClassification === 'out-of-scope') {
|
|
2020
|
+
return;
|
|
2021
|
+
}
|
|
2022
|
+
if (scopeClassification === 'unknown') {
|
|
2023
|
+
this.markLinkNeedsReconcile(pushLinkKey, pushLink, 'push-scope-unclassified');
|
|
2024
|
+
return;
|
|
2025
|
+
}
|
|
1777
2026
|
}
|
|
1778
2027
|
// Accumulate the message CID for a debounced push.
|
|
1779
2028
|
const targetKey = pushLinkKey;
|
|
@@ -1788,7 +2037,11 @@ export class SyncEngineLevel {
|
|
|
1788
2037
|
return;
|
|
1789
2038
|
}
|
|
1790
2039
|
const pushRuntime = this.getOrCreatePushRuntime(targetKey, {
|
|
1791
|
-
did,
|
|
2040
|
+
did,
|
|
2041
|
+
dwnUrl,
|
|
2042
|
+
delegateDid,
|
|
2043
|
+
protocol,
|
|
2044
|
+
permissionGrantIds: target.permissionGrantIds,
|
|
1792
2045
|
});
|
|
1793
2046
|
pushRuntime.entries.push({ cid });
|
|
1794
2047
|
// Immediate-first: if no push is in flight and no batch timer is
|
|
@@ -1806,20 +2059,20 @@ export class SyncEngineLevel {
|
|
|
1806
2059
|
target: did,
|
|
1807
2060
|
messageType: DwnInterface.MessagesSubscribe,
|
|
1808
2061
|
granteeDid: delegateDid,
|
|
1809
|
-
messageParams: { filters,
|
|
2062
|
+
messageParams: { filters, permissionGrantIds: toMessagesPermissionGrantIds(target.permissionGrantIds) },
|
|
1810
2063
|
subscriptionHandler: subscriptionHandler,
|
|
1811
2064
|
});
|
|
1812
2065
|
const reply = response.reply;
|
|
1813
2066
|
if (reply.status.code !== 200 || !reply.subscription) {
|
|
1814
2067
|
throw new Error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
|
|
1815
2068
|
}
|
|
2069
|
+
const close = () => __awaiter(this, void 0, void 0, function* () { yield reply.subscription.close(); });
|
|
1816
2070
|
this._localSubscriptions.push({
|
|
1817
|
-
linkKey:
|
|
2071
|
+
linkKey: target.linkKey,
|
|
1818
2072
|
did,
|
|
1819
2073
|
dwnUrl,
|
|
1820
2074
|
delegateDid,
|
|
1821
|
-
|
|
1822
|
-
close: () => __awaiter(this, void 0, void 0, function* () { yield reply.subscription.close(); }),
|
|
2075
|
+
close,
|
|
1823
2076
|
});
|
|
1824
2077
|
});
|
|
1825
2078
|
}
|
|
@@ -1835,108 +2088,148 @@ export class SyncEngineLevel {
|
|
|
1835
2088
|
}
|
|
1836
2089
|
flushPendingPushesForLink(linkKey) {
|
|
1837
2090
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
// debounce timer or retry callback could send pushes after the DID
|
|
1841
|
-
// was removed.
|
|
1842
|
-
if (!this._activeLinks.has(linkKey)) {
|
|
1843
|
-
return;
|
|
1844
|
-
}
|
|
1845
|
-
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
1846
|
-
if (!pushRuntime) {
|
|
2091
|
+
const batch = this.takePushFlushBatch(linkKey);
|
|
2092
|
+
if (!batch) {
|
|
1847
2093
|
return;
|
|
1848
2094
|
}
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
const flushLink = this._activeLinks.get(linkKey);
|
|
1852
|
-
const isFlushStale = () => !this._activeLinks.has(linkKey) ||
|
|
1853
|
-
(flushLink !== undefined && this._activeLinks.get(linkKey) !== flushLink);
|
|
1854
|
-
const { did, dwnUrl, delegateDid, protocol, entries: pushEntries, retryCount } = pushRuntime;
|
|
1855
|
-
pushRuntime.entries = [];
|
|
1856
|
-
if (pushEntries.length === 0) {
|
|
1857
|
-
if (!pushRuntime.timer && !pushRuntime.flushing && retryCount === 0) {
|
|
1858
|
-
this._pushRuntimes.delete(linkKey);
|
|
1859
|
-
}
|
|
1860
|
-
return;
|
|
1861
|
-
}
|
|
1862
|
-
const cids = pushEntries.map((entry) => entry.cid);
|
|
1863
|
-
pushRuntime.flushing = true;
|
|
2095
|
+
const { pushRuntime, pushEntries, isStale } = batch;
|
|
2096
|
+
const { did, dwnUrl, delegateDid, protocol, permissionGrantIds, retryCount } = pushRuntime;
|
|
1864
2097
|
try {
|
|
1865
2098
|
const result = yield pushMessages({
|
|
1866
|
-
did,
|
|
1867
|
-
|
|
2099
|
+
did,
|
|
2100
|
+
dwnUrl,
|
|
2101
|
+
delegateDid,
|
|
2102
|
+
permissionGrantIds,
|
|
2103
|
+
messageCids: pushEntries.map((entry) => entry.cid),
|
|
1868
2104
|
agent: this.agent,
|
|
1869
|
-
permissionsApi: this._permissionsApi,
|
|
1870
2105
|
});
|
|
1871
|
-
|
|
1872
|
-
// post-push state mutations — the replacement session owns this key.
|
|
1873
|
-
if (isFlushStale()) {
|
|
1874
|
-
return;
|
|
1875
|
-
}
|
|
1876
|
-
// Auto-clear dead letters for CIDs that succeeded — a previously
|
|
1877
|
-
// failed message may have been repaired by reconciliation.
|
|
1878
|
-
for (const cid of result.succeeded) {
|
|
1879
|
-
this.clearFailedMessage(cid, dwnUrl).catch(() => { });
|
|
1880
|
-
}
|
|
1881
|
-
// Record permanently failed messages in the dead letter store.
|
|
1882
|
-
for (const entry of result.permanentlyFailed) {
|
|
1883
|
-
yield this.recordDeadLetter({
|
|
1884
|
-
messageCid: entry.cid,
|
|
1885
|
-
tenantDid: did,
|
|
1886
|
-
remoteEndpoint: dwnUrl,
|
|
1887
|
-
protocol,
|
|
1888
|
-
category: 'push-permanent',
|
|
1889
|
-
errorCode: String((_a = entry.statusCode) !== null && _a !== void 0 ? _a : ''),
|
|
1890
|
-
errorDetail: (_b = entry.detail) !== null && _b !== void 0 ? _b : 'permanent push failure',
|
|
1891
|
-
});
|
|
1892
|
-
}
|
|
1893
|
-
if (result.failed.length > 0) {
|
|
1894
|
-
if (isFlushStale()) {
|
|
1895
|
-
return;
|
|
1896
|
-
}
|
|
1897
|
-
const failedSet = new Set(result.failed);
|
|
1898
|
-
const failedEntries = pushEntries.filter((entry) => failedSet.has(entry.cid));
|
|
1899
|
-
this.requeueOrReconcile(linkKey, {
|
|
1900
|
-
did, dwnUrl, delegateDid, protocol,
|
|
1901
|
-
entries: failedEntries,
|
|
1902
|
-
retryCount: retryCount + 1,
|
|
1903
|
-
});
|
|
1904
|
-
}
|
|
1905
|
-
else {
|
|
1906
|
-
// Successful push — reset retry count so subsequent unrelated
|
|
1907
|
-
// batches on this link start with a fresh budget.
|
|
1908
|
-
pushRuntime.retryCount = 0;
|
|
1909
|
-
if (!pushRuntime.timer && pushRuntime.entries.length === 0) {
|
|
1910
|
-
this._pushRuntimes.delete(linkKey);
|
|
1911
|
-
}
|
|
1912
|
-
}
|
|
2106
|
+
yield this.handlePushBatchResult(linkKey, batch, result);
|
|
1913
2107
|
}
|
|
1914
2108
|
catch (error) {
|
|
1915
|
-
if (
|
|
2109
|
+
if (isStale()) {
|
|
1916
2110
|
return;
|
|
1917
2111
|
}
|
|
1918
2112
|
console.error(`SyncEngineLevel: Push batch failed for ${did} -> ${dwnUrl}`, error);
|
|
1919
2113
|
this.requeueOrReconcile(linkKey, {
|
|
1920
|
-
did,
|
|
2114
|
+
did,
|
|
2115
|
+
dwnUrl,
|
|
2116
|
+
delegateDid,
|
|
2117
|
+
protocol,
|
|
2118
|
+
permissionGrantIds,
|
|
1921
2119
|
entries: pushEntries,
|
|
1922
2120
|
retryCount: retryCount + 1,
|
|
1923
2121
|
});
|
|
1924
2122
|
}
|
|
1925
2123
|
finally {
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
2124
|
+
this.finishPushFlush(linkKey, pushRuntime);
|
|
2125
|
+
}
|
|
2126
|
+
});
|
|
2127
|
+
}
|
|
2128
|
+
takePushFlushBatch(linkKey) {
|
|
2129
|
+
// Guard: bail if this link was hot-removed or is no longer live. Without
|
|
2130
|
+
// this, a stale debounce timer or retry callback could send pushes after
|
|
2131
|
+
// the DID was removed or the link entered repair/terminal state.
|
|
2132
|
+
const flushLink = this._activeLinks.get(linkKey);
|
|
2133
|
+
if ((flushLink === null || flushLink === void 0 ? void 0 : flushLink.status) !== 'live') {
|
|
2134
|
+
const staleRuntime = this._pushRuntimes.get(linkKey);
|
|
2135
|
+
if (staleRuntime === null || staleRuntime === void 0 ? void 0 : staleRuntime.timer) {
|
|
2136
|
+
clearTimeout(staleRuntime.timer);
|
|
2137
|
+
}
|
|
2138
|
+
this._pushRuntimes.delete(linkKey);
|
|
2139
|
+
return undefined;
|
|
2140
|
+
}
|
|
2141
|
+
const pushRuntime = this._pushRuntimes.get(linkKey);
|
|
2142
|
+
if (!pushRuntime) {
|
|
2143
|
+
return undefined;
|
|
2144
|
+
}
|
|
2145
|
+
const { entries: pushEntries, retryCount } = pushRuntime;
|
|
2146
|
+
pushRuntime.entries = [];
|
|
2147
|
+
if (pushEntries.length === 0) {
|
|
2148
|
+
if (!pushRuntime.timer && !pushRuntime.flushing && retryCount === 0) {
|
|
2149
|
+
this._pushRuntimes.delete(linkKey);
|
|
2150
|
+
}
|
|
2151
|
+
return undefined;
|
|
2152
|
+
}
|
|
2153
|
+
// Capture the current active link identity so we can detect
|
|
2154
|
+
// remove+re-add during the await pushMessages() call.
|
|
2155
|
+
const isStale = () => !this._activeLinks.has(linkKey) ||
|
|
2156
|
+
(flushLink !== undefined && this._activeLinks.get(linkKey) !== flushLink);
|
|
2157
|
+
pushRuntime.flushing = true;
|
|
2158
|
+
return { pushRuntime, pushEntries, isStale };
|
|
2159
|
+
}
|
|
2160
|
+
handlePushBatchResult(linkKey, batch, result) {
|
|
2161
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2162
|
+
if (batch.isStale()) {
|
|
2163
|
+
return;
|
|
1937
2164
|
}
|
|
2165
|
+
this.clearSucceededPushFailures(result.succeeded, batch.pushRuntime.dwnUrl);
|
|
2166
|
+
yield this.recordPermanentPushFailures(batch.pushRuntime, result.permanentlyFailed);
|
|
2167
|
+
if (result.failed.length > 0) {
|
|
2168
|
+
this.requeueFailedPushes(linkKey, batch, result.failed);
|
|
2169
|
+
return;
|
|
2170
|
+
}
|
|
2171
|
+
this.cleanupSuccessfulPushRuntime(linkKey, batch.pushRuntime);
|
|
1938
2172
|
});
|
|
1939
2173
|
}
|
|
2174
|
+
clearSucceededPushFailures(cids, dwnUrl) {
|
|
2175
|
+
for (const cid of cids) {
|
|
2176
|
+
this.clearFailedMessage(cid, dwnUrl).catch(() => { });
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
recordPermanentPushFailures(pushRuntime, permanentlyFailed) {
|
|
2180
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2181
|
+
var _a, _b;
|
|
2182
|
+
for (const entry of permanentlyFailed) {
|
|
2183
|
+
yield this.recordDeadLetter({
|
|
2184
|
+
messageCid: entry.cid,
|
|
2185
|
+
tenantDid: pushRuntime.did,
|
|
2186
|
+
remoteEndpoint: pushRuntime.dwnUrl,
|
|
2187
|
+
protocol: pushRuntime.protocol,
|
|
2188
|
+
category: 'push-permanent',
|
|
2189
|
+
errorCode: String((_a = entry.statusCode) !== null && _a !== void 0 ? _a : ''),
|
|
2190
|
+
errorDetail: (_b = entry.detail) !== null && _b !== void 0 ? _b : 'permanent push failure',
|
|
2191
|
+
});
|
|
2192
|
+
}
|
|
2193
|
+
});
|
|
2194
|
+
}
|
|
2195
|
+
requeueFailedPushes(linkKey, batch, failedCids) {
|
|
2196
|
+
if (batch.isStale()) {
|
|
2197
|
+
return;
|
|
2198
|
+
}
|
|
2199
|
+
const { did, dwnUrl, delegateDid, protocol, permissionGrantIds, retryCount } = batch.pushRuntime;
|
|
2200
|
+
const failedSet = new Set(failedCids);
|
|
2201
|
+
const failedEntries = batch.pushEntries.filter((entry) => failedSet.has(entry.cid));
|
|
2202
|
+
this.requeueOrReconcile(linkKey, {
|
|
2203
|
+
did,
|
|
2204
|
+
dwnUrl,
|
|
2205
|
+
delegateDid,
|
|
2206
|
+
protocol,
|
|
2207
|
+
permissionGrantIds,
|
|
2208
|
+
entries: failedEntries,
|
|
2209
|
+
retryCount: retryCount + 1,
|
|
2210
|
+
});
|
|
2211
|
+
}
|
|
2212
|
+
cleanupSuccessfulPushRuntime(linkKey, pushRuntime) {
|
|
2213
|
+
// Successful push — reset retry count so subsequent unrelated batches on
|
|
2214
|
+
// this link start with a fresh budget.
|
|
2215
|
+
pushRuntime.retryCount = 0;
|
|
2216
|
+
if (!pushRuntime.timer && pushRuntime.entries.length === 0) {
|
|
2217
|
+
this._pushRuntimes.delete(linkKey);
|
|
2218
|
+
}
|
|
2219
|
+
}
|
|
2220
|
+
finishPushFlush(linkKey, pushRuntime) {
|
|
2221
|
+
pushRuntime.flushing = false;
|
|
2222
|
+
// If new entries accumulated while this push was in flight, schedule a
|
|
2223
|
+
// short drain to flush them. This gives a brief batching window for burst
|
|
2224
|
+
// writes while keeping single-write latency low.
|
|
2225
|
+
const rt = this._pushRuntimes.get(linkKey);
|
|
2226
|
+
if (rt && rt.entries.length > 0 && !rt.timer) {
|
|
2227
|
+
rt.timer = setTimeout(() => {
|
|
2228
|
+
rt.timer = undefined;
|
|
2229
|
+
void this.flushPendingPushesForLink(linkKey);
|
|
2230
|
+
}, PUSH_DEBOUNCE_MS);
|
|
2231
|
+
}
|
|
2232
|
+
}
|
|
1940
2233
|
/**
|
|
1941
2234
|
* Re-queues a failed push batch for retry, or marks the link
|
|
1942
2235
|
* `needsReconcile` if retries are exhausted. Bounded to prevent
|
|
@@ -1964,34 +2257,701 @@ export class SyncEngineLevel {
|
|
|
1964
2257
|
}
|
|
1965
2258
|
this._pushRuntimes.delete(targetKey);
|
|
1966
2259
|
const link = this._activeLinks.get(targetKey);
|
|
1967
|
-
if (link
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
2260
|
+
if (link) {
|
|
2261
|
+
this.markLinkNeedsReconcile(targetKey, link, 'push-retry-exhausted');
|
|
2262
|
+
}
|
|
2263
|
+
return;
|
|
2264
|
+
}
|
|
2265
|
+
pushRuntime.entries.push(...pending.entries);
|
|
2266
|
+
pushRuntime.retryCount = pending.retryCount;
|
|
2267
|
+
const delayMs = (_a = SyncEngineLevel.PUSH_RETRY_BACKOFF_MS[pending.retryCount]) !== null && _a !== void 0 ? _a : 2000;
|
|
2268
|
+
if (pushRuntime.timer) {
|
|
2269
|
+
clearTimeout(pushRuntime.timer);
|
|
2270
|
+
}
|
|
2271
|
+
pushRuntime.timer = setTimeout(() => {
|
|
2272
|
+
pushRuntime.timer = undefined;
|
|
2273
|
+
void this.flushPendingPushesForLink(targetKey);
|
|
2274
|
+
}, delayMs);
|
|
2275
|
+
}
|
|
2276
|
+
markLinkNeedsReconcile(linkKey, link, reason) {
|
|
2277
|
+
if (link.needsReconcile) {
|
|
2278
|
+
this.scheduleReconcile(linkKey);
|
|
2279
|
+
return;
|
|
2280
|
+
}
|
|
2281
|
+
link.needsReconcile = true;
|
|
2282
|
+
void this.ledger.saveLink(link).then(() => {
|
|
2283
|
+
this.emitEvent(Object.assign(Object.assign({ type: 'reconcile:needed', tenantDid: link.tenantDid, remoteEndpoint: link.remoteEndpoint }, syncEventScope(link.scope)), { reason }));
|
|
2284
|
+
this.scheduleReconcile(linkKey);
|
|
2285
|
+
}).catch((error) => {
|
|
2286
|
+
console.error(`SyncEngineLevel: Failed to mark link for reconciliation ${link.tenantDid} -> ${link.remoteEndpoint}`, error);
|
|
2287
|
+
});
|
|
2288
|
+
}
|
|
2289
|
+
createLinkReconciler(shouldContinue) {
|
|
2290
|
+
return new SyncLinkReconciler({
|
|
2291
|
+
getLocalRoot: (did, delegateDid, protocol, permissionGrantIds) => __awaiter(this, void 0, void 0, function* () { return this.getLocalRoot(did, delegateDid, protocol, permissionGrantIds); }),
|
|
2292
|
+
getRemoteRoot: (did, dwnUrl, delegateDid, protocol, permissionGrantIds) => __awaiter(this, void 0, void 0, function* () { return this.getRemoteRoot(did, dwnUrl, delegateDid, protocol, permissionGrantIds); }),
|
|
2293
|
+
diffWithRemote: (target) => __awaiter(this, void 0, void 0, function* () { return this.diffWithRemote(target); }),
|
|
2294
|
+
pullMessages: (params) => __awaiter(this, void 0, void 0, function* () { return this.pullMessages(params); }),
|
|
2295
|
+
pushMessages: (params) => __awaiter(this, void 0, void 0, function* () { return this.pushMessages(params); }),
|
|
2296
|
+
shouldContinue,
|
|
2297
|
+
});
|
|
2298
|
+
}
|
|
2299
|
+
getReconcileProtocols(scope) {
|
|
2300
|
+
var _a;
|
|
2301
|
+
return (_a = protocolsForSyncScope(scope)) !== null && _a !== void 0 ? _a : [undefined];
|
|
2302
|
+
}
|
|
2303
|
+
getAuthorizationGrantIds(authorization) {
|
|
2304
|
+
return authorization.kind === 'delegate' ? authorization.permissionGrantIds : undefined;
|
|
2305
|
+
}
|
|
2306
|
+
reconcileProjectionTarget(target, options, shouldContinue) {
|
|
2307
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2308
|
+
if (target.scope.kind === 'recordsProjection') {
|
|
2309
|
+
return this.reconcileRecordsProjectionTarget(target, target.scope, options, shouldContinue);
|
|
2310
|
+
}
|
|
2311
|
+
if (target.scope.kind === 'protocolSet' && target.scope.protocols.length > 1) {
|
|
2312
|
+
return this.reconcileProtocolSetProjectionTarget(target, options, shouldContinue);
|
|
2313
|
+
}
|
|
2314
|
+
let converged = true;
|
|
2315
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
2316
|
+
const reconciler = this.createLinkReconciler(shouldContinue);
|
|
2317
|
+
for (const protocol of this.getReconcileProtocols(target.scope)) {
|
|
2318
|
+
const outcome = yield reconciler.reconcile({
|
|
2319
|
+
did: target.did,
|
|
2320
|
+
dwnUrl: target.dwnUrl,
|
|
2321
|
+
delegateDid: target.delegateDid,
|
|
2322
|
+
protocol,
|
|
2323
|
+
permissionGrantIds,
|
|
2324
|
+
}, options);
|
|
2325
|
+
if (outcome.aborted) {
|
|
2326
|
+
return { aborted: true };
|
|
2327
|
+
}
|
|
2328
|
+
if ((options === null || options === void 0 ? void 0 : options.verifyConvergence) === true && outcome.converged !== true) {
|
|
2329
|
+
converged = false;
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
return (options === null || options === void 0 ? void 0 : options.verifyConvergence) === true ? { converged } : {};
|
|
2333
|
+
});
|
|
2334
|
+
}
|
|
2335
|
+
reconcileRecordsProjectionTarget(target, scope, options, shouldContinue) {
|
|
2336
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2337
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
2338
|
+
const localRoot = yield this.getLocalProjectedRoot(target.did, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2339
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2340
|
+
return { aborted: true };
|
|
2341
|
+
}
|
|
2342
|
+
const remoteRoot = yield this.getRemoteProjectedRoot(target.did, target.dwnUrl, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2343
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2344
|
+
return { aborted: true };
|
|
2345
|
+
}
|
|
2346
|
+
if (localRoot !== remoteRoot) {
|
|
2347
|
+
const diff = yield this.diffProjectedWithRemote({
|
|
2348
|
+
did: target.did,
|
|
2349
|
+
dwnUrl: target.dwnUrl,
|
|
2350
|
+
delegateDid: target.delegateDid,
|
|
2351
|
+
scopes: scope.scopes,
|
|
2352
|
+
permissionGrantIds,
|
|
2353
|
+
});
|
|
2354
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2355
|
+
return { aborted: true };
|
|
2356
|
+
}
|
|
2357
|
+
const aborted = yield this.applyProjectedDiff(target, scope, diff, permissionGrantIds, options, shouldContinue);
|
|
2358
|
+
if (aborted) {
|
|
2359
|
+
return { aborted: true };
|
|
2360
|
+
}
|
|
2361
|
+
}
|
|
2362
|
+
if ((options === null || options === void 0 ? void 0 : options.verifyConvergence) !== true) {
|
|
2363
|
+
return {};
|
|
2364
|
+
}
|
|
2365
|
+
const postLocalRoot = yield this.getLocalProjectedRoot(target.did, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2366
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2367
|
+
return { aborted: true };
|
|
2368
|
+
}
|
|
2369
|
+
const postRemoteRoot = yield this.getRemoteProjectedRoot(target.did, target.dwnUrl, target.delegateDid, scope.scopes, permissionGrantIds);
|
|
2370
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2371
|
+
return { aborted: true };
|
|
2372
|
+
}
|
|
2373
|
+
return { converged: postLocalRoot === postRemoteRoot };
|
|
2374
|
+
});
|
|
2375
|
+
}
|
|
2376
|
+
reconcileProtocolSetProjectionTarget(target, options, shouldContinue) {
|
|
2377
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2378
|
+
if (target.scope.kind !== 'protocolSet') {
|
|
2379
|
+
return {};
|
|
2380
|
+
}
|
|
2381
|
+
const scope = target.scope;
|
|
2382
|
+
const permissionGrantIds = this.getAuthorizationGrantIds(target.authorization);
|
|
2383
|
+
const diffPlan = yield this.collectProtocolSetDiffPlan(target, scope, permissionGrantIds, shouldContinue);
|
|
2384
|
+
if (!diffPlan) {
|
|
2385
|
+
return { aborted: true };
|
|
2386
|
+
}
|
|
2387
|
+
if (diffPlan.changedProtocols.length === 0) {
|
|
2388
|
+
return (options === null || options === void 0 ? void 0 : options.verifyConvergence) === true ? { converged: true } : {};
|
|
2389
|
+
}
|
|
2390
|
+
const aborted = yield this.applyProtocolSetDiffPlan(target, scope, diffPlan, permissionGrantIds, options, shouldContinue);
|
|
2391
|
+
if (aborted) {
|
|
2392
|
+
return { aborted: true };
|
|
2393
|
+
}
|
|
2394
|
+
if ((options === null || options === void 0 ? void 0 : options.verifyConvergence) !== true) {
|
|
2395
|
+
return {};
|
|
2396
|
+
}
|
|
2397
|
+
return this.verifyProtocolSetConvergence(target, diffPlan.changedProtocols, permissionGrantIds, shouldContinue);
|
|
2398
|
+
});
|
|
2399
|
+
}
|
|
2400
|
+
collectProtocolSetDiffPlan(target, scope, permissionGrantIds, shouldContinue) {
|
|
2401
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2402
|
+
const plan = { changedProtocols: [], onlyRemote: [], onlyLocal: [] };
|
|
2403
|
+
for (const protocol of scope.protocols) {
|
|
2404
|
+
const roots = yield this.getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue);
|
|
2405
|
+
if (!roots) {
|
|
2406
|
+
return undefined;
|
|
2407
|
+
}
|
|
2408
|
+
if (roots.localRoot === roots.remoteRoot) {
|
|
2409
|
+
continue;
|
|
2410
|
+
}
|
|
2411
|
+
plan.changedProtocols.push(protocol);
|
|
2412
|
+
const diff = yield this.diffWithRemote({
|
|
2413
|
+
did: target.did,
|
|
2414
|
+
dwnUrl: target.dwnUrl,
|
|
2415
|
+
delegateDid: target.delegateDid,
|
|
2416
|
+
protocol,
|
|
2417
|
+
permissionGrantIds,
|
|
2418
|
+
});
|
|
2419
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2420
|
+
return undefined;
|
|
2421
|
+
}
|
|
2422
|
+
plan.onlyRemote.push(...diff.onlyRemote);
|
|
2423
|
+
plan.onlyLocal.push(...diff.onlyLocal);
|
|
2424
|
+
}
|
|
2425
|
+
return plan;
|
|
2426
|
+
});
|
|
2427
|
+
}
|
|
2428
|
+
getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue) {
|
|
2429
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2430
|
+
const localRoot = yield this.getLocalRoot(target.did, target.delegateDid, protocol, permissionGrantIds);
|
|
2431
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2432
|
+
return undefined;
|
|
2433
|
+
}
|
|
2434
|
+
const remoteRoot = yield this.getRemoteRoot(target.did, target.dwnUrl, target.delegateDid, protocol, permissionGrantIds);
|
|
2435
|
+
if ((shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false) {
|
|
2436
|
+
return undefined;
|
|
2437
|
+
}
|
|
2438
|
+
return { localRoot, remoteRoot };
|
|
2439
|
+
});
|
|
2440
|
+
}
|
|
2441
|
+
applyProtocolSetDiffPlan(target, scope, diffPlan, permissionGrantIds, options, shouldContinue) {
|
|
2442
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2443
|
+
// Keep the remote diff combined across protocols so topologicalSort can
|
|
2444
|
+
// order composed protocol configs before records that use them. Any future
|
|
2445
|
+
// chunking for large protocol sets must preserve this global dependency
|
|
2446
|
+
// order instead of reverting to independent per-protocol chunks.
|
|
2447
|
+
if ((options === null || options === void 0 ? void 0 : options.direction) !== 'push' &&
|
|
2448
|
+
diffPlan.onlyRemote.length > 0 &&
|
|
2449
|
+
(yield this.pullRemoteDiffEntries(target, scope, diffPlan.onlyRemote, permissionGrantIds, shouldContinue))) {
|
|
2450
|
+
return true;
|
|
2451
|
+
}
|
|
2452
|
+
if ((options === null || options === void 0 ? void 0 : options.direction) === 'pull' || diffPlan.onlyLocal.length === 0) {
|
|
2453
|
+
return false;
|
|
2454
|
+
}
|
|
2455
|
+
return this.pushLocalDiffEntries(target, diffPlan.onlyLocal, permissionGrantIds, shouldContinue);
|
|
2456
|
+
});
|
|
2457
|
+
}
|
|
2458
|
+
applyProjectedDiff(target, scope, diff, permissionGrantIds, options, shouldContinue) {
|
|
2459
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2460
|
+
if (yield this.pullProjectedRemoteDiff(target, scope, diff, permissionGrantIds, options, shouldContinue)) {
|
|
2461
|
+
return true;
|
|
2462
|
+
}
|
|
2463
|
+
return this.pushProjectedLocalDiff(target, diff.onlyLocal, permissionGrantIds, options, shouldContinue);
|
|
2464
|
+
});
|
|
2465
|
+
}
|
|
2466
|
+
pullProjectedRemoteDiff(target, scope, diff, permissionGrantIds, options, shouldContinue) {
|
|
2467
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2468
|
+
var _a;
|
|
2469
|
+
if ((options === null || options === void 0 ? void 0 : options.direction) === 'push' || diff.onlyRemote.length === 0) {
|
|
2470
|
+
return false;
|
|
2471
|
+
}
|
|
2472
|
+
return this.pullRemoteDiffEntries(target, scope, diff.onlyRemote, permissionGrantIds, shouldContinue, (_a = diff.dependencies) !== null && _a !== void 0 ? _a : []);
|
|
2473
|
+
});
|
|
2474
|
+
}
|
|
2475
|
+
pushProjectedLocalDiff(target, onlyLocal, permissionGrantIds, options, shouldContinue) {
|
|
2476
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2477
|
+
if ((options === null || options === void 0 ? void 0 : options.direction) === 'pull' || onlyLocal.length === 0) {
|
|
2478
|
+
return false;
|
|
2479
|
+
}
|
|
2480
|
+
return this.pushLocalDiffEntries(target, onlyLocal, permissionGrantIds, shouldContinue);
|
|
2481
|
+
});
|
|
2482
|
+
}
|
|
2483
|
+
pullRemoteDiffEntries(target_1, scope_1, onlyRemote_1, permissionGrantIds_1, shouldContinue_1) {
|
|
2484
|
+
return __awaiter(this, arguments, void 0, function* (target, scope, onlyRemote, permissionGrantIds, shouldContinue, dependencies = []) {
|
|
2485
|
+
const primaryEntries = SyncEngineLevel.dedupeRemoteEntries(onlyRemote);
|
|
2486
|
+
try {
|
|
2487
|
+
let verifiedInitialWrites = [];
|
|
2488
|
+
if (scope.kind === 'recordsProjection') {
|
|
2489
|
+
verifiedInitialWrites = yield this.pullProjectedDependencyHints(target, scope, primaryEntries, dependencies, permissionGrantIds, shouldContinue);
|
|
2490
|
+
}
|
|
2491
|
+
const { prefetched, needsFetchCids } = partitionRemoteEntries(primaryEntries);
|
|
2492
|
+
yield this.pullMessages({
|
|
2493
|
+
did: target.did,
|
|
2494
|
+
dwnUrl: target.dwnUrl,
|
|
2495
|
+
delegateDid: target.delegateDid,
|
|
2496
|
+
scope,
|
|
2497
|
+
permissionGrantIds,
|
|
2498
|
+
messageCids: needsFetchCids,
|
|
2499
|
+
prefetched,
|
|
2500
|
+
verifiedInitialWrites,
|
|
2501
|
+
shouldContinue,
|
|
1972
2502
|
});
|
|
1973
2503
|
}
|
|
2504
|
+
catch (error) {
|
|
2505
|
+
if (error instanceof SyncPullAbortedError) {
|
|
2506
|
+
return true;
|
|
2507
|
+
}
|
|
2508
|
+
throw error;
|
|
2509
|
+
}
|
|
2510
|
+
return (shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false;
|
|
2511
|
+
});
|
|
2512
|
+
}
|
|
2513
|
+
pullProjectedDependencyHints(target, scope, primaryEntries, dependencies, permissionGrantIds, shouldContinue) {
|
|
2514
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2515
|
+
const verified = yield this.verifyProjectedDependencies(target.did, scope, primaryEntries, dependencies);
|
|
2516
|
+
if (verified.length === 0) {
|
|
2517
|
+
return [];
|
|
2518
|
+
}
|
|
2519
|
+
yield this.pullMessages({
|
|
2520
|
+
did: target.did,
|
|
2521
|
+
dwnUrl: target.dwnUrl,
|
|
2522
|
+
delegateDid: target.delegateDid,
|
|
2523
|
+
scope: SyncEngineLevel.protocolSetScopeForProjectedDependencies(verified),
|
|
2524
|
+
permissionGrantIds,
|
|
2525
|
+
messageCids: [],
|
|
2526
|
+
prefetched: verified,
|
|
2527
|
+
shouldContinue,
|
|
2528
|
+
});
|
|
2529
|
+
return SyncEngineLevel.recordsInitialWritesFromVerifiedDependencies(verified);
|
|
2530
|
+
});
|
|
2531
|
+
}
|
|
2532
|
+
verifyProjectedDependencies(tenantDid, scope, primaryEntries, dependencies) {
|
|
2533
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2534
|
+
const primaryByCid = SyncEngineLevel.indexEntriesWithMessage(primaryEntries);
|
|
2535
|
+
const initialWritesByRoot = yield this.collectProjectedRecordsInitialWriteDependencies(scope, primaryByCid, dependencies);
|
|
2536
|
+
const protocolConfigs = yield this.verifyProjectedProtocolConfigDependencies(tenantDid, scope, primaryEntries, dependencies, initialWritesByRoot);
|
|
2537
|
+
return SyncEngineLevel.dedupeDependencyEntries([
|
|
2538
|
+
...protocolConfigs,
|
|
2539
|
+
...initialWritesByRoot.values(),
|
|
2540
|
+
]);
|
|
2541
|
+
});
|
|
2542
|
+
}
|
|
2543
|
+
verifyProjectedProtocolConfigDependencies(tenantDid_1, scope_1, primaryEntries_1, dependencies_1) {
|
|
2544
|
+
return __awaiter(this, arguments, void 0, function* (tenantDid, scope, primaryEntries, dependencies, initialWritesByRoot = new Map()) {
|
|
2545
|
+
var _a;
|
|
2546
|
+
// Projected sync dependency entries are untrusted server hints. Before any
|
|
2547
|
+
// config is applied, bind it to an accepted primary record by CID,
|
|
2548
|
+
// tenant authorship, signature, timestamp, scope, and protocol closure;
|
|
2549
|
+
// malformed or unrelated hints are ignored.
|
|
2550
|
+
const primaryByCid = SyncEngineLevel.indexEntriesWithMessage(primaryEntries);
|
|
2551
|
+
const candidatesByRoot = yield this.collectProjectedProtocolConfigCandidates(tenantDid, scope, primaryByCid, dependencies, initialWritesByRoot);
|
|
2552
|
+
const verified = new Map();
|
|
2553
|
+
for (const [rootMessageCid, rootCandidates] of candidatesByRoot) {
|
|
2554
|
+
const primary = primaryByCid.get(rootMessageCid);
|
|
2555
|
+
const rootRecordsWrite = primary === undefined
|
|
2556
|
+
? undefined
|
|
2557
|
+
: SyncEngineLevel.protocolConfigRootRecordsWrite(primary.message, (_a = initialWritesByRoot.get(rootMessageCid)) === null || _a === void 0 ? void 0 : _a.message);
|
|
2558
|
+
const rootProtocol = SyncEngineLevel.recordsWriteProtocol(rootRecordsWrite);
|
|
2559
|
+
if (rootProtocol === undefined) {
|
|
2560
|
+
continue;
|
|
2561
|
+
}
|
|
2562
|
+
for (const dependency of SyncEngineLevel.filterProtocolConfigClosure(rootProtocol, rootCandidates)) {
|
|
2563
|
+
verified.set(dependency.messageCid, dependency);
|
|
2564
|
+
}
|
|
2565
|
+
}
|
|
2566
|
+
return [...verified.values()];
|
|
2567
|
+
});
|
|
2568
|
+
}
|
|
2569
|
+
static indexEntriesWithMessage(entries) {
|
|
2570
|
+
const entriesByCid = new Map();
|
|
2571
|
+
for (const entry of entries) {
|
|
2572
|
+
if (SyncEngineLevel.hasMessage(entry)) {
|
|
2573
|
+
entriesByCid.set(entry.messageCid, entry);
|
|
2574
|
+
}
|
|
2575
|
+
}
|
|
2576
|
+
return entriesByCid;
|
|
2577
|
+
}
|
|
2578
|
+
static recordsInitialWritesFromVerifiedDependencies(entries) {
|
|
2579
|
+
const initialWrites = [];
|
|
2580
|
+
for (const entry of entries) {
|
|
2581
|
+
if (SyncEngineLevel.hasMessage(entry) && SyncEngineLevel.isRecordsWriteMessage(entry.message)) {
|
|
2582
|
+
initialWrites.push(entry.message);
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
return initialWrites;
|
|
2586
|
+
}
|
|
2587
|
+
collectProjectedRecordsInitialWriteDependencies(scope, primaryByCid, dependencies) {
|
|
2588
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2589
|
+
const dependenciesByRoot = new Map();
|
|
2590
|
+
for (const dependency of dependencies) {
|
|
2591
|
+
const verified = yield this.verifyRecordsInitialWriteCandidate(scope, primaryByCid, dependency);
|
|
2592
|
+
if (verified === undefined) {
|
|
2593
|
+
continue;
|
|
2594
|
+
}
|
|
2595
|
+
dependenciesByRoot.set(verified.rootMessageCid, verified.dependency);
|
|
2596
|
+
}
|
|
2597
|
+
return dependenciesByRoot;
|
|
2598
|
+
});
|
|
2599
|
+
}
|
|
2600
|
+
verifyRecordsInitialWriteCandidate(scope, primaryByCid, dependency) {
|
|
2601
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2602
|
+
if (dependency.dependencyClass !== 'recordsInitialWrite' ||
|
|
2603
|
+
!SyncEngineLevel.hasMessage(dependency) ||
|
|
2604
|
+
SyncEngineLevel.hasDependencyPayloadBytes(dependency)) {
|
|
2605
|
+
return undefined;
|
|
2606
|
+
}
|
|
2607
|
+
const primary = primaryByCid.get(dependency.rootMessageCid);
|
|
2608
|
+
if (primary === undefined ||
|
|
2609
|
+
!SyncEngineLevel.isRecordsDeleteMessage(primary.message) ||
|
|
2610
|
+
!(yield SyncEngineLevel.projectedDependencyCidsMatch({
|
|
2611
|
+
dependencyCid: dependency.messageCid,
|
|
2612
|
+
dependencyMessage: dependency.message,
|
|
2613
|
+
primaryCid: primary.messageCid,
|
|
2614
|
+
primaryMessage: primary.message,
|
|
2615
|
+
}))) {
|
|
2616
|
+
return undefined;
|
|
2617
|
+
}
|
|
2618
|
+
const initialWrite = yield this.toAuthenticatedRecordsInitialWriteDependency(dependency);
|
|
2619
|
+
if (initialWrite === undefined ||
|
|
2620
|
+
initialWrite.message.recordId !== SyncEngineLevel.recordsDeleteRecordId(primary.message) ||
|
|
2621
|
+
classifySyncMessageScope({ message: primary.message, initialWrite: initialWrite.message, scope }) !== 'in-scope') {
|
|
2622
|
+
return undefined;
|
|
2623
|
+
}
|
|
2624
|
+
return { dependency: initialWrite, rootMessageCid: dependency.rootMessageCid };
|
|
2625
|
+
});
|
|
2626
|
+
}
|
|
2627
|
+
toAuthenticatedRecordsInitialWriteDependency(dependency) {
|
|
2628
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2629
|
+
if (!SyncEngineLevel.isRecordsWriteMessage(dependency.message)) {
|
|
2630
|
+
return undefined;
|
|
2631
|
+
}
|
|
2632
|
+
try {
|
|
2633
|
+
const recordsWrite = yield RecordsWrite.parse(dependency.message);
|
|
2634
|
+
yield authenticate(recordsWrite.message.authorization, this.agent.did, recordsWrite.message.attestation);
|
|
2635
|
+
return (yield recordsWrite.isInitialWrite())
|
|
2636
|
+
? Object.assign(Object.assign({}, dependency), { message: recordsWrite.message }) : undefined;
|
|
2637
|
+
}
|
|
2638
|
+
catch (_a) {
|
|
2639
|
+
return undefined;
|
|
2640
|
+
}
|
|
2641
|
+
});
|
|
2642
|
+
}
|
|
2643
|
+
collectProjectedProtocolConfigCandidates(tenantDid, scope, primaryByCid, dependencies, initialWritesByRoot) {
|
|
2644
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2645
|
+
var _a;
|
|
2646
|
+
const candidatesByRoot = new Map();
|
|
2647
|
+
for (const dependency of dependencies) {
|
|
2648
|
+
const verified = yield this.verifyProtocolConfigCandidate(tenantDid, scope, primaryByCid, dependency, initialWritesByRoot);
|
|
2649
|
+
if (verified === undefined) {
|
|
2650
|
+
continue;
|
|
2651
|
+
}
|
|
2652
|
+
const rootCandidates = (_a = candidatesByRoot.get(verified.rootMessageCid)) !== null && _a !== void 0 ? _a : [];
|
|
2653
|
+
rootCandidates.push(verified.dependency);
|
|
2654
|
+
candidatesByRoot.set(verified.rootMessageCid, rootCandidates);
|
|
2655
|
+
}
|
|
2656
|
+
return candidatesByRoot;
|
|
2657
|
+
});
|
|
2658
|
+
}
|
|
2659
|
+
verifyProtocolConfigCandidate(tenantDid, scope, primaryByCid, dependency, initialWritesByRoot) {
|
|
2660
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2661
|
+
var _a;
|
|
2662
|
+
if (dependency.dependencyClass !== 'protocolsConfigure' || !SyncEngineLevel.hasMessage(dependency)) {
|
|
2663
|
+
return undefined;
|
|
2664
|
+
}
|
|
2665
|
+
const primary = primaryByCid.get(dependency.rootMessageCid);
|
|
2666
|
+
if (primary === undefined) {
|
|
2667
|
+
return undefined;
|
|
2668
|
+
}
|
|
2669
|
+
const verifiedDependency = yield this.verifyProtocolConfigCandidateMessage(tenantDid, scope, primary, dependency, (_a = initialWritesByRoot.get(dependency.rootMessageCid)) === null || _a === void 0 ? void 0 : _a.message);
|
|
2670
|
+
return verifiedDependency === undefined
|
|
2671
|
+
? undefined
|
|
2672
|
+
: { dependency: verifiedDependency, rootMessageCid: dependency.rootMessageCid };
|
|
2673
|
+
});
|
|
2674
|
+
}
|
|
2675
|
+
verifyProtocolConfigCandidateMessage(tenantDid, scope, primary, dependency, initialWrite) {
|
|
2676
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2677
|
+
// Protocol authorization is temporal: a record is governed by the protocol
|
|
2678
|
+
// definition active at its creation timestamp. Future configs may add
|
|
2679
|
+
// unrelated `uses` dependencies, so they must not widen this primary's
|
|
2680
|
+
// dependency closure.
|
|
2681
|
+
if (!(yield SyncEngineLevel.projectedDependencyCidsMatch({
|
|
2682
|
+
dependencyCid: dependency.messageCid,
|
|
2683
|
+
dependencyMessage: dependency.message,
|
|
2684
|
+
primaryCid: primary.messageCid,
|
|
2685
|
+
primaryMessage: primary.message,
|
|
2686
|
+
}))) {
|
|
2687
|
+
return undefined;
|
|
2688
|
+
}
|
|
2689
|
+
const authenticatedDependency = yield this.toAuthenticatedProtocolConfigDependency(tenantDid, dependency);
|
|
2690
|
+
if (authenticatedDependency === undefined) {
|
|
2691
|
+
return undefined;
|
|
2692
|
+
}
|
|
2693
|
+
const rootRecordsWrite = SyncEngineLevel.protocolConfigRootRecordsWrite(primary.message, initialWrite);
|
|
2694
|
+
const primaryIsInScope = rootRecordsWrite !== undefined &&
|
|
2695
|
+
classifySyncMessageScope({ message: primary.message, initialWrite, scope }) === 'in-scope';
|
|
2696
|
+
if (!primaryIsInScope ||
|
|
2697
|
+
!SyncEngineLevel.protocolsConfigureIsNotNewerThanRecordsWrite(authenticatedDependency.message, rootRecordsWrite)) {
|
|
2698
|
+
return undefined;
|
|
2699
|
+
}
|
|
2700
|
+
return authenticatedDependency;
|
|
2701
|
+
});
|
|
2702
|
+
}
|
|
2703
|
+
toAuthenticatedProtocolConfigDependency(tenantDid, dependency) {
|
|
2704
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2705
|
+
if (!SyncEngineLevel.isProtocolsConfigureDefinitionMessage(dependency.message)) {
|
|
2706
|
+
return undefined;
|
|
2707
|
+
}
|
|
2708
|
+
try {
|
|
2709
|
+
yield ProtocolsConfigure.parse(dependency.message);
|
|
2710
|
+
if (Message.getAuthor(dependency.message) !== tenantDid) {
|
|
2711
|
+
return undefined;
|
|
2712
|
+
}
|
|
2713
|
+
yield authenticate(dependency.message.authorization, this.agent.did);
|
|
2714
|
+
return Object.assign(Object.assign({}, dependency), { message: dependency.message });
|
|
2715
|
+
}
|
|
2716
|
+
catch (_a) {
|
|
2717
|
+
return undefined;
|
|
2718
|
+
}
|
|
2719
|
+
});
|
|
2720
|
+
}
|
|
2721
|
+
static projectedDependencyCidsMatch(_a) {
|
|
2722
|
+
return __awaiter(this, arguments, void 0, function* ({ dependencyCid, dependencyMessage, primaryCid, primaryMessage, }) {
|
|
2723
|
+
return (yield Message.getCid(primaryMessage)) === primaryCid &&
|
|
2724
|
+
(yield Message.getCid(dependencyMessage)) === dependencyCid;
|
|
2725
|
+
});
|
|
2726
|
+
}
|
|
2727
|
+
static recordsWriteProtocol(message) {
|
|
2728
|
+
if (!SyncEngineLevel.isRecordsWriteProtocolMessage(message)) {
|
|
2729
|
+
return undefined;
|
|
2730
|
+
}
|
|
2731
|
+
const { protocol } = message.descriptor;
|
|
2732
|
+
return typeof protocol === 'string' ? protocol : undefined;
|
|
2733
|
+
}
|
|
2734
|
+
static recordsDeleteRecordId(message) {
|
|
2735
|
+
if (!SyncEngineLevel.isRecordsDeleteMessage(message)) {
|
|
2736
|
+
return undefined;
|
|
2737
|
+
}
|
|
2738
|
+
const recordId = message.descriptor.recordId;
|
|
2739
|
+
return typeof recordId === 'string' ? recordId : undefined;
|
|
2740
|
+
}
|
|
2741
|
+
static protocolConfigRootRecordsWrite(primary, initialWrite) {
|
|
2742
|
+
if (SyncEngineLevel.isRecordsWriteMessage(primary)) {
|
|
2743
|
+
return primary;
|
|
2744
|
+
}
|
|
2745
|
+
return SyncEngineLevel.isRecordsDeleteMessage(primary) ? initialWrite : undefined;
|
|
2746
|
+
}
|
|
2747
|
+
static protocolsConfigureProtocol(message) {
|
|
2748
|
+
return message.descriptor.definition.protocol;
|
|
2749
|
+
}
|
|
2750
|
+
static protocolsConfigureProtocolFromGenericMessage(message) {
|
|
2751
|
+
if (!SyncEngineLevel.isProtocolsConfigureDefinitionMessage(message)) {
|
|
2752
|
+
return undefined;
|
|
2753
|
+
}
|
|
2754
|
+
return message.descriptor.definition.protocol;
|
|
2755
|
+
}
|
|
2756
|
+
static protocolsConfigureIsNotNewerThanRecordsWrite(protocolsConfigureMessage, recordsWriteMessage) {
|
|
2757
|
+
return protocolsConfigureMessage.descriptor.messageTimestamp <= recordsWriteMessage.descriptor.messageTimestamp;
|
|
2758
|
+
}
|
|
2759
|
+
static protocolsConfigureUses(message) {
|
|
2760
|
+
var _a;
|
|
2761
|
+
const uses = (_a = message.descriptor.definition) === null || _a === void 0 ? void 0 : _a.uses;
|
|
2762
|
+
return uses === undefined
|
|
2763
|
+
? []
|
|
2764
|
+
: Object.values(uses).filter((protocol) => typeof protocol === 'string');
|
|
2765
|
+
}
|
|
2766
|
+
static hasMessage(entry) {
|
|
2767
|
+
return (entry === null || entry === void 0 ? void 0 : entry.message) !== undefined;
|
|
2768
|
+
}
|
|
2769
|
+
static isRecordsWriteProtocolMessage(message) {
|
|
2770
|
+
return (message === null || message === void 0 ? void 0 : message.descriptor.interface) === DwnInterfaceName.Records &&
|
|
2771
|
+
message.descriptor.method === DwnMethodName.Write;
|
|
2772
|
+
}
|
|
2773
|
+
static isRecordsWriteMessage(message) {
|
|
2774
|
+
return SyncEngineLevel.isRecordsWriteProtocolMessage(message) &&
|
|
2775
|
+
'recordId' in message &&
|
|
2776
|
+
typeof message.recordId === 'string' &&
|
|
2777
|
+
'contextId' in message &&
|
|
2778
|
+
typeof message.contextId === 'string';
|
|
2779
|
+
}
|
|
2780
|
+
static isRecordsDeleteMessage(message) {
|
|
2781
|
+
return message.descriptor.interface === DwnInterfaceName.Records &&
|
|
2782
|
+
message.descriptor.method === DwnMethodName.Delete;
|
|
2783
|
+
}
|
|
2784
|
+
static isProtocolsConfigureDefinitionMessage(message) {
|
|
2785
|
+
return message.descriptor.interface === DwnInterfaceName.Protocols &&
|
|
2786
|
+
message.descriptor.method === DwnMethodName.Configure &&
|
|
2787
|
+
message.authorization !== undefined &&
|
|
2788
|
+
SyncEngineLevel.hasProtocolsConfigureDefinition(message.descriptor);
|
|
2789
|
+
}
|
|
2790
|
+
static hasProtocolsConfigureDefinition(descriptor) {
|
|
2791
|
+
return SyncEngineLevel.isProtocolsConfigureDefinition(descriptor.definition);
|
|
2792
|
+
}
|
|
2793
|
+
static isProtocolsConfigureDefinition(definition) {
|
|
2794
|
+
return typeof definition === 'object' &&
|
|
2795
|
+
definition !== null &&
|
|
2796
|
+
'protocol' in definition &&
|
|
2797
|
+
typeof definition.protocol === 'string';
|
|
2798
|
+
}
|
|
2799
|
+
static filterProtocolConfigClosure(primaryProtocol, candidates) {
|
|
2800
|
+
// Start from the primary record's protocol and walk only protocols named by
|
|
2801
|
+
// accepted, signed config definitions. This keeps composed-protocol support
|
|
2802
|
+
// narrow: the governing config can admit its `uses` targets, but arbitrary
|
|
2803
|
+
// protocol config hints cannot enter the apply set.
|
|
2804
|
+
const candidatesByProtocol = SyncEngineLevel.groupGoverningProtocolConfigCandidatesByProtocol(candidates);
|
|
2805
|
+
const visitedProtocols = new Set();
|
|
2806
|
+
const pendingProtocols = [primaryProtocol];
|
|
2807
|
+
const accepted = new Map();
|
|
2808
|
+
for (let protocol = SyncEngineLevel.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols); protocol !== undefined; protocol = SyncEngineLevel.takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols)) {
|
|
2809
|
+
SyncEngineLevel.acceptProtocolConfigCandidates({
|
|
2810
|
+
protocol,
|
|
2811
|
+
candidatesByProtocol,
|
|
2812
|
+
visitedProtocols,
|
|
2813
|
+
pendingProtocols,
|
|
2814
|
+
accepted,
|
|
2815
|
+
});
|
|
2816
|
+
}
|
|
2817
|
+
return [...accepted.values()];
|
|
2818
|
+
}
|
|
2819
|
+
static groupGoverningProtocolConfigCandidatesByProtocol(candidates) {
|
|
2820
|
+
const candidatesByProtocol = new Map();
|
|
2821
|
+
for (const candidate of candidates) {
|
|
2822
|
+
const protocol = SyncEngineLevel.protocolsConfigureProtocol(candidate.message);
|
|
2823
|
+
const existing = candidatesByProtocol.get(protocol);
|
|
2824
|
+
if (existing !== undefined && SyncEngineLevel.isProtocolConfigCandidateAtLeastAsNew(existing, candidate)) {
|
|
2825
|
+
continue;
|
|
2826
|
+
}
|
|
2827
|
+
candidatesByProtocol.set(protocol, candidate);
|
|
2828
|
+
}
|
|
2829
|
+
return candidatesByProtocol;
|
|
2830
|
+
}
|
|
2831
|
+
static isProtocolConfigCandidateAtLeastAsNew(existing, candidate) {
|
|
2832
|
+
const existingTimestamp = existing.message.descriptor.messageTimestamp;
|
|
2833
|
+
const candidateTimestamp = candidate.message.descriptor.messageTimestamp;
|
|
2834
|
+
if (existingTimestamp !== candidateTimestamp) {
|
|
2835
|
+
return existingTimestamp > candidateTimestamp;
|
|
2836
|
+
}
|
|
2837
|
+
return lexicographicalCompare(existing.messageCid, candidate.messageCid) >= 0;
|
|
2838
|
+
}
|
|
2839
|
+
static takeNextUnvisitedProtocol(pendingProtocols, visitedProtocols) {
|
|
2840
|
+
while (pendingProtocols.length > 0) {
|
|
2841
|
+
const protocol = pendingProtocols.shift();
|
|
2842
|
+
if (visitedProtocols.has(protocol)) {
|
|
2843
|
+
continue;
|
|
2844
|
+
}
|
|
2845
|
+
visitedProtocols.add(protocol);
|
|
2846
|
+
return protocol;
|
|
2847
|
+
}
|
|
2848
|
+
return undefined;
|
|
2849
|
+
}
|
|
2850
|
+
static acceptProtocolConfigCandidates({ protocol, candidatesByProtocol, visitedProtocols, pendingProtocols, accepted, }) {
|
|
2851
|
+
const candidate = candidatesByProtocol.get(protocol);
|
|
2852
|
+
if (candidate === undefined) {
|
|
1974
2853
|
return;
|
|
1975
2854
|
}
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
2855
|
+
accepted.set(candidate.messageCid, candidate);
|
|
2856
|
+
SyncEngineLevel.queueUnvisitedProtocols(SyncEngineLevel.protocolsConfigureUses(candidate.message), visitedProtocols, pendingProtocols);
|
|
2857
|
+
}
|
|
2858
|
+
static queueUnvisitedProtocols(protocols, visitedProtocols, pendingProtocols) {
|
|
2859
|
+
for (const protocol of protocols) {
|
|
2860
|
+
if (!visitedProtocols.has(protocol)) {
|
|
2861
|
+
pendingProtocols.push(protocol);
|
|
2862
|
+
}
|
|
1981
2863
|
}
|
|
1982
|
-
pushRuntime.timer = setTimeout(() => {
|
|
1983
|
-
pushRuntime.timer = undefined;
|
|
1984
|
-
void this.flushPendingPushesForLink(targetKey);
|
|
1985
|
-
}, delayMs);
|
|
1986
2864
|
}
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
2865
|
+
static protocolSetScopeForProjectedDependencies(dependencies) {
|
|
2866
|
+
// Verification above is the security boundary. This protocolSet scope only
|
|
2867
|
+
// routes already-verified config dependencies through the existing
|
|
2868
|
+
// pull/apply path, which expects every prefetched message to be accepted by
|
|
2869
|
+
// the supplied sync scope before it reaches processRawMessage().
|
|
2870
|
+
const protocols = SyncEngineLevel.dedupeStrings(dependencies
|
|
2871
|
+
.flatMap(dependency => SyncEngineLevel.projectedDependencyProtocols(dependency))).sort(lexicographicalCompare);
|
|
2872
|
+
if (protocols.length === 0) {
|
|
2873
|
+
throw new Error('SyncEngineLevel: projected dependency hints contained no protocols.');
|
|
2874
|
+
}
|
|
2875
|
+
return {
|
|
2876
|
+
kind: 'protocolSet',
|
|
2877
|
+
protocols: protocols,
|
|
2878
|
+
};
|
|
2879
|
+
}
|
|
2880
|
+
static projectedDependencyProtocols(dependency) {
|
|
2881
|
+
if (dependency.message === undefined) {
|
|
2882
|
+
return [];
|
|
2883
|
+
}
|
|
2884
|
+
const protocol = dependency.dependencyClass === 'protocolsConfigure'
|
|
2885
|
+
? SyncEngineLevel.protocolsConfigureProtocolFromGenericMessage(dependency.message)
|
|
2886
|
+
: SyncEngineLevel.recordsWriteProtocol(dependency.message);
|
|
2887
|
+
return protocol === undefined ? [] : [protocol];
|
|
2888
|
+
}
|
|
2889
|
+
static hasDependencyPayloadBytes(dependency) {
|
|
2890
|
+
if (dependency.encodedData !== undefined) {
|
|
2891
|
+
return true;
|
|
2892
|
+
}
|
|
2893
|
+
const message = dependency.message;
|
|
2894
|
+
return message !== undefined && 'encodedData' in message;
|
|
2895
|
+
}
|
|
2896
|
+
static dedupeDependencyEntries(dependencies) {
|
|
2897
|
+
const deduped = new Map();
|
|
2898
|
+
for (const dependency of dependencies) {
|
|
2899
|
+
deduped.set(dependency.messageCid, dependency);
|
|
2900
|
+
}
|
|
2901
|
+
return [...deduped.values()];
|
|
2902
|
+
}
|
|
2903
|
+
pushLocalDiffEntries(target, onlyLocal, permissionGrantIds, shouldContinue) {
|
|
2904
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2905
|
+
yield this.pushMessages({
|
|
2906
|
+
did: target.did,
|
|
2907
|
+
dwnUrl: target.dwnUrl,
|
|
2908
|
+
delegateDid: target.delegateDid,
|
|
2909
|
+
permissionGrantIds,
|
|
2910
|
+
messageCids: SyncEngineLevel.dedupeStrings(onlyLocal),
|
|
2911
|
+
});
|
|
2912
|
+
return (shouldContinue === null || shouldContinue === void 0 ? void 0 : shouldContinue()) === false;
|
|
2913
|
+
});
|
|
2914
|
+
}
|
|
2915
|
+
verifyProtocolSetConvergence(target, changedProtocols, permissionGrantIds, shouldContinue) {
|
|
2916
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2917
|
+
for (const protocol of changedProtocols) {
|
|
2918
|
+
const roots = yield this.getProtocolRoots(target, protocol, permissionGrantIds, shouldContinue);
|
|
2919
|
+
if (!roots) {
|
|
2920
|
+
return { aborted: true };
|
|
2921
|
+
}
|
|
2922
|
+
if (roots.localRoot !== roots.remoteRoot) {
|
|
2923
|
+
return { converged: false };
|
|
2924
|
+
}
|
|
2925
|
+
}
|
|
2926
|
+
return { converged: true };
|
|
2927
|
+
});
|
|
2928
|
+
}
|
|
2929
|
+
static dedupeRemoteEntries(entries) {
|
|
2930
|
+
const seen = new Set();
|
|
2931
|
+
const unique = [];
|
|
2932
|
+
for (const entry of entries) {
|
|
2933
|
+
if (seen.has(entry.messageCid)) {
|
|
2934
|
+
continue;
|
|
2935
|
+
}
|
|
2936
|
+
seen.add(entry.messageCid);
|
|
2937
|
+
unique.push(entry);
|
|
2938
|
+
}
|
|
2939
|
+
return unique;
|
|
2940
|
+
}
|
|
2941
|
+
static dedupeStrings(values) {
|
|
2942
|
+
return [...new Set(values)];
|
|
2943
|
+
}
|
|
2944
|
+
clearRootConvergenceDeadLettersForScope(tenantDid, remoteEndpoint, scope) {
|
|
2945
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
2946
|
+
if (scope.kind === 'recordsProjection' || (scope.kind === 'protocolSet' && scope.protocols.length > 1)) {
|
|
2947
|
+
// Batched multi-protocol and projected pulls pass the full scope to
|
|
2948
|
+
// pullMessages, so pull dead letters can be recorded without a single
|
|
2949
|
+
// protocol bucket.
|
|
2950
|
+
yield this.clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint);
|
|
2951
|
+
}
|
|
2952
|
+
for (const protocol of this.getReconcileProtocols(scope)) {
|
|
2953
|
+
yield this.clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint, protocol);
|
|
2954
|
+
}
|
|
1995
2955
|
});
|
|
1996
2956
|
}
|
|
1997
2957
|
/**
|
|
@@ -2068,18 +3028,28 @@ export class SyncEngineLevel {
|
|
|
2068
3028
|
// closure's captured `link` reference may no longer be the active
|
|
2069
3029
|
// link object. Bail before mutating the replacement's state.
|
|
2070
3030
|
const isStaleLink = () => this._activeLinks.get(linkKey) !== link;
|
|
2071
|
-
const
|
|
3031
|
+
const shouldContinue = () => this._engineGeneration === generation &&
|
|
3032
|
+
!isStaleLink() &&
|
|
3033
|
+
link.status === 'live';
|
|
3034
|
+
const { tenantDid: did, remoteEndpoint: dwnUrl, delegateDid, scope, authorization } = link;
|
|
3035
|
+
const eventScope = syncEventScope(scope);
|
|
2072
3036
|
try {
|
|
2073
|
-
const reconcileOutcome = yield this.
|
|
3037
|
+
const reconcileOutcome = yield this.reconcileProjectionTarget({
|
|
3038
|
+
did,
|
|
3039
|
+
dwnUrl,
|
|
3040
|
+
delegateDid,
|
|
3041
|
+
scope,
|
|
3042
|
+
authorization,
|
|
3043
|
+
}, { verifyConvergence: true }, shouldContinue);
|
|
2074
3044
|
if (reconcileOutcome.aborted || isStaleLink()) {
|
|
2075
3045
|
return;
|
|
2076
3046
|
}
|
|
2077
3047
|
if (reconcileOutcome.converged) {
|
|
2078
3048
|
yield this.ledger.clearNeedsReconcile(link);
|
|
2079
|
-
// SMT roots match
|
|
2080
|
-
//
|
|
2081
|
-
|
|
2082
|
-
this.emitEvent({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl,
|
|
3049
|
+
// SMT roots match, so transport/apply failures for this link may no
|
|
3050
|
+
// longer be current. Closure failures are not cleared by root equality.
|
|
3051
|
+
yield this.clearRootConvergenceDeadLettersForScope(did, dwnUrl, scope);
|
|
3052
|
+
this.emitEvent(Object.assign({ type: 'reconcile:completed', tenantDid: did, remoteEndpoint: dwnUrl }, eventScope));
|
|
2083
3053
|
}
|
|
2084
3054
|
else {
|
|
2085
3055
|
// Roots still differ — retry after a delay. This can happen when
|
|
@@ -2117,69 +3087,11 @@ export class SyncEngineLevel {
|
|
|
2117
3087
|
* Live-mode subscription methods (`openLivePullSubscription`,
|
|
2118
3088
|
* `openLocalPushSubscription`) receive `linkKey` directly and never
|
|
2119
3089
|
* call this. The remaining callers are poll-mode `sync()` and the
|
|
2120
|
-
* live-mode startup/error paths that already have
|
|
2121
|
-
*
|
|
2122
|
-
* The `undefined` fallback (which produces a legacy cursor key) exists
|
|
2123
|
-
* only for the no-protocol full-tenant targets in poll mode.
|
|
2124
|
-
*/
|
|
2125
|
-
buildLinkKey(did, dwnUrl, scopeIdOrProtocol) {
|
|
2126
|
-
return scopeIdOrProtocol ? buildLinkId(did, dwnUrl, scopeIdOrProtocol) : buildLegacyCursorKey(did, dwnUrl);
|
|
2127
|
-
}
|
|
2128
|
-
/**
|
|
2129
|
-
* @deprecated Used by poll-mode sync and one-time migration only. Live mode
|
|
2130
|
-
* uses ReplicationLedger checkpoints. Handles migration from old string cursors:
|
|
2131
|
-
* if the stored value is a bare string (pre-ProgressToken format), it is treated
|
|
2132
|
-
* as absent — the sync engine will do a full SMT reconciliation on first startup
|
|
2133
|
-
* after upgrade, which is correct and safe.
|
|
2134
|
-
*/
|
|
2135
|
-
getCursor(key) {
|
|
2136
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
2137
|
-
const cursors = this._db.sublevel('syncCursors');
|
|
2138
|
-
try {
|
|
2139
|
-
const raw = yield cursors.get(key);
|
|
2140
|
-
try {
|
|
2141
|
-
const parsed = JSON.parse(raw);
|
|
2142
|
-
if (parsed && typeof parsed === 'object' &&
|
|
2143
|
-
typeof parsed.streamId === 'string' && parsed.streamId.length > 0 &&
|
|
2144
|
-
typeof parsed.epoch === 'string' && parsed.epoch.length > 0 &&
|
|
2145
|
-
typeof parsed.position === 'string' && parsed.position.length > 0 &&
|
|
2146
|
-
typeof parsed.messageCid === 'string' && parsed.messageCid.length > 0) {
|
|
2147
|
-
return parsed;
|
|
2148
|
-
}
|
|
2149
|
-
}
|
|
2150
|
-
catch (_a) {
|
|
2151
|
-
// Not valid JSON (old string cursor) — fall through to delete.
|
|
2152
|
-
}
|
|
2153
|
-
// Entry exists but is unparseable or has invalid/empty fields. Delete it
|
|
2154
|
-
// so subsequent startups don't re-check it on every launch.
|
|
2155
|
-
yield this.deleteLegacyCursor(key);
|
|
2156
|
-
return undefined;
|
|
2157
|
-
}
|
|
2158
|
-
catch (error) {
|
|
2159
|
-
const e = error;
|
|
2160
|
-
if (e.code === 'LEVEL_NOT_FOUND') {
|
|
2161
|
-
return undefined;
|
|
2162
|
-
}
|
|
2163
|
-
throw error;
|
|
2164
|
-
}
|
|
2165
|
-
});
|
|
2166
|
-
}
|
|
2167
|
-
/**
|
|
2168
|
-
* Delete a legacy cursor from the old syncCursors sublevel.
|
|
2169
|
-
* Called as part of one-time migration to ReplicationLedger.
|
|
3090
|
+
* live-mode startup/error paths that already have a projection ID and
|
|
3091
|
+
* authorization epoch.
|
|
2170
3092
|
*/
|
|
2171
|
-
|
|
2172
|
-
return
|
|
2173
|
-
const cursors = this._db.sublevel('syncCursors');
|
|
2174
|
-
try {
|
|
2175
|
-
yield cursors.del(key);
|
|
2176
|
-
}
|
|
2177
|
-
catch (_a) {
|
|
2178
|
-
// Best-effort — ignore LEVEL_NOT_FOUND and transient I/O errors alike.
|
|
2179
|
-
// A failed delete leaves the bad entry for one more re-check on the
|
|
2180
|
-
// next startup, which is harmless.
|
|
2181
|
-
}
|
|
2182
|
-
});
|
|
3093
|
+
buildLinkKey(did, dwnUrl, projectionId, authorizationEpoch) {
|
|
3094
|
+
return buildLinkId(did, dwnUrl, projectionId, authorizationEpoch);
|
|
2183
3095
|
}
|
|
2184
3096
|
// ---------------------------------------------------------------------------
|
|
2185
3097
|
// Utility helpers
|
|
@@ -2270,7 +3182,7 @@ export class SyncEngineLevel {
|
|
|
2270
3182
|
*
|
|
2271
3183
|
* Returns a hex-encoded root hash string.
|
|
2272
3184
|
*/
|
|
2273
|
-
getLocalRoot(did, delegateDid, protocol) {
|
|
3185
|
+
getLocalRoot(did, delegateDid, protocol, permissionGrantIds) {
|
|
2274
3186
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2275
3187
|
var _a;
|
|
2276
3188
|
const si = this.stateIndex;
|
|
@@ -2281,7 +3193,6 @@ export class SyncEngineLevel {
|
|
|
2281
3193
|
return hashToHex(rootHash);
|
|
2282
3194
|
}
|
|
2283
3195
|
// Remote mode fallback: go through processRequest → RPC.
|
|
2284
|
-
const permissionGrantId = yield this.getSyncPermissionGrantId(did, delegateDid, protocol);
|
|
2285
3196
|
const response = yield this.agent.dwn.processRequest({
|
|
2286
3197
|
author: did,
|
|
2287
3198
|
target: did,
|
|
@@ -2290,7 +3201,7 @@ export class SyncEngineLevel {
|
|
|
2290
3201
|
messageParams: {
|
|
2291
3202
|
action: 'root',
|
|
2292
3203
|
protocol,
|
|
2293
|
-
|
|
3204
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds),
|
|
2294
3205
|
}
|
|
2295
3206
|
});
|
|
2296
3207
|
const reply = response.reply;
|
|
@@ -2301,10 +3212,9 @@ export class SyncEngineLevel {
|
|
|
2301
3212
|
* Get the SMT root hash from a remote DWN via a MessagesSync 'root' action.
|
|
2302
3213
|
* Returns a hex-encoded root hash string.
|
|
2303
3214
|
*/
|
|
2304
|
-
getRemoteRoot(did, dwnUrl, delegateDid, protocol) {
|
|
3215
|
+
getRemoteRoot(did, dwnUrl, delegateDid, protocol, permissionGrantIds) {
|
|
2305
3216
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2306
3217
|
var _a;
|
|
2307
|
-
const permissionGrantId = yield this.getSyncPermissionGrantId(did, delegateDid, protocol);
|
|
2308
3218
|
const syncMessage = yield this.agent.dwn.processRequest({
|
|
2309
3219
|
store: false,
|
|
2310
3220
|
author: did,
|
|
@@ -2314,7 +3224,59 @@ export class SyncEngineLevel {
|
|
|
2314
3224
|
messageParams: {
|
|
2315
3225
|
action: 'root',
|
|
2316
3226
|
protocol,
|
|
2317
|
-
|
|
3227
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds)
|
|
3228
|
+
}
|
|
3229
|
+
});
|
|
3230
|
+
const reply = yield this.agent.rpc.sendDwnRequest({
|
|
3231
|
+
dwnUrl,
|
|
3232
|
+
targetDid: did,
|
|
3233
|
+
message: syncMessage.message,
|
|
3234
|
+
});
|
|
3235
|
+
return (_a = reply.root) !== null && _a !== void 0 ? _a : '';
|
|
3236
|
+
});
|
|
3237
|
+
}
|
|
3238
|
+
getLocalProjectedRoot(did, delegateDid, scopes, permissionGrantIds) {
|
|
3239
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3240
|
+
var _a;
|
|
3241
|
+
if (this.stateIndex) {
|
|
3242
|
+
// Local projected roots use the already-derived scope directly. The
|
|
3243
|
+
// remote root/diff request still re-authorizes the invoked grant set.
|
|
3244
|
+
return RecordsProjection.getRootHex({
|
|
3245
|
+
tenant: did,
|
|
3246
|
+
messageStore: this.agent.dwn.node.storage.messageStore,
|
|
3247
|
+
scopes,
|
|
3248
|
+
});
|
|
3249
|
+
}
|
|
3250
|
+
const response = yield this.agent.dwn.processRequest({
|
|
3251
|
+
author: did,
|
|
3252
|
+
target: did,
|
|
3253
|
+
messageType: DwnInterface.MessagesSync,
|
|
3254
|
+
granteeDid: delegateDid,
|
|
3255
|
+
messageParams: {
|
|
3256
|
+
action: 'root',
|
|
3257
|
+
projectionRootVersion: RECORDS_PROJECTION_ROOT_VERSION,
|
|
3258
|
+
projectionScopes: [...scopes],
|
|
3259
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds),
|
|
3260
|
+
}
|
|
3261
|
+
});
|
|
3262
|
+
const reply = response.reply;
|
|
3263
|
+
return (_a = reply.root) !== null && _a !== void 0 ? _a : '';
|
|
3264
|
+
});
|
|
3265
|
+
}
|
|
3266
|
+
getRemoteProjectedRoot(did, dwnUrl, delegateDid, scopes, permissionGrantIds) {
|
|
3267
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3268
|
+
var _a;
|
|
3269
|
+
const syncMessage = yield this.agent.dwn.processRequest({
|
|
3270
|
+
store: false,
|
|
3271
|
+
author: did,
|
|
3272
|
+
target: did,
|
|
3273
|
+
messageType: DwnInterface.MessagesSync,
|
|
3274
|
+
granteeDid: delegateDid,
|
|
3275
|
+
messageParams: {
|
|
3276
|
+
action: 'root',
|
|
3277
|
+
projectionRootVersion: RECORDS_PROJECTION_ROOT_VERSION,
|
|
3278
|
+
projectionScopes: [...scopes],
|
|
3279
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds),
|
|
2318
3280
|
}
|
|
2319
3281
|
});
|
|
2320
3282
|
const reply = yield this.agent.rpc.sendDwnRequest({
|
|
@@ -2343,43 +3305,62 @@ export class SyncEngineLevel {
|
|
|
2343
3305
|
* This replaces `walkTreeDiff()` which required one HTTP call per tree node.
|
|
2344
3306
|
*/
|
|
2345
3307
|
diffWithRemote(_a) {
|
|
2346
|
-
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, protocol }) {
|
|
2347
|
-
var _b, _c;
|
|
3308
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, protocol, permissionGrantIds }) {
|
|
2348
3309
|
// Step 1: Collect local subtree hashes at BATCHED_DIFF_DEPTH directly from StateIndex.
|
|
2349
|
-
const localHashes = yield this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH);
|
|
3310
|
+
const localHashes = yield this.collectLocalSubtreeHashes(did, protocol, BATCHED_DIFF_DEPTH, permissionGrantIds);
|
|
2350
3311
|
// Step 2: Send a single 'diff' request to the remote with our hashes.
|
|
2351
|
-
const
|
|
3312
|
+
const messageParams = {
|
|
3313
|
+
action: 'diff',
|
|
3314
|
+
protocol,
|
|
3315
|
+
hashes: localHashes,
|
|
3316
|
+
depth: BATCHED_DIFF_DEPTH,
|
|
3317
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds),
|
|
3318
|
+
};
|
|
3319
|
+
// Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
|
|
3320
|
+
// Reuse the same grant set from step 2.
|
|
3321
|
+
return this.diffRemoteMessages({ did, dwnUrl, delegateDid }, messageParams, prefix => this.getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantIds), 'diff');
|
|
3322
|
+
});
|
|
3323
|
+
}
|
|
3324
|
+
diffProjectedWithRemote(_a) {
|
|
3325
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, scopes, permissionGrantIds }) {
|
|
3326
|
+
const localHashes = yield this.collectLocalProjectedSubtreeHashes(did, scopes, BATCHED_DIFF_DEPTH, delegateDid, permissionGrantIds);
|
|
3327
|
+
const messageParams = {
|
|
3328
|
+
action: 'diff',
|
|
3329
|
+
projectionRootVersion: RECORDS_PROJECTION_ROOT_VERSION,
|
|
3330
|
+
projectionScopes: [...scopes],
|
|
3331
|
+
hashes: localHashes,
|
|
3332
|
+
depth: BATCHED_DIFF_DEPTH,
|
|
3333
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds),
|
|
3334
|
+
};
|
|
3335
|
+
return this.diffRemoteMessages({ did, dwnUrl, delegateDid }, messageParams, prefix => this.getLocalProjectedLeaves(did, prefix, delegateDid, scopes, permissionGrantIds), 'projected diff');
|
|
3336
|
+
});
|
|
3337
|
+
}
|
|
3338
|
+
diffRemoteMessages(target, messageParams, getLocalLeavesForPrefix, operationName) {
|
|
3339
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3340
|
+
var _a, _b, _c;
|
|
2352
3341
|
const syncMessage = yield this.agent.dwn.processRequest({
|
|
2353
3342
|
store: false,
|
|
2354
|
-
author: did,
|
|
2355
|
-
target: did,
|
|
3343
|
+
author: target.did,
|
|
3344
|
+
target: target.did,
|
|
2356
3345
|
messageType: DwnInterface.MessagesSync,
|
|
2357
|
-
granteeDid: delegateDid,
|
|
2358
|
-
messageParams
|
|
2359
|
-
action: 'diff',
|
|
2360
|
-
protocol,
|
|
2361
|
-
hashes: localHashes,
|
|
2362
|
-
depth: BATCHED_DIFF_DEPTH,
|
|
2363
|
-
permissionGrantId,
|
|
2364
|
-
}
|
|
3346
|
+
granteeDid: target.delegateDid,
|
|
3347
|
+
messageParams,
|
|
2365
3348
|
});
|
|
2366
3349
|
const reply = yield this.agent.rpc.sendDwnRequest({
|
|
2367
|
-
dwnUrl,
|
|
2368
|
-
targetDid: did,
|
|
3350
|
+
dwnUrl: target.dwnUrl,
|
|
3351
|
+
targetDid: target.did,
|
|
2369
3352
|
message: syncMessage.message,
|
|
2370
3353
|
});
|
|
2371
3354
|
if (reply.status.code !== 200) {
|
|
2372
|
-
throw new Error(`SyncEngineLevel:
|
|
3355
|
+
throw new Error(`SyncEngineLevel: ${operationName} failed with ${reply.status.code}: ${reply.status.detail}`);
|
|
2373
3356
|
}
|
|
2374
|
-
// Step 3: Enumerate local leaves for prefixes the remote reported as onlyLocal.
|
|
2375
|
-
// Reuse the same grant ID from step 2 (avoids redundant lookup).
|
|
2376
|
-
const permissionGrantIdForLeaves = permissionGrantId;
|
|
2377
3357
|
const onlyLocalCids = [];
|
|
2378
|
-
for (const prefix of (
|
|
2379
|
-
const leaves = yield
|
|
3358
|
+
for (const prefix of (_a = reply.onlyLocal) !== null && _a !== void 0 ? _a : []) {
|
|
3359
|
+
const leaves = yield getLocalLeavesForPrefix(prefix);
|
|
2380
3360
|
onlyLocalCids.push(...leaves);
|
|
2381
3361
|
}
|
|
2382
3362
|
return {
|
|
3363
|
+
dependencies: (_b = reply.dependencies) !== null && _b !== void 0 ? _b : [],
|
|
2383
3364
|
onlyRemote: (_c = reply.onlyRemote) !== null && _c !== void 0 ? _c : [],
|
|
2384
3365
|
onlyLocal: onlyLocalCids,
|
|
2385
3366
|
};
|
|
@@ -2393,7 +3374,7 @@ export class SyncEngineLevel {
|
|
|
2393
3374
|
* Uses direct StateIndex access in local mode. In remote mode, falls back
|
|
2394
3375
|
* to `getLocalSubtreeHash` which routes through RPC.
|
|
2395
3376
|
*/
|
|
2396
|
-
collectLocalSubtreeHashes(did, protocol, depth) {
|
|
3377
|
+
collectLocalSubtreeHashes(did, protocol, depth, permissionGrantIds) {
|
|
2397
3378
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2398
3379
|
const result = {};
|
|
2399
3380
|
const defaultHash = yield this.getDefaultHashHex(depth);
|
|
@@ -2410,7 +3391,7 @@ export class SyncEngineLevel {
|
|
|
2410
3391
|
}
|
|
2411
3392
|
else {
|
|
2412
3393
|
// Remote mode fallback.
|
|
2413
|
-
hexHash = yield this.getLocalSubtreeHash(did, prefix, undefined, protocol);
|
|
3394
|
+
hexHash = yield this.getLocalSubtreeHash(did, prefix, undefined, protocol, permissionGrantIds);
|
|
2414
3395
|
}
|
|
2415
3396
|
if (hexHash === defaultHash) {
|
|
2416
3397
|
// Empty subtree — omit from the map.
|
|
@@ -2430,13 +3411,50 @@ export class SyncEngineLevel {
|
|
|
2430
3411
|
return result;
|
|
2431
3412
|
});
|
|
2432
3413
|
}
|
|
3414
|
+
collectLocalProjectedSubtreeHashes(did, scopes, depth, delegateDid, permissionGrantIds) {
|
|
3415
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3416
|
+
const result = {};
|
|
3417
|
+
const snapshot = this.stateIndex
|
|
3418
|
+
? yield RecordsProjection.createSnapshot({
|
|
3419
|
+
tenant: did,
|
|
3420
|
+
messageStore: this.agent.dwn.node.storage.messageStore,
|
|
3421
|
+
scopes,
|
|
3422
|
+
})
|
|
3423
|
+
: undefined;
|
|
3424
|
+
try {
|
|
3425
|
+
const walk = (prefix, currentDepth) => __awaiter(this, void 0, void 0, function* () {
|
|
3426
|
+
const bitPath = SyncEngineLevel.parseBitPrefix(prefix);
|
|
3427
|
+
const hexHash = snapshot
|
|
3428
|
+
? hashToHex(yield snapshot.getSubtreeHash(bitPath))
|
|
3429
|
+
: yield this.getLocalProjectedSubtreeHash(did, prefix, delegateDid, scopes, permissionGrantIds);
|
|
3430
|
+
const defaultHash = yield this.getDefaultHashHex(currentDepth);
|
|
3431
|
+
if (hexHash === defaultHash) {
|
|
3432
|
+
return;
|
|
3433
|
+
}
|
|
3434
|
+
if (currentDepth >= depth) {
|
|
3435
|
+
result[prefix] = hexHash;
|
|
3436
|
+
return;
|
|
3437
|
+
}
|
|
3438
|
+
yield Promise.all([
|
|
3439
|
+
walk(prefix + '0', currentDepth + 1),
|
|
3440
|
+
walk(prefix + '1', currentDepth + 1),
|
|
3441
|
+
]);
|
|
3442
|
+
});
|
|
3443
|
+
yield walk('', 0);
|
|
3444
|
+
return result;
|
|
3445
|
+
}
|
|
3446
|
+
finally {
|
|
3447
|
+
yield (snapshot === null || snapshot === void 0 ? void 0 : snapshot.close());
|
|
3448
|
+
}
|
|
3449
|
+
});
|
|
3450
|
+
}
|
|
2433
3451
|
/**
|
|
2434
3452
|
* Get the subtree hash at a given bit prefix from the local DWN.
|
|
2435
3453
|
*
|
|
2436
3454
|
* In local mode: queries the StateIndex directly.
|
|
2437
3455
|
* In remote mode: constructs a signed MessagesSync message and routes through RPC.
|
|
2438
3456
|
*/
|
|
2439
|
-
getLocalSubtreeHash(did, prefix, delegateDid, protocol,
|
|
3457
|
+
getLocalSubtreeHash(did, prefix, delegateDid, protocol, permissionGrantIds) {
|
|
2440
3458
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2441
3459
|
var _a;
|
|
2442
3460
|
const si = this.stateIndex;
|
|
@@ -2457,7 +3475,27 @@ export class SyncEngineLevel {
|
|
|
2457
3475
|
action: 'subtree',
|
|
2458
3476
|
prefix,
|
|
2459
3477
|
protocol,
|
|
2460
|
-
|
|
3478
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds)
|
|
3479
|
+
}
|
|
3480
|
+
});
|
|
3481
|
+
const reply = response.reply;
|
|
3482
|
+
return (_a = reply.hash) !== null && _a !== void 0 ? _a : '';
|
|
3483
|
+
});
|
|
3484
|
+
}
|
|
3485
|
+
getLocalProjectedSubtreeHash(did, prefix, delegateDid, scopes, permissionGrantIds) {
|
|
3486
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3487
|
+
var _a;
|
|
3488
|
+
const response = yield this.agent.dwn.processRequest({
|
|
3489
|
+
author: did,
|
|
3490
|
+
target: did,
|
|
3491
|
+
messageType: DwnInterface.MessagesSync,
|
|
3492
|
+
granteeDid: delegateDid,
|
|
3493
|
+
messageParams: {
|
|
3494
|
+
action: 'subtree',
|
|
3495
|
+
prefix,
|
|
3496
|
+
projectionRootVersion: RECORDS_PROJECTION_ROOT_VERSION,
|
|
3497
|
+
projectionScopes: [...scopes],
|
|
3498
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds)
|
|
2461
3499
|
}
|
|
2462
3500
|
});
|
|
2463
3501
|
const reply = response.reply;
|
|
@@ -2470,7 +3508,7 @@ export class SyncEngineLevel {
|
|
|
2470
3508
|
* In local mode: queries the StateIndex directly.
|
|
2471
3509
|
* In remote mode: constructs a signed MessagesSync message and routes through RPC.
|
|
2472
3510
|
*/
|
|
2473
|
-
getLocalLeaves(did, prefix, delegateDid, protocol,
|
|
3511
|
+
getLocalLeaves(did, prefix, delegateDid, protocol, permissionGrantIds) {
|
|
2474
3512
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2475
3513
|
var _a;
|
|
2476
3514
|
const si = this.stateIndex;
|
|
@@ -2490,7 +3528,35 @@ export class SyncEngineLevel {
|
|
|
2490
3528
|
action: 'leaves',
|
|
2491
3529
|
prefix,
|
|
2492
3530
|
protocol,
|
|
2493
|
-
|
|
3531
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds)
|
|
3532
|
+
}
|
|
3533
|
+
});
|
|
3534
|
+
const reply = response.reply;
|
|
3535
|
+
return (_a = reply.entries) !== null && _a !== void 0 ? _a : [];
|
|
3536
|
+
});
|
|
3537
|
+
}
|
|
3538
|
+
getLocalProjectedLeaves(did, prefix, delegateDid, scopes, permissionGrantIds) {
|
|
3539
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3540
|
+
var _a;
|
|
3541
|
+
if (this.stateIndex) {
|
|
3542
|
+
return RecordsProjection.getLeaves({
|
|
3543
|
+
tenant: did,
|
|
3544
|
+
messageStore: this.agent.dwn.node.storage.messageStore,
|
|
3545
|
+
scopes,
|
|
3546
|
+
prefix: SyncEngineLevel.parseBitPrefix(prefix),
|
|
3547
|
+
});
|
|
3548
|
+
}
|
|
3549
|
+
const response = yield this.agent.dwn.processRequest({
|
|
3550
|
+
author: did,
|
|
3551
|
+
target: did,
|
|
3552
|
+
messageType: DwnInterface.MessagesSync,
|
|
3553
|
+
granteeDid: delegateDid,
|
|
3554
|
+
messageParams: {
|
|
3555
|
+
action: 'leaves',
|
|
3556
|
+
prefix,
|
|
3557
|
+
projectionRootVersion: RECORDS_PROJECTION_ROOT_VERSION,
|
|
3558
|
+
projectionScopes: [...scopes],
|
|
3559
|
+
permissionGrantIds: toMessagesPermissionGrantIds(permissionGrantIds)
|
|
2494
3560
|
}
|
|
2495
3561
|
});
|
|
2496
3562
|
const reply = response.reply;
|
|
@@ -2509,25 +3575,109 @@ export class SyncEngineLevel {
|
|
|
2509
3575
|
* Only `messageCids` that were NOT prefetched are fetched individually.
|
|
2510
3576
|
*/
|
|
2511
3577
|
pullMessages(_a) {
|
|
2512
|
-
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, protocol, messageCids, prefetched }) {
|
|
3578
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, protocol, scope, permissionGrantIds, messageCids, prefetched, verifiedInitialWrites, shouldContinue, }) {
|
|
3579
|
+
const acceptanceScope = scope !== null && scope !== void 0 ? scope : (protocol === undefined
|
|
3580
|
+
? { kind: 'full' }
|
|
3581
|
+
: { kind: 'protocolSet', protocols: [protocol] });
|
|
3582
|
+
const rejectedPullEntries = new Map();
|
|
2513
3583
|
const failedCids = yield pullMessages({
|
|
2514
|
-
did,
|
|
3584
|
+
did,
|
|
3585
|
+
dwnUrl,
|
|
3586
|
+
delegateDid,
|
|
3587
|
+
permissionGrantIds,
|
|
3588
|
+
messageCids,
|
|
3589
|
+
prefetched,
|
|
3590
|
+
shouldContinue,
|
|
2515
3591
|
agent: this.agent,
|
|
2516
|
-
|
|
3592
|
+
acceptEntry: (entry, entries) => __awaiter(this, void 0, void 0, function* () {
|
|
3593
|
+
const result = yield this.acceptPulledSyncEntry(did, acceptanceScope, entry, entries, verifiedInitialWrites);
|
|
3594
|
+
if (!result.accepted) {
|
|
3595
|
+
rejectedPullEntries.set(yield getMessageCid(entry.message), result);
|
|
3596
|
+
}
|
|
3597
|
+
return result.accepted;
|
|
3598
|
+
}),
|
|
2517
3599
|
});
|
|
2518
3600
|
// Record permanently failed pull entries in the dead letter store.
|
|
2519
3601
|
for (const cid of failedCids) {
|
|
3602
|
+
const rejection = rejectedPullEntries.get(cid);
|
|
2520
3603
|
yield this.recordDeadLetter({
|
|
2521
3604
|
messageCid: cid,
|
|
2522
3605
|
tenantDid: did,
|
|
2523
3606
|
remoteEndpoint: dwnUrl,
|
|
2524
3607
|
protocol,
|
|
2525
|
-
category: 'pull-processing',
|
|
2526
|
-
|
|
3608
|
+
category: rejection ? 'pull-scope-rejected' : 'pull-processing',
|
|
3609
|
+
errorCode: rejection === null || rejection === void 0 ? void 0 : rejection.classification,
|
|
3610
|
+
errorDetail: rejection
|
|
3611
|
+
? `pulled message rejected by ${rejection.classification} sync scope gate`
|
|
3612
|
+
: 'pull processing failed after retry passes exhausted',
|
|
2527
3613
|
});
|
|
2528
3614
|
}
|
|
2529
3615
|
});
|
|
2530
3616
|
}
|
|
3617
|
+
acceptPulledSyncEntry(did_1, scope_1, entry_1, entries_1) {
|
|
3618
|
+
return __awaiter(this, arguments, void 0, function* (did, scope, entry, entries, verifiedInitialWrites = []) {
|
|
3619
|
+
if (scope.kind === 'full') {
|
|
3620
|
+
return { accepted: true };
|
|
3621
|
+
}
|
|
3622
|
+
const initialWrite = yield this.resolvePulledDeleteInitialWrite(did, entry.message, entries, verifiedInitialWrites);
|
|
3623
|
+
const classification = classifySyncMessageScope({
|
|
3624
|
+
message: entry.message,
|
|
3625
|
+
initialWrite,
|
|
3626
|
+
scope,
|
|
3627
|
+
});
|
|
3628
|
+
if (classification === 'in-scope') {
|
|
3629
|
+
return { accepted: true };
|
|
3630
|
+
}
|
|
3631
|
+
const messageCid = yield getMessageCid(entry.message);
|
|
3632
|
+
console.warn(`SyncEngineLevel: refusing to apply ${classification} pulled message ${messageCid}`);
|
|
3633
|
+
return { accepted: false, classification };
|
|
3634
|
+
});
|
|
3635
|
+
}
|
|
3636
|
+
resolvePulledDeleteInitialWrite(did_1, message_1, entries_1) {
|
|
3637
|
+
return __awaiter(this, arguments, void 0, function* (did, message, entries, verifiedInitialWrites = []) {
|
|
3638
|
+
const descriptor = message.descriptor;
|
|
3639
|
+
if (descriptor.interface !== DwnInterfaceName.Records ||
|
|
3640
|
+
descriptor.method !== DwnMethodName.Delete ||
|
|
3641
|
+
typeof descriptor.recordId !== 'string') {
|
|
3642
|
+
return undefined;
|
|
3643
|
+
}
|
|
3644
|
+
if (!this.agent.dwn.isRemoteMode) {
|
|
3645
|
+
const localInitialWrite = yield RecordsWrite.fetchInitialRecordsWriteMessage(this.agent.dwn.node.storage.messageStore, did, descriptor.recordId);
|
|
3646
|
+
if (localInitialWrite) {
|
|
3647
|
+
return localInitialWrite;
|
|
3648
|
+
}
|
|
3649
|
+
}
|
|
3650
|
+
const verifiedInitialWrite = SyncEngineLevel.findInitialWriteByRecordId(descriptor.recordId, verifiedInitialWrites);
|
|
3651
|
+
if (verifiedInitialWrite !== undefined) {
|
|
3652
|
+
return verifiedInitialWrite;
|
|
3653
|
+
}
|
|
3654
|
+
// Batch entries are only used when the initial write has not been applied
|
|
3655
|
+
// locally yet. Verified dependency hints cover the projected remote-mode
|
|
3656
|
+
// path where the initial write was applied in the previous pull batch and
|
|
3657
|
+
// no embedded local message store is available. Batch entries are still
|
|
3658
|
+
// parsed as initial RecordsWrite messages, and processRawMessage
|
|
3659
|
+
// authenticates the delete before any local mutation occurs.
|
|
3660
|
+
return this.findInitialWriteInPullBatch(descriptor.recordId, entries);
|
|
3661
|
+
});
|
|
3662
|
+
}
|
|
3663
|
+
static findInitialWriteByRecordId(recordId, initialWrites) {
|
|
3664
|
+
return initialWrites.find(initialWrite => initialWrite.recordId === recordId);
|
|
3665
|
+
}
|
|
3666
|
+
findInitialWriteInPullBatch(recordId, entries) {
|
|
3667
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3668
|
+
for (const entry of entries) {
|
|
3669
|
+
if (entry.message.descriptor.interface !== DwnInterfaceName.Records ||
|
|
3670
|
+
entry.message.descriptor.method !== DwnMethodName.Write) {
|
|
3671
|
+
continue;
|
|
3672
|
+
}
|
|
3673
|
+
const candidate = entry.message;
|
|
3674
|
+
if (candidate.recordId === recordId && (yield RecordsWrite.isInitialWrite(candidate))) {
|
|
3675
|
+
return candidate;
|
|
3676
|
+
}
|
|
3677
|
+
}
|
|
3678
|
+
return undefined;
|
|
3679
|
+
});
|
|
3680
|
+
}
|
|
2531
3681
|
// ---------------------------------------------------------------------------
|
|
2532
3682
|
// Echo-loop suppression
|
|
2533
3683
|
// ---------------------------------------------------------------------------
|
|
@@ -2578,11 +3728,10 @@ export class SyncEngineLevel {
|
|
|
2578
3728
|
* in dependency order (topological sort).
|
|
2579
3729
|
*/
|
|
2580
3730
|
pushMessages(_a) {
|
|
2581
|
-
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid,
|
|
3731
|
+
return __awaiter(this, arguments, void 0, function* ({ did, dwnUrl, delegateDid, permissionGrantIds, messageCids }) {
|
|
2582
3732
|
return pushMessages({
|
|
2583
|
-
did, dwnUrl, delegateDid,
|
|
3733
|
+
did, dwnUrl, delegateDid, permissionGrantIds, messageCids,
|
|
2584
3734
|
agent: this.agent,
|
|
2585
|
-
permissionsApi: this._permissionsApi,
|
|
2586
3735
|
});
|
|
2587
3736
|
});
|
|
2588
3737
|
}
|
|
@@ -2606,8 +3755,8 @@ export class SyncEngineLevel {
|
|
|
2606
3755
|
* When `protocol` is undefined (full-tenant link), clears entries that
|
|
2607
3756
|
* also have no protocol.
|
|
2608
3757
|
*/
|
|
2609
|
-
clearDeadLettersForLink(
|
|
2610
|
-
return __awaiter(this,
|
|
3758
|
+
clearDeadLettersForLink(tenantDid_1, remoteEndpoint_1, protocol_1) {
|
|
3759
|
+
return __awaiter(this, arguments, void 0, function* (tenantDid, remoteEndpoint, protocol, options = {}) {
|
|
2611
3760
|
var _a, e_1, _b, _c;
|
|
2612
3761
|
const batch = [];
|
|
2613
3762
|
try {
|
|
@@ -2619,7 +3768,8 @@ export class SyncEngineLevel {
|
|
|
2619
3768
|
const entry = JSON.parse(value);
|
|
2620
3769
|
if (entry.tenantDid === tenantDid &&
|
|
2621
3770
|
entry.remoteEndpoint === remoteEndpoint &&
|
|
2622
|
-
entry.protocol === protocol
|
|
3771
|
+
entry.protocol === protocol &&
|
|
3772
|
+
(options.categories === undefined || options.categories.has(entry.category))) {
|
|
2623
3773
|
batch.push({ type: 'del', key });
|
|
2624
3774
|
}
|
|
2625
3775
|
}
|
|
@@ -2643,6 +3793,18 @@ export class SyncEngineLevel {
|
|
|
2643
3793
|
}
|
|
2644
3794
|
});
|
|
2645
3795
|
}
|
|
3796
|
+
clearRootConvergenceDeadLetters(tenantDid, remoteEndpoint, protocol) {
|
|
3797
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3798
|
+
try {
|
|
3799
|
+
yield this.clearDeadLettersForLink(tenantDid, remoteEndpoint, protocol, {
|
|
3800
|
+
categories: SyncEngineLevel.ROOT_CONVERGENCE_CLEARABLE_DEAD_LETTER_CATEGORIES,
|
|
3801
|
+
});
|
|
3802
|
+
}
|
|
3803
|
+
catch (error) {
|
|
3804
|
+
console.warn(`SyncEngineLevel: Failed to clear root-convergence dead letters for ${tenantDid} -> ${remoteEndpoint}`, error);
|
|
3805
|
+
}
|
|
3806
|
+
});
|
|
3807
|
+
}
|
|
2646
3808
|
/**
|
|
2647
3809
|
* Build a compound dead letter key. Different remotes can fail the same CID
|
|
2648
3810
|
* for different reasons, so the key includes the remote endpoint.
|
|
@@ -2689,7 +3851,7 @@ export class SyncEngineLevel {
|
|
|
2689
3851
|
finally { if (e_2) throw e_2.error; }
|
|
2690
3852
|
}
|
|
2691
3853
|
// Deterministic ordering: newest first so apps see the most recent failures.
|
|
2692
|
-
entries.sort((a, b) => b.failedAt
|
|
3854
|
+
entries.sort((a, b) => lexicographicalCompare(b.failedAt, a.failedAt));
|
|
2693
3855
|
return entries;
|
|
2694
3856
|
});
|
|
2695
3857
|
}
|
|
@@ -2775,12 +3937,17 @@ export class SyncEngineLevel {
|
|
|
2775
3937
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2776
3938
|
var _a, e_5, _b, _c;
|
|
2777
3939
|
let failedMessageCount = 0;
|
|
3940
|
+
let closureFailureCount = 0;
|
|
2778
3941
|
try {
|
|
2779
3942
|
for (var _d = true, _e = __asyncValues(this._deadLetters.iterator()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
|
|
2780
3943
|
_c = _f.value;
|
|
2781
3944
|
_d = false;
|
|
2782
|
-
const
|
|
3945
|
+
const [, value] = _c;
|
|
2783
3946
|
failedMessageCount++;
|
|
3947
|
+
const entry = JSON.parse(value);
|
|
3948
|
+
if (entry.category === 'closure') {
|
|
3949
|
+
closureFailureCount++;
|
|
3950
|
+
}
|
|
2784
3951
|
}
|
|
2785
3952
|
}
|
|
2786
3953
|
catch (e_5_1) { e_5 = { error: e_5_1 }; }
|
|
@@ -2790,35 +3957,90 @@ export class SyncEngineLevel {
|
|
|
2790
3957
|
}
|
|
2791
3958
|
finally { if (e_5) throw e_5.error; }
|
|
2792
3959
|
}
|
|
2793
|
-
//
|
|
2794
|
-
//
|
|
2795
|
-
//
|
|
3960
|
+
// Superseded authorization epochs can leave durable link state behind. Only
|
|
3961
|
+
// links that still belong to the current registered projection/epoch should
|
|
3962
|
+
// affect health. Endpoint-level orphan cleanup is a separate GC concern.
|
|
3963
|
+
const currentLinkIdentityKeys = yield this.getCurrentDurableLinkIdentityKeys();
|
|
2796
3964
|
let degradedLinkCount = 0;
|
|
2797
3965
|
const allLinks = yield this.ledger.getAllLinks();
|
|
2798
3966
|
for (const link of allLinks) {
|
|
2799
|
-
|
|
3967
|
+
const isCurrentLink = currentLinkIdentityKeys === undefined || currentLinkIdentityKeys.has(this.getDurableLinkIdentityKey(link));
|
|
3968
|
+
if (isCurrentLink && SyncEngineLevel.isUnhealthyLinkStatus(link.status)) {
|
|
2800
3969
|
degradedLinkCount++;
|
|
2801
3970
|
}
|
|
2802
3971
|
}
|
|
2803
3972
|
return {
|
|
2804
3973
|
connectivity: this.connectivityState,
|
|
2805
|
-
failedMessageCount,
|
|
2806
|
-
|
|
3974
|
+
failedMessageCount: failedMessageCount,
|
|
3975
|
+
closureFailureCount: closureFailureCount,
|
|
3976
|
+
degradedLinkCount: degradedLinkCount,
|
|
3977
|
+
syncHealthy: failedMessageCount === 0 && degradedLinkCount === 0,
|
|
2807
3978
|
};
|
|
2808
3979
|
});
|
|
2809
3980
|
}
|
|
3981
|
+
getCurrentDurableLinkIdentityKeys() {
|
|
3982
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
3983
|
+
var _a, e_6, _b, _c;
|
|
3984
|
+
try {
|
|
3985
|
+
const identityKeys = new Set();
|
|
3986
|
+
try {
|
|
3987
|
+
for (var _d = true, _e = __asyncValues(this._db.sublevel('registeredIdentities').iterator()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
|
|
3988
|
+
_c = _f.value;
|
|
3989
|
+
_d = false;
|
|
3990
|
+
const [did, options] = _c;
|
|
3991
|
+
let parsed;
|
|
3992
|
+
try {
|
|
3993
|
+
parsed = JSON.parse(options);
|
|
3994
|
+
}
|
|
3995
|
+
catch (error) {
|
|
3996
|
+
console.warn(`SyncEngineLevel: Corrupt sync options for ${did}, skipping health target:`, error);
|
|
3997
|
+
continue;
|
|
3998
|
+
}
|
|
3999
|
+
const scope = syncScopeFromProtocols(parsed.protocols);
|
|
4000
|
+
const resolutions = yield this.buildSyncTargetResolutions(did, scope, parsed);
|
|
4001
|
+
for (const resolution of resolutions) {
|
|
4002
|
+
const projectionId = yield computeProjectionId(did, resolution.scope);
|
|
4003
|
+
identityKeys.add(SyncEngineLevel.durableLinkIdentityKey(did, projectionId, resolution.authorizationEpoch));
|
|
4004
|
+
}
|
|
4005
|
+
}
|
|
4006
|
+
}
|
|
4007
|
+
catch (e_6_1) { e_6 = { error: e_6_1 }; }
|
|
4008
|
+
finally {
|
|
4009
|
+
try {
|
|
4010
|
+
if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
|
|
4011
|
+
}
|
|
4012
|
+
finally { if (e_6) throw e_6.error; }
|
|
4013
|
+
}
|
|
4014
|
+
return identityKeys;
|
|
4015
|
+
}
|
|
4016
|
+
catch (error) {
|
|
4017
|
+
console.warn('SyncEngineLevel: Failed to resolve current durable link identity keys for health; falling back to all durable links', error);
|
|
4018
|
+
return undefined;
|
|
4019
|
+
}
|
|
4020
|
+
});
|
|
4021
|
+
}
|
|
4022
|
+
getDurableLinkIdentityKey(link) {
|
|
4023
|
+
return SyncEngineLevel.durableLinkIdentityKey(link.tenantDid, link.projectionId, link.authorizationEpoch);
|
|
4024
|
+
}
|
|
4025
|
+
static durableLinkIdentityKey(tenantDid, projectionId, authorizationEpoch) {
|
|
4026
|
+
return `${tenantDid}^${projectionId}^${authorizationEpoch}`;
|
|
4027
|
+
}
|
|
4028
|
+
static isUnhealthyLinkStatus(status) {
|
|
4029
|
+
return status === 'repairing' || status === 'degraded_poll' || status === 'terminal_incomplete';
|
|
4030
|
+
}
|
|
2810
4031
|
// ---------------------------------------------------------------------------
|
|
2811
4032
|
// Sync targets
|
|
2812
4033
|
// ---------------------------------------------------------------------------
|
|
2813
4034
|
/**
|
|
2814
|
-
* Returns the list of sync targets:
|
|
4035
|
+
* Returns the list of sync targets: one canonical projection target per
|
|
4036
|
+
* registered DID and resolved DWN endpoint.
|
|
2815
4037
|
* Results are cached for up to 30 seconds to avoid redundant DID resolution
|
|
2816
4038
|
* on every sync tick. The cache is invalidated when identities are registered,
|
|
2817
4039
|
* unregistered, or updated.
|
|
2818
4040
|
*/
|
|
2819
4041
|
getSyncTargets() {
|
|
2820
4042
|
return __awaiter(this, void 0, void 0, function* () {
|
|
2821
|
-
var _a,
|
|
4043
|
+
var _a, e_7, _b, _c;
|
|
2822
4044
|
// Return cached targets if still valid.
|
|
2823
4045
|
if (this._syncTargetsCache
|
|
2824
4046
|
&& (Date.now() - this._syncTargetsCache.timestamp) < SyncEngineLevel.SYNC_TARGETS_CACHE_TTL_MS) {
|
|
@@ -2845,31 +4067,22 @@ export class SyncEngineLevel {
|
|
|
2845
4067
|
console.warn(`SyncEngineLevel: Corrupt sync options for ${did}, skipping identity:`, error);
|
|
2846
4068
|
continue;
|
|
2847
4069
|
}
|
|
2848
|
-
const { protocols, delegateDid } = parsed;
|
|
2849
4070
|
const dwnEndpointUrls = yield this.agent.dwn.getDwnEndpointUrlsForTarget(did);
|
|
2850
4071
|
if (dwnEndpointUrls.length === 0) {
|
|
2851
4072
|
anyEndpointMissing = true;
|
|
2852
4073
|
continue;
|
|
2853
4074
|
}
|
|
2854
4075
|
for (const dwnUrl of dwnEndpointUrls) {
|
|
2855
|
-
|
|
2856
|
-
// Sync all protocols (global tree).
|
|
2857
|
-
targets.push({ did, delegateDid, dwnUrl });
|
|
2858
|
-
}
|
|
2859
|
-
else {
|
|
2860
|
-
for (const protocol of protocols) {
|
|
2861
|
-
targets.push({ did, delegateDid, dwnUrl, protocol });
|
|
2862
|
-
}
|
|
2863
|
-
}
|
|
4076
|
+
targets.push(...yield this.buildSyncTargetsForEndpoint(did, dwnUrl, parsed));
|
|
2864
4077
|
}
|
|
2865
4078
|
}
|
|
2866
4079
|
}
|
|
2867
|
-
catch (
|
|
4080
|
+
catch (e_7_1) { e_7 = { error: e_7_1 }; }
|
|
2868
4081
|
finally {
|
|
2869
4082
|
try {
|
|
2870
4083
|
if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
|
|
2871
4084
|
}
|
|
2872
|
-
finally { if (
|
|
4085
|
+
finally { if (e_7) throw e_7.error; }
|
|
2873
4086
|
}
|
|
2874
4087
|
// Only cache when:
|
|
2875
4088
|
// - The result is non-empty (empty = transient resolution failure).
|
|
@@ -2885,25 +4098,6 @@ export class SyncEngineLevel {
|
|
|
2885
4098
|
return targets;
|
|
2886
4099
|
});
|
|
2887
4100
|
}
|
|
2888
|
-
/**
|
|
2889
|
-
* Gets the permission grant ID for MessagesSync if a delegateDid is provided.
|
|
2890
|
-
* Returns undefined if no delegate is in use (owner access).
|
|
2891
|
-
*/
|
|
2892
|
-
getSyncPermissionGrantId(did, delegateDid, protocol) {
|
|
2893
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
2894
|
-
if (!delegateDid) {
|
|
2895
|
-
return undefined;
|
|
2896
|
-
}
|
|
2897
|
-
const messagesSyncGrant = yield this._permissionsApi.getPermissionForRequest({
|
|
2898
|
-
connectedDid: did,
|
|
2899
|
-
messageType: DwnInterface.MessagesSync,
|
|
2900
|
-
delegateDid,
|
|
2901
|
-
protocol,
|
|
2902
|
-
cached: true
|
|
2903
|
-
});
|
|
2904
|
-
return messagesSyncGrant.grant.id;
|
|
2905
|
-
});
|
|
2906
|
-
}
|
|
2907
4101
|
}
|
|
2908
4102
|
/** TTL for echo-loop suppression entries (60 seconds). */
|
|
2909
4103
|
SyncEngineLevel.ECHO_SUPPRESS_TTL_MS = 60000;
|
|
@@ -2911,12 +4105,14 @@ SyncEngineLevel.ECHO_SUPPRESS_TTL_MS = 60000;
|
|
|
2911
4105
|
SyncEngineLevel.ECHO_SUPPRESS_MAX_ENTRIES = 10000;
|
|
2912
4106
|
/** TTL for the sync targets cache (30 seconds). */
|
|
2913
4107
|
SyncEngineLevel.SYNC_TARGETS_CACHE_TTL_MS = 30000;
|
|
4108
|
+
/** Backoff schedule for recently published did:dht records. */
|
|
4109
|
+
SyncEngineLevel.DID_RESOLUTION_RETRY_BACKOFF_MS = [2000, 4000, 8000];
|
|
2914
4110
|
/** Maximum consecutive failures before entering backoff. */
|
|
2915
4111
|
SyncEngineLevel.MAX_CONSECUTIVE_FAILURES = 5;
|
|
2916
4112
|
/** Backoff multiplier for consecutive failures (caps at 4x the configured interval). */
|
|
2917
4113
|
SyncEngineLevel.MAX_BACKOFF_MULTIPLIER = 4;
|
|
2918
4114
|
// ---------------------------------------------------------------------------
|
|
2919
|
-
// Per-link repair and degraded-poll orchestration
|
|
4115
|
+
// Per-link repair and degraded-poll orchestration
|
|
2920
4116
|
// ---------------------------------------------------------------------------
|
|
2921
4117
|
/** Maximum consecutive repair attempts before falling back to degraded_poll. */
|
|
2922
4118
|
SyncEngineLevel.MAX_REPAIR_ATTEMPTS = 3;
|
|
@@ -2924,4 +4120,5 @@ SyncEngineLevel.MAX_REPAIR_ATTEMPTS = 3;
|
|
|
2924
4120
|
SyncEngineLevel.REPAIR_BACKOFF_MS = [1000, 3000, 10000];
|
|
2925
4121
|
/** Push retry backoff schedule: immediate, 250ms, 1s, 2s, then give up. */
|
|
2926
4122
|
SyncEngineLevel.PUSH_RETRY_BACKOFF_MS = [0, 250, 1000, 2000];
|
|
4123
|
+
SyncEngineLevel.ROOT_CONVERGENCE_CLEARABLE_DEAD_LETTER_CATEGORIES = new Set(['push-permanent', 'push-exhausted', 'pull-processing', 'pull-scope-rejected']);
|
|
2927
4124
|
//# sourceMappingURL=sync-engine-level.js.map
|