@enbox/agent 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.mjs +11 -11
- package/dist/browser.mjs.map +4 -4
- package/dist/esm/anonymous-dwn-api.js +1 -1
- package/dist/esm/anonymous-dwn-api.js.map +1 -1
- package/dist/esm/connect.js +4 -10
- package/dist/esm/connect.js.map +1 -1
- package/dist/esm/dwn-api.js +144 -195
- package/dist/esm/dwn-api.js.map +1 -1
- package/dist/esm/dwn-protocol-cache.js +149 -0
- package/dist/esm/dwn-protocol-cache.js.map +1 -0
- package/dist/esm/dwn-record-upgrade.js +3 -3
- package/dist/esm/dwn-record-upgrade.js.map +1 -1
- package/dist/esm/hd-identity-vault.js +6 -5
- package/dist/esm/hd-identity-vault.js.map +1 -1
- package/dist/esm/identity-api.js +0 -2
- package/dist/esm/identity-api.js.map +1 -1
- package/dist/esm/oidc.js +2 -1
- package/dist/esm/oidc.js.map +1 -1
- package/dist/esm/permissions-api.js +24 -6
- package/dist/esm/permissions-api.js.map +1 -1
- package/dist/esm/prototyping/crypto/jose/jwe-flattened.js +1 -1
- package/dist/esm/prototyping/crypto/jose/jwe-flattened.js.map +1 -1
- package/dist/esm/prototyping/crypto/jose/jwe.js +11 -3
- package/dist/esm/prototyping/crypto/jose/jwe.js.map +1 -1
- package/dist/esm/store-data-protocols.js +2 -2
- package/dist/esm/store-data-protocols.js.map +1 -1
- package/dist/esm/sync-api.js +3 -0
- package/dist/esm/sync-api.js.map +1 -1
- package/dist/esm/sync-engine-level.js +447 -29
- package/dist/esm/sync-engine-level.js.map +1 -1
- package/dist/esm/test-harness.js +3 -5
- package/dist/esm/test-harness.js.map +1 -1
- package/dist/esm/types/dwn.js.map +1 -1
- package/dist/types/anonymous-dwn-api.d.ts +3 -3
- package/dist/types/anonymous-dwn-api.d.ts.map +1 -1
- package/dist/types/connect.d.ts.map +1 -1
- package/dist/types/dwn-api.d.ts +11 -18
- package/dist/types/dwn-api.d.ts.map +1 -1
- package/dist/types/dwn-protocol-cache.d.ts +76 -0
- package/dist/types/dwn-protocol-cache.d.ts.map +1 -0
- package/dist/types/hd-identity-vault.d.ts.map +1 -1
- package/dist/types/identity-api.d.ts.map +1 -1
- package/dist/types/oidc.d.ts.map +1 -1
- package/dist/types/permissions-api.d.ts.map +1 -1
- package/dist/types/prototyping/crypto/jose/jwe-flattened.d.ts.map +1 -1
- package/dist/types/prototyping/crypto/jose/jwe.d.ts +12 -2
- package/dist/types/prototyping/crypto/jose/jwe.d.ts.map +1 -1
- package/dist/types/sync-api.d.ts +3 -4
- package/dist/types/sync-api.d.ts.map +1 -1
- package/dist/types/sync-engine-level.d.ts +63 -5
- package/dist/types/sync-engine-level.d.ts.map +1 -1
- package/dist/types/test-harness.d.ts.map +1 -1
- package/dist/types/types/dwn.d.ts +18 -19
- package/dist/types/types/dwn.d.ts.map +1 -1
- package/dist/types/types/sync.d.ts +47 -5
- package/dist/types/types/sync.d.ts.map +1 -1
- package/package.json +6 -6
- package/src/anonymous-dwn-api.ts +4 -4
- package/src/connect.ts +4 -10
- package/src/dwn-api.ts +192 -250
- package/src/dwn-protocol-cache.ts +216 -0
- package/src/dwn-record-upgrade.ts +3 -3
- package/src/hd-identity-vault.ts +6 -5
- package/src/identity-api.ts +0 -2
- package/src/oidc.ts +2 -1
- package/src/permissions-api.ts +28 -6
- package/src/prototyping/crypto/jose/jwe-flattened.ts +4 -1
- package/src/prototyping/crypto/jose/jwe.ts +24 -2
- package/src/store-data-protocols.ts +2 -2
- package/src/sync-api.ts +7 -3
- package/src/sync-engine-level.ts +509 -32
- package/src/test-harness.ts +3 -5
- package/src/types/dwn.ts +19 -21
- package/src/types/sync.ts +56 -5
package/src/sync-engine-level.ts
CHANGED
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
import type { AbstractLevel } from 'abstract-level';
|
|
2
|
-
import type { GenericMessage, MessagesSyncReply } from '@enbox/dwn-sdk-js';
|
|
2
|
+
import type { GenericMessage, MessageEvent, MessagesSubscribeReply, MessagesSyncReply, SubscriptionMessage } from '@enbox/dwn-sdk-js';
|
|
3
3
|
|
|
4
4
|
import ms from 'ms';
|
|
5
5
|
|
|
6
6
|
import { Level } from 'level';
|
|
7
|
-
import { hashToHex, initDefaultHashes } from '@enbox/dwn-sdk-js';
|
|
7
|
+
import { hashToHex, initDefaultHashes, Message } from '@enbox/dwn-sdk-js';
|
|
8
8
|
|
|
9
9
|
import type { PermissionsApi } from './types/permissions.js';
|
|
10
|
-
import type { SyncEngine, SyncIdentityOptions } from './types/sync.js';
|
|
10
|
+
import type { StartSyncParams, SyncConnectivityState, SyncEngine, SyncIdentityOptions, SyncMode } from './types/sync.js';
|
|
11
11
|
import type { Web5Agent, Web5PlatformAgent } from './types/agent.js';
|
|
12
12
|
|
|
13
13
|
import { AgentPermissionsApi } from './permissions-api.js';
|
|
14
14
|
import { DwnInterface } from './types/dwn.js';
|
|
15
15
|
import { getDwnServiceEndpointUrls } from './utils.js';
|
|
16
|
+
import { isRecordsWrite } from './utils.js';
|
|
16
17
|
import { topologicalSort } from './sync-topological-sort.js';
|
|
17
18
|
import { pullMessages, pushMessages } from './sync-messages.js';
|
|
18
19
|
|
|
@@ -29,6 +30,36 @@ export type SyncEngineLevelParams = {
|
|
|
29
30
|
*/
|
|
30
31
|
const MAX_DIFF_DEPTH = 16;
|
|
31
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Key for the subscription cursor sublevel. Cursors are keyed by
|
|
35
|
+
* `{did}^{dwnUrl}[^{protocol}]` and store an opaque EventLog cursor string.
|
|
36
|
+
*/
|
|
37
|
+
const CURSOR_SEPARATOR = '^';
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Debounce window for push-on-write. When the local EventLog emits events,
|
|
41
|
+
* we batch them and push after this delay to avoid a push per individual write.
|
|
42
|
+
*/
|
|
43
|
+
const PUSH_DEBOUNCE_MS = 250;
|
|
44
|
+
|
|
45
|
+
/** Tracks a live subscription to a remote DWN for one sync target. */
|
|
46
|
+
type LiveSubscription = {
|
|
47
|
+
did: string;
|
|
48
|
+
dwnUrl: string;
|
|
49
|
+
delegateDid?: string;
|
|
50
|
+
protocol?: string;
|
|
51
|
+
close: () => Promise<void>;
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
/** Tracks a local EventLog subscription for push-on-write. */
|
|
55
|
+
type LocalSubscription = {
|
|
56
|
+
did: string;
|
|
57
|
+
dwnUrl: string;
|
|
58
|
+
delegateDid?: string;
|
|
59
|
+
protocol?: string;
|
|
60
|
+
close: () => Promise<void>;
|
|
61
|
+
};
|
|
62
|
+
|
|
32
63
|
export class SyncEngineLevel implements SyncEngine {
|
|
33
64
|
/**
|
|
34
65
|
* Holds the instance of a `Web5PlatformAgent` that represents the current execution context for
|
|
@@ -54,6 +85,37 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
54
85
|
*/
|
|
55
86
|
private _defaultHashHex?: Map<number, string>;
|
|
56
87
|
|
|
88
|
+
// ---------------------------------------------------------------------------
|
|
89
|
+
// Live sync state
|
|
90
|
+
// ---------------------------------------------------------------------------
|
|
91
|
+
|
|
92
|
+
/** Current sync mode, set by `startSync`. */
|
|
93
|
+
private _syncMode: SyncMode = 'poll';
|
|
94
|
+
|
|
95
|
+
/** Active live pull subscriptions (remote -> local via MessagesSubscribe). */
|
|
96
|
+
private _liveSubscriptions: LiveSubscription[] = [];
|
|
97
|
+
|
|
98
|
+
/** Active local EventLog subscriptions for push-on-write (local -> remote). */
|
|
99
|
+
private _localSubscriptions: LocalSubscription[] = [];
|
|
100
|
+
|
|
101
|
+
/** Connectivity state derived from subscription health. */
|
|
102
|
+
private _connectivityState: SyncConnectivityState = 'unknown';
|
|
103
|
+
|
|
104
|
+
/** Debounce timer for batched push-on-write. */
|
|
105
|
+
private _pushDebounceTimer?: ReturnType<typeof setTimeout>;
|
|
106
|
+
|
|
107
|
+
/** Pending message CIDs to push, accumulated during the debounce window. */
|
|
108
|
+
private _pendingPushCids: Map<string, { did: string; dwnUrl: string; delegateDid?: string; protocol?: string; cids: string[] }> = new Map();
|
|
109
|
+
|
|
110
|
+
/** Count of consecutive SMT sync failures (for backoff in poll mode). */
|
|
111
|
+
private _consecutiveFailures = 0;
|
|
112
|
+
|
|
113
|
+
/** Maximum consecutive failures before entering backoff. */
|
|
114
|
+
private static readonly MAX_CONSECUTIVE_FAILURES = 5;
|
|
115
|
+
|
|
116
|
+
/** Backoff multiplier for consecutive failures (caps at 4x the configured interval). */
|
|
117
|
+
private static readonly MAX_BACKOFF_MULTIPLIER = 4;
|
|
118
|
+
|
|
57
119
|
constructor({ agent, dataPath, db }: SyncEngineLevelParams) {
|
|
58
120
|
this._agent = agent;
|
|
59
121
|
this._permissionsApi = new AgentPermissionsApi({ agent: agent as Web5Agent });
|
|
@@ -79,6 +141,10 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
79
141
|
this._permissionsApi = new AgentPermissionsApi({ agent: agent as Web5Agent });
|
|
80
142
|
}
|
|
81
143
|
|
|
144
|
+
get connectivityState(): SyncConnectivityState {
|
|
145
|
+
return this._connectivityState;
|
|
146
|
+
}
|
|
147
|
+
|
|
82
148
|
public async clear(): Promise<void> {
|
|
83
149
|
await this._permissionsApi.clear();
|
|
84
150
|
await this._db.clear();
|
|
@@ -140,6 +206,10 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
140
206
|
await registeredIdentities.put(did, JSON.stringify(options));
|
|
141
207
|
}
|
|
142
208
|
|
|
209
|
+
// ---------------------------------------------------------------------------
|
|
210
|
+
// One-shot sync (SMT set reconciliation)
|
|
211
|
+
// ---------------------------------------------------------------------------
|
|
212
|
+
|
|
143
213
|
public async sync(direction?: 'push' | 'pull'): Promise<void> {
|
|
144
214
|
if (this._syncLock) {
|
|
145
215
|
throw new Error('SyncEngineLevel: Sync operation is already in progress.');
|
|
@@ -150,6 +220,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
150
220
|
// Iterate over all registered identities and their DWN endpoints.
|
|
151
221
|
const syncTargets = await this.getSyncTargets();
|
|
152
222
|
const errored = new Set<string>();
|
|
223
|
+
let hadFailure = false;
|
|
153
224
|
|
|
154
225
|
for (const target of syncTargets) {
|
|
155
226
|
const { did, delegateDid, dwnUrl, protocol } = target;
|
|
@@ -189,19 +260,84 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
189
260
|
} catch (error: any) {
|
|
190
261
|
// Skip this DWN endpoint for remaining targets and log the real cause.
|
|
191
262
|
errored.add(dwnUrl);
|
|
263
|
+
hadFailure = true;
|
|
192
264
|
console.error(`SyncEngineLevel: Error syncing ${did} with ${dwnUrl}`, error);
|
|
193
265
|
}
|
|
194
266
|
}
|
|
267
|
+
|
|
268
|
+
// Track consecutive failures for backoff in poll mode.
|
|
269
|
+
if (hadFailure) {
|
|
270
|
+
this._consecutiveFailures++;
|
|
271
|
+
if (this._connectivityState === 'online') {
|
|
272
|
+
this._connectivityState = 'offline';
|
|
273
|
+
}
|
|
274
|
+
} else {
|
|
275
|
+
this._consecutiveFailures = 0;
|
|
276
|
+
if (syncTargets.length > 0) {
|
|
277
|
+
this._connectivityState = 'online';
|
|
278
|
+
}
|
|
279
|
+
}
|
|
195
280
|
} finally {
|
|
196
281
|
this._syncLock = false;
|
|
197
282
|
}
|
|
198
283
|
}
|
|
199
284
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
285
|
+
// ---------------------------------------------------------------------------
|
|
286
|
+
// startSync / stopSync
|
|
287
|
+
// ---------------------------------------------------------------------------
|
|
288
|
+
|
|
289
|
+
public async startSync(params: StartSyncParams): Promise<void> {
|
|
290
|
+
const mode = params.mode ?? 'poll';
|
|
291
|
+
const intervalStr = params.interval ?? (mode === 'live' ? '5m' : '2m');
|
|
292
|
+
const intervalMilliseconds = ms(intervalStr);
|
|
204
293
|
|
|
294
|
+
// Tear down previous mode if there are active live resources.
|
|
295
|
+
if (this._liveSubscriptions.length > 0 || this._localSubscriptions.length > 0) {
|
|
296
|
+
await this.teardownLiveSync();
|
|
297
|
+
}
|
|
298
|
+
if (this._syncIntervalId) {
|
|
299
|
+
clearInterval(this._syncIntervalId);
|
|
300
|
+
this._syncIntervalId = undefined;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
this._syncMode = mode;
|
|
304
|
+
|
|
305
|
+
if (mode === 'live') {
|
|
306
|
+
await this.startLiveSync(intervalMilliseconds);
|
|
307
|
+
} else {
|
|
308
|
+
await this.startPollSync(intervalMilliseconds);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/**
|
|
313
|
+
* stopSync awaits the completion of the current sync operation before stopping the sync interval
|
|
314
|
+
* and tearing down any live subscriptions.
|
|
315
|
+
*/
|
|
316
|
+
public async stopSync(timeout: number = 2000): Promise<void> {
|
|
317
|
+
let elapsedTimeout = 0;
|
|
318
|
+
|
|
319
|
+
while (this._syncLock) {
|
|
320
|
+
if (elapsedTimeout >= timeout) {
|
|
321
|
+
throw new Error(`SyncEngineLevel: Existing sync operation did not complete within ${timeout} milliseconds.`);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
elapsedTimeout += 100;
|
|
325
|
+
await new Promise((resolve): void => { setTimeout(resolve, timeout < 100 ? timeout : 100); });
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
if (this._syncIntervalId) {
|
|
329
|
+
clearInterval(this._syncIntervalId);
|
|
330
|
+
this._syncIntervalId = undefined;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
await this.teardownLiveSync();
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// ---------------------------------------------------------------------------
|
|
337
|
+
// Poll-mode sync (legacy)
|
|
338
|
+
// ---------------------------------------------------------------------------
|
|
339
|
+
|
|
340
|
+
private async startPollSync(intervalMilliseconds: number): Promise<void> {
|
|
205
341
|
const intervalSync = async (): Promise<void> => {
|
|
206
342
|
if (this._syncLock) {
|
|
207
343
|
return;
|
|
@@ -216,8 +352,17 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
216
352
|
console.error('SyncEngineLevel: Error during sync operation', error);
|
|
217
353
|
}
|
|
218
354
|
|
|
355
|
+
// Apply backoff on consecutive failures.
|
|
356
|
+
const backoffMultiplier = Math.min(
|
|
357
|
+
Math.pow(2, this._consecutiveFailures),
|
|
358
|
+
SyncEngineLevel.MAX_BACKOFF_MULTIPLIER,
|
|
359
|
+
);
|
|
360
|
+
const effectiveInterval = this._consecutiveFailures > 0
|
|
361
|
+
? intervalMilliseconds * backoffMultiplier
|
|
362
|
+
: intervalMilliseconds;
|
|
363
|
+
|
|
219
364
|
if (!this._syncIntervalId) {
|
|
220
|
-
this._syncIntervalId = setInterval(intervalSync,
|
|
365
|
+
this._syncIntervalId = setInterval(intervalSync, effectiveInterval);
|
|
221
366
|
}
|
|
222
367
|
};
|
|
223
368
|
|
|
@@ -233,25 +378,362 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
233
378
|
}
|
|
234
379
|
}
|
|
235
380
|
|
|
381
|
+
// ---------------------------------------------------------------------------
|
|
382
|
+
// Live-mode sync
|
|
383
|
+
// ---------------------------------------------------------------------------
|
|
384
|
+
|
|
236
385
|
/**
|
|
237
|
-
*
|
|
386
|
+
* Starts live sync:
|
|
387
|
+
* 1. Performs an initial SMT reconciliation to catch up.
|
|
388
|
+
* 2. Opens MessagesSubscribe subscriptions to each remote DWN for real-time pull.
|
|
389
|
+
* 3. Subscribes to the local EventLog for push-on-write.
|
|
390
|
+
* 4. Schedules an infrequent SMT integrity check at `interval`.
|
|
238
391
|
*/
|
|
239
|
-
|
|
240
|
-
|
|
392
|
+
private async startLiveSync(intervalMilliseconds: number): Promise<void> {
|
|
393
|
+
// Step 1: Initial SMT catch-up.
|
|
394
|
+
try {
|
|
395
|
+
await this.sync();
|
|
396
|
+
} catch (error) {
|
|
397
|
+
console.error('SyncEngineLevel: Error during initial live-sync catch-up', error);
|
|
398
|
+
}
|
|
241
399
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
400
|
+
// Step 2: Open live subscriptions for each sync target.
|
|
401
|
+
const syncTargets = await this.getSyncTargets();
|
|
402
|
+
for (const target of syncTargets) {
|
|
403
|
+
try {
|
|
404
|
+
await this.openLivePullSubscription(target);
|
|
405
|
+
await this.openLocalPushSubscription(target);
|
|
406
|
+
} catch (error: any) {
|
|
407
|
+
console.error(`SyncEngineLevel: Failed to open live subscription for ${target.did} -> ${target.dwnUrl}`, error);
|
|
245
408
|
}
|
|
409
|
+
}
|
|
246
410
|
|
|
247
|
-
|
|
248
|
-
|
|
411
|
+
// Step 3: Schedule infrequent SMT integrity check.
|
|
412
|
+
const integrityCheck = async (): Promise<void> => {
|
|
413
|
+
if (this._syncLock) {
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
try {
|
|
418
|
+
await this.sync();
|
|
419
|
+
} catch (error) {
|
|
420
|
+
console.error('SyncEngineLevel: Error during SMT integrity check', error);
|
|
421
|
+
}
|
|
422
|
+
};
|
|
423
|
+
|
|
424
|
+
this._syncIntervalId = setInterval(integrityCheck, intervalMilliseconds);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/**
|
|
428
|
+
* Tears down all live subscriptions and push listeners.
|
|
429
|
+
*/
|
|
430
|
+
private async teardownLiveSync(): Promise<void> {
|
|
431
|
+
// Clear the push debounce timer.
|
|
432
|
+
if (this._pushDebounceTimer) {
|
|
433
|
+
clearTimeout(this._pushDebounceTimer);
|
|
434
|
+
this._pushDebounceTimer = undefined;
|
|
249
435
|
}
|
|
250
436
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
437
|
+
// Flush any pending push CIDs.
|
|
438
|
+
this._pendingPushCids.clear();
|
|
439
|
+
|
|
440
|
+
// Close all live pull subscriptions.
|
|
441
|
+
for (const sub of this._liveSubscriptions) {
|
|
442
|
+
try {
|
|
443
|
+
await sub.close();
|
|
444
|
+
} catch {
|
|
445
|
+
// Best-effort cleanup.
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
this._liveSubscriptions = [];
|
|
449
|
+
|
|
450
|
+
// Close all local push subscriptions.
|
|
451
|
+
for (const sub of this._localSubscriptions) {
|
|
452
|
+
try {
|
|
453
|
+
await sub.close();
|
|
454
|
+
} catch {
|
|
455
|
+
// Best-effort cleanup.
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
this._localSubscriptions = [];
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
// ---------------------------------------------------------------------------
|
|
462
|
+
// Live pull: MessagesSubscribe to remote DWN
|
|
463
|
+
// ---------------------------------------------------------------------------
|
|
464
|
+
|
|
465
|
+
/**
|
|
466
|
+
* Opens a MessagesSubscribe WebSocket subscription to a remote DWN.
|
|
467
|
+
* Incoming events are processed locally as they arrive.
|
|
468
|
+
*/
|
|
469
|
+
private async openLivePullSubscription(target: {
|
|
470
|
+
did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
|
|
471
|
+
}): Promise<void> {
|
|
472
|
+
const { did, delegateDid, dwnUrl, protocol } = target;
|
|
473
|
+
|
|
474
|
+
// Resolve the cursor from the last session (if any).
|
|
475
|
+
const cursorKey = this.buildCursorKey(did, dwnUrl, protocol);
|
|
476
|
+
const cursor = await this.getCursor(cursorKey);
|
|
477
|
+
|
|
478
|
+
// Build the MessagesSubscribe filters.
|
|
479
|
+
const filters = protocol ? [{ protocol }] : [];
|
|
480
|
+
|
|
481
|
+
// Look up permission grant for MessagesSubscribe if using a delegate.
|
|
482
|
+
let permissionGrantId: string | undefined;
|
|
483
|
+
if (delegateDid) {
|
|
484
|
+
try {
|
|
485
|
+
const grant = await this._permissionsApi.getPermissionForRequest({
|
|
486
|
+
connectedDid : did,
|
|
487
|
+
messageType : DwnInterface.MessagesSubscribe,
|
|
488
|
+
delegateDid,
|
|
489
|
+
protocol,
|
|
490
|
+
cached : true
|
|
491
|
+
});
|
|
492
|
+
permissionGrantId = grant.grant.id;
|
|
493
|
+
} catch {
|
|
494
|
+
// Fall back to trying MessagesRead which is a unified scope.
|
|
495
|
+
try {
|
|
496
|
+
const grant = await this._permissionsApi.getPermissionForRequest({
|
|
497
|
+
connectedDid : did,
|
|
498
|
+
messageType : DwnInterface.MessagesRead,
|
|
499
|
+
delegateDid,
|
|
500
|
+
protocol,
|
|
501
|
+
cached : true
|
|
502
|
+
});
|
|
503
|
+
permissionGrantId = grant.grant.id;
|
|
504
|
+
} catch (error: any) {
|
|
505
|
+
console.error('SyncEngineLevel: Could not find permission grant for live pull subscription', error);
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
// Define the subscription handler that processes incoming events.
|
|
512
|
+
const subscriptionHandler = async (subMessage: SubscriptionMessage): Promise<void> => {
|
|
513
|
+
if (subMessage.type === 'eose') {
|
|
514
|
+
// End-of-stored-events — catch-up complete, persist cursor.
|
|
515
|
+
await this.setCursor(cursorKey, subMessage.cursor);
|
|
516
|
+
this._connectivityState = 'online';
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
if (subMessage.type === 'event') {
|
|
521
|
+
const event: MessageEvent = subMessage.event;
|
|
522
|
+
try {
|
|
523
|
+
// Process the message locally.
|
|
524
|
+
const dataStream = this.extractDataStream(event);
|
|
525
|
+
await this.agent.dwn.node.processMessage(did, event.message, { dataStream });
|
|
526
|
+
} catch (error: any) {
|
|
527
|
+
console.error(`SyncEngineLevel: Error processing live-pull event for ${did}`, error);
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
// Persist cursor for resume on reconnect.
|
|
531
|
+
await this.setCursor(cursorKey, subMessage.cursor);
|
|
532
|
+
}
|
|
533
|
+
};
|
|
534
|
+
|
|
535
|
+
// Send the subscription request through the agent's DWN API.
|
|
536
|
+
const response = await this.agent.dwn.sendRequest({
|
|
537
|
+
author : did,
|
|
538
|
+
target : did,
|
|
539
|
+
messageType : DwnInterface.MessagesSubscribe,
|
|
540
|
+
granteeDid : delegateDid,
|
|
541
|
+
messageParams : {
|
|
542
|
+
filters,
|
|
543
|
+
cursor,
|
|
544
|
+
permissionGrantId,
|
|
545
|
+
},
|
|
546
|
+
subscriptionHandler: subscriptionHandler as any,
|
|
547
|
+
});
|
|
548
|
+
|
|
549
|
+
const reply = response.reply as MessagesSubscribeReply;
|
|
550
|
+
if (reply.status.code !== 200 || !reply.subscription) {
|
|
551
|
+
console.error(`SyncEngineLevel: MessagesSubscribe failed for ${did} -> ${dwnUrl}: ${reply.status.code} ${reply.status.detail}`);
|
|
552
|
+
return;
|
|
254
553
|
}
|
|
554
|
+
|
|
555
|
+
this._liveSubscriptions.push({
|
|
556
|
+
did,
|
|
557
|
+
dwnUrl,
|
|
558
|
+
delegateDid,
|
|
559
|
+
protocol,
|
|
560
|
+
close: async (): Promise<void> => { await reply.subscription!.close(); },
|
|
561
|
+
});
|
|
562
|
+
|
|
563
|
+
this._connectivityState = 'online';
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
// ---------------------------------------------------------------------------
|
|
567
|
+
// Live push: local EventLog subscription for immediate push
|
|
568
|
+
// ---------------------------------------------------------------------------
|
|
569
|
+
|
|
570
|
+
/**
|
|
571
|
+
* Subscribes to the local DWN's EventLog so that writes by the user are
|
|
572
|
+
* immediately pushed to the remote DWN instead of waiting for the next poll.
|
|
573
|
+
*/
|
|
574
|
+
private async openLocalPushSubscription(target: {
|
|
575
|
+
did: string; dwnUrl: string; delegateDid?: string; protocol?: string;
|
|
576
|
+
}): Promise<void> {
|
|
577
|
+
const { did, delegateDid, dwnUrl, protocol } = target;
|
|
578
|
+
|
|
579
|
+
// Build filters scoped to the protocol (if any).
|
|
580
|
+
const filters = protocol ? [{ protocol }] : [];
|
|
581
|
+
|
|
582
|
+
// Look up permission grant for local subscription.
|
|
583
|
+
let permissionGrantId: string | undefined;
|
|
584
|
+
if (delegateDid) {
|
|
585
|
+
try {
|
|
586
|
+
const grant = await this._permissionsApi.getPermissionForRequest({
|
|
587
|
+
connectedDid : did,
|
|
588
|
+
messageType : DwnInterface.MessagesRead,
|
|
589
|
+
delegateDid,
|
|
590
|
+
protocol,
|
|
591
|
+
cached : true,
|
|
592
|
+
});
|
|
593
|
+
permissionGrantId = grant.grant.id;
|
|
594
|
+
} catch {
|
|
595
|
+
// No grant available — skip push-on-write for this target.
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
// Subscribe to the local DWN's EventLog.
|
|
601
|
+
const subscriptionHandler = (subMessage: SubscriptionMessage): void => {
|
|
602
|
+
if (subMessage.type !== 'event') {
|
|
603
|
+
return;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
// Accumulate the message CID for a debounced push.
|
|
607
|
+
const targetKey = this.buildCursorKey(did, dwnUrl, protocol);
|
|
608
|
+
const cid = this.tryGetCidSync(subMessage.event.message);
|
|
609
|
+
if (cid === undefined) {
|
|
610
|
+
return;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
let pending = this._pendingPushCids.get(targetKey);
|
|
614
|
+
if (!pending) {
|
|
615
|
+
pending = { did, dwnUrl, delegateDid, protocol, cids: [] };
|
|
616
|
+
this._pendingPushCids.set(targetKey, pending);
|
|
617
|
+
}
|
|
618
|
+
pending.cids.push(cid);
|
|
619
|
+
|
|
620
|
+
// Debounce the push.
|
|
621
|
+
if (this._pushDebounceTimer) {
|
|
622
|
+
clearTimeout(this._pushDebounceTimer);
|
|
623
|
+
}
|
|
624
|
+
this._pushDebounceTimer = setTimeout((): void => {
|
|
625
|
+
void this.flushPendingPushes();
|
|
626
|
+
}, PUSH_DEBOUNCE_MS);
|
|
627
|
+
};
|
|
628
|
+
|
|
629
|
+
// Process the local subscription request.
|
|
630
|
+
const response = await this.agent.dwn.processRequest({
|
|
631
|
+
author : did,
|
|
632
|
+
target : did,
|
|
633
|
+
messageType : DwnInterface.MessagesSubscribe,
|
|
634
|
+
granteeDid : delegateDid,
|
|
635
|
+
messageParams : { filters, permissionGrantId },
|
|
636
|
+
subscriptionHandler : subscriptionHandler as any,
|
|
637
|
+
});
|
|
638
|
+
|
|
639
|
+
const reply = response.reply as MessagesSubscribeReply;
|
|
640
|
+
if (reply.status.code !== 200 || !reply.subscription) {
|
|
641
|
+
console.error(`SyncEngineLevel: Local MessagesSubscribe failed for ${did}: ${reply.status.code} ${reply.status.detail}`);
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
this._localSubscriptions.push({
|
|
646
|
+
did,
|
|
647
|
+
dwnUrl,
|
|
648
|
+
delegateDid,
|
|
649
|
+
protocol,
|
|
650
|
+
close: async (): Promise<void> => { await reply.subscription!.close(); },
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
/**
|
|
655
|
+
* Flushes accumulated push CIDs to remote DWNs.
|
|
656
|
+
*/
|
|
657
|
+
private async flushPendingPushes(): Promise<void> {
|
|
658
|
+
this._pushDebounceTimer = undefined;
|
|
659
|
+
|
|
660
|
+
const entries = [...this._pendingPushCids.entries()];
|
|
661
|
+
this._pendingPushCids.clear();
|
|
662
|
+
|
|
663
|
+
for (const [, pending] of entries) {
|
|
664
|
+
const { did, dwnUrl, delegateDid, protocol, cids } = pending;
|
|
665
|
+
if (cids.length === 0) {
|
|
666
|
+
continue;
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
try {
|
|
670
|
+
await pushMessages({
|
|
671
|
+
did, dwnUrl, delegateDid, protocol,
|
|
672
|
+
messageCids : cids,
|
|
673
|
+
agent : this.agent,
|
|
674
|
+
permissionsApi : this._permissionsApi,
|
|
675
|
+
});
|
|
676
|
+
} catch (error: any) {
|
|
677
|
+
console.error(`SyncEngineLevel: Push-on-write failed for ${did} -> ${dwnUrl}`, error);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// ---------------------------------------------------------------------------
|
|
683
|
+
// Cursor persistence
|
|
684
|
+
// ---------------------------------------------------------------------------
|
|
685
|
+
|
|
686
|
+
private buildCursorKey(did: string, dwnUrl: string, protocol?: string): string {
|
|
687
|
+
const base = `${did}${CURSOR_SEPARATOR}${dwnUrl}`;
|
|
688
|
+
return protocol ? `${base}${CURSOR_SEPARATOR}${protocol}` : base;
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
private async getCursor(key: string): Promise<string | undefined> {
|
|
692
|
+
const cursors = this._db.sublevel('syncCursors');
|
|
693
|
+
try {
|
|
694
|
+
return await cursors.get(key);
|
|
695
|
+
} catch (error) {
|
|
696
|
+
const e = error as { code: string };
|
|
697
|
+
if (e.code === 'LEVEL_NOT_FOUND') {
|
|
698
|
+
return undefined;
|
|
699
|
+
}
|
|
700
|
+
throw error;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
private async setCursor(key: string, cursor: string): Promise<void> {
|
|
705
|
+
const cursors = this._db.sublevel('syncCursors');
|
|
706
|
+
await cursors.put(key, cursor);
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
// ---------------------------------------------------------------------------
|
|
710
|
+
// Utility helpers
|
|
711
|
+
// ---------------------------------------------------------------------------
|
|
712
|
+
|
|
713
|
+
/**
|
|
714
|
+
* Extracts a ReadableStream from a MessageEvent if it contains a RecordsWrite with data.
|
|
715
|
+
*/
|
|
716
|
+
private extractDataStream(event: MessageEvent): ReadableStream<Uint8Array> | undefined {
|
|
717
|
+
if (isRecordsWrite(event) && (event as any).data) {
|
|
718
|
+
return (event as any).data;
|
|
719
|
+
}
|
|
720
|
+
return undefined;
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/**
|
|
724
|
+
* Synchronously attempts to get a message CID. Returns undefined on failure.
|
|
725
|
+
* This is used in the synchronous EventLog callback; the actual CID computation
|
|
726
|
+
* is fast for already-constructed messages.
|
|
727
|
+
*/
|
|
728
|
+
private tryGetCidSync(message: GenericMessage): string | undefined {
|
|
729
|
+
// Message.getCid is async but very fast (SHA-256 of the descriptor).
|
|
730
|
+
// We fire-and-forget into a microtask and store the result.
|
|
731
|
+
// For the debounced push, the CID will be resolved by the time we flush.
|
|
732
|
+
let cid: string | undefined;
|
|
733
|
+
void Message.getCid(message).then((result): void => { cid = result; });
|
|
734
|
+
// Since this is a microtask, it may not resolve immediately.
|
|
735
|
+
// Use the descriptor's CID field if available as a synchronous fallback.
|
|
736
|
+
return cid ?? (message as any).messageCid ?? undefined;
|
|
255
737
|
}
|
|
256
738
|
|
|
257
739
|
// ---------------------------------------------------------------------------
|
|
@@ -507,10 +989,6 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
507
989
|
return reply.entries ?? [];
|
|
508
990
|
}
|
|
509
991
|
|
|
510
|
-
// ---------------------------------------------------------------------------
|
|
511
|
-
// Pull — fetch messages from remote, process locally in dependency order
|
|
512
|
-
// ---------------------------------------------------------------------------
|
|
513
|
-
|
|
514
992
|
// ---------------------------------------------------------------------------
|
|
515
993
|
// Pull / Push — delegates to standalone functions in sync-messages.ts
|
|
516
994
|
// ---------------------------------------------------------------------------
|
|
@@ -559,7 +1037,7 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
559
1037
|
* Delegate to the standalone `topologicalSort` function.
|
|
560
1038
|
* Tests call `SyncEngineLevel.topologicalSort(...)` so this static method must remain.
|
|
561
1039
|
*/
|
|
562
|
-
static topologicalSort<T extends { message: GenericMessage }>(
|
|
1040
|
+
public static topologicalSort<T extends { message: GenericMessage }>(
|
|
563
1041
|
messages: T[]
|
|
564
1042
|
): T[] {
|
|
565
1043
|
return topologicalSort(messages);
|
|
@@ -577,14 +1055,13 @@ export class SyncEngineLevel implements SyncEngine {
|
|
|
577
1055
|
const targets: { did: string; dwnUrl: string; delegateDid?: string; protocol?: string }[] = [];
|
|
578
1056
|
|
|
579
1057
|
for await (const [did, options] of this._db.sublevel('registeredIdentities').iterator()) {
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
});
|
|
1058
|
+
let parsed: SyncIdentityOptions;
|
|
1059
|
+
try {
|
|
1060
|
+
parsed = JSON.parse(options) as SyncIdentityOptions;
|
|
1061
|
+
} catch {
|
|
1062
|
+
parsed = { protocols: [] };
|
|
1063
|
+
}
|
|
1064
|
+
const { protocols, delegateDid } = parsed;
|
|
588
1065
|
|
|
589
1066
|
const dwnEndpointUrls = await getDwnServiceEndpointUrls(did, this.agent.did);
|
|
590
1067
|
if (dwnEndpointUrls.length === 0) {
|
package/src/test-harness.ts
CHANGED
|
@@ -6,7 +6,7 @@ import type { KeyValueStore } from '@enbox/common';
|
|
|
6
6
|
import type { Web5PlatformAgent } from './types/agent.js';
|
|
7
7
|
|
|
8
8
|
import { Level } from 'level';
|
|
9
|
-
import { DataStoreLevel,
|
|
9
|
+
import { DataStoreLevel, EventEmitterEventLog, MessageStoreLevel, ResumableTaskStoreLevel, StateIndexLevel } from '@enbox/dwn-sdk-js';
|
|
10
10
|
import { DidDht, DidJwk, DidResolverCacheMemory } from '@enbox/dids';
|
|
11
11
|
import { LevelStore, MemoryStore } from '@enbox/common';
|
|
12
12
|
|
|
@@ -193,8 +193,6 @@ export class PlatformAgentTestHarness {
|
|
|
193
193
|
id : 'dwn',
|
|
194
194
|
type : 'DecentralizedWebNode',
|
|
195
195
|
serviceEndpoint : testDwnUrls,
|
|
196
|
-
enc : '#enc',
|
|
197
|
-
sig : '#sig',
|
|
198
196
|
}
|
|
199
197
|
],
|
|
200
198
|
verificationMethods: [
|
|
@@ -259,7 +257,7 @@ export class PlatformAgentTestHarness {
|
|
|
259
257
|
// Note: There is no in-memory store for DWN, so we always use LevelDB-based disk stores.
|
|
260
258
|
const dwnDataStore = new DataStoreLevel({ blockstoreLocation: testDataPath('DWN_DATASTORE') });
|
|
261
259
|
const dwnStateIndex = new StateIndexLevel({ location: testDataPath('DWN_STATEINDEX') });
|
|
262
|
-
const
|
|
260
|
+
const dwnEventLog = new EventEmitterEventLog();
|
|
263
261
|
const dwnResumableTaskStore = new ResumableTaskStoreLevel({ location: testDataPath('DWN_RESUMABLETASKSTORE') });
|
|
264
262
|
|
|
265
263
|
const dwnMessageStore = new MessageStoreLevel({
|
|
@@ -273,7 +271,7 @@ export class PlatformAgentTestHarness {
|
|
|
273
271
|
dataStore : dwnDataStore,
|
|
274
272
|
didResolver : didApi,
|
|
275
273
|
stateIndex : dwnStateIndex,
|
|
276
|
-
|
|
274
|
+
eventLog : dwnEventLog,
|
|
277
275
|
messageStore : dwnMessageStore,
|
|
278
276
|
resumableTaskStore : dwnResumableTaskStore
|
|
279
277
|
});
|