@dxos/client-services 0.6.13 → 0.6.14-main.7bd9c89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-CRXXOI45.mjs → chunk-URZYQU7T.mjs} +6383 -5185
- package/dist/lib/browser/chunk-URZYQU7T.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +7 -3
- package/dist/lib/browser/index.mjs.map +3 -3
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +12 -7
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node/{chunk-PZ3JJJ3K.cjs → chunk-APHU7U5B.cjs} +6294 -5106
- package/dist/lib/node/chunk-APHU7U5B.cjs.map +7 -0
- package/dist/lib/node/index.cjs +50 -46
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +18 -13
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/lib/node-esm/chunk-L5NEKNEY.mjs +8914 -0
- package/dist/lib/node-esm/chunk-L5NEKNEY.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +420 -0
- package/dist/lib/node-esm/index.mjs.map +7 -0
- package/dist/lib/node-esm/meta.json +1 -0
- package/dist/lib/node-esm/testing/index.mjs +424 -0
- package/dist/lib/node-esm/testing/index.mjs.map +7 -0
- package/dist/types/src/index.d.ts +1 -0
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/packlets/agents/edge-agent-manager.d.ts +35 -0
- package/dist/types/src/packlets/agents/edge-agent-manager.d.ts.map +1 -0
- package/dist/types/src/packlets/agents/edge-agent-service.d.ts +10 -0
- package/dist/types/src/packlets/agents/edge-agent-service.d.ts.map +1 -0
- package/dist/types/src/packlets/agents/index.d.ts +3 -0
- package/dist/types/src/packlets/agents/index.d.ts.map +1 -0
- package/dist/types/src/packlets/diagnostics/diagnostics-broadcast.d.ts.map +1 -1
- package/dist/types/src/packlets/identity/authenticator.d.ts.map +1 -1
- package/dist/types/src/packlets/identity/authenticator.node.test.d.ts +2 -0
- package/dist/types/src/packlets/identity/authenticator.node.test.d.ts.map +1 -0
- package/dist/types/src/packlets/identity/contacts-service.d.ts +1 -1
- package/dist/types/src/packlets/identity/contacts-service.d.ts.map +1 -1
- package/dist/types/src/packlets/identity/identity-manager.d.ts +28 -9
- package/dist/types/src/packlets/identity/identity-manager.d.ts.map +1 -1
- package/dist/types/src/packlets/identity/identity-recovery-manager.d.ts +18 -0
- package/dist/types/src/packlets/identity/identity-recovery-manager.d.ts.map +1 -0
- package/dist/types/src/packlets/identity/identity-service.d.ts +7 -2
- package/dist/types/src/packlets/identity/identity-service.d.ts.map +1 -1
- package/dist/types/src/packlets/identity/identity.d.ts +12 -3
- package/dist/types/src/packlets/identity/identity.d.ts.map +1 -1
- package/dist/types/src/packlets/invitations/device-invitation-protocol.d.ts.map +1 -1
- package/dist/types/src/packlets/invitations/edge-invitation-handler.d.ts +30 -0
- package/dist/types/src/packlets/invitations/edge-invitation-handler.d.ts.map +1 -0
- package/dist/types/src/packlets/invitations/invitation-guest-extenstion.d.ts +2 -1
- package/dist/types/src/packlets/invitations/invitation-guest-extenstion.d.ts.map +1 -1
- package/dist/types/src/packlets/invitations/invitation-host-extension.d.ts +2 -1
- package/dist/types/src/packlets/invitations/invitation-host-extension.d.ts.map +1 -1
- package/dist/types/src/packlets/invitations/invitation-state.d.ts +19 -0
- package/dist/types/src/packlets/invitations/invitation-state.d.ts.map +1 -0
- package/dist/types/src/packlets/invitations/invitations-handler.d.ts +8 -8
- package/dist/types/src/packlets/invitations/invitations-handler.d.ts.map +1 -1
- package/dist/types/src/packlets/invitations/space-invitation-protocol.d.ts.map +1 -1
- package/dist/types/src/packlets/services/service-context.d.ts +14 -9
- package/dist/types/src/packlets/services/service-context.d.ts.map +1 -1
- package/dist/types/src/packlets/services/service-host.d.ts +2 -0
- package/dist/types/src/packlets/services/service-host.d.ts.map +1 -1
- package/dist/types/src/packlets/spaces/data-space-manager.d.ts +7 -3
- package/dist/types/src/packlets/spaces/data-space-manager.d.ts.map +1 -1
- package/dist/types/src/packlets/spaces/data-space.d.ts +5 -3
- package/dist/types/src/packlets/spaces/data-space.d.ts.map +1 -1
- package/dist/types/src/packlets/spaces/edge-feed-replicator.d.ts +3 -0
- package/dist/types/src/packlets/spaces/edge-feed-replicator.d.ts.map +1 -1
- package/dist/types/src/packlets/spaces/edge-feed-replicator.test.d.ts +2 -0
- package/dist/types/src/packlets/spaces/edge-feed-replicator.test.d.ts.map +1 -0
- package/dist/types/src/packlets/spaces/epoch-migrations.d.ts +1 -1
- package/dist/types/src/packlets/spaces/epoch-migrations.d.ts.map +1 -1
- package/dist/types/src/packlets/spaces/notarization-plugin.d.ts +35 -6
- package/dist/types/src/packlets/spaces/notarization-plugin.d.ts.map +1 -1
- package/dist/types/src/packlets/spaces/spaces-service.d.ts +1 -1
- package/dist/types/src/packlets/spaces/spaces-service.d.ts.map +1 -1
- package/dist/types/src/packlets/storage/storage.d.ts.map +1 -1
- package/dist/types/src/packlets/testing/test-builder.d.ts +1 -2
- package/dist/types/src/packlets/testing/test-builder.d.ts.map +1 -1
- package/dist/types/src/packlets/worker/worker-runtime.d.ts.map +1 -1
- package/dist/types/src/testing/setup.d.ts +3 -0
- package/dist/types/src/testing/setup.d.ts.map +1 -0
- package/dist/types/src/version.d.ts +1 -1
- package/dist/types/src/version.d.ts.map +1 -1
- package/package.json +43 -39
- package/src/index.ts +1 -0
- package/src/packlets/agents/edge-agent-manager.ts +163 -0
- package/src/packlets/agents/edge-agent-service.ts +42 -0
- package/src/packlets/agents/index.ts +6 -0
- package/src/packlets/devices/devices-service.test.ts +4 -5
- package/src/packlets/diagnostics/diagnostics-broadcast.ts +1 -0
- package/src/packlets/identity/{authenticator.test.ts → authenticator.node.test.ts} +2 -3
- package/src/packlets/identity/authenticator.ts +5 -2
- package/src/packlets/identity/contacts-service.ts +1 -1
- package/src/packlets/identity/identity-manager.test.ts +31 -16
- package/src/packlets/identity/identity-manager.ts +76 -32
- package/src/packlets/identity/identity-recovery-manager.ts +95 -0
- package/src/packlets/identity/identity-service.test.ts +5 -8
- package/src/packlets/identity/identity-service.ts +11 -5
- package/src/packlets/identity/identity.test.ts +130 -239
- package/src/packlets/identity/identity.ts +60 -17
- package/src/packlets/invitations/device-invitation-protocol.test.ts +7 -4
- package/src/packlets/invitations/device-invitation-protocol.ts +5 -1
- package/src/packlets/invitations/edge-invitation-handler.ts +185 -0
- package/src/packlets/invitations/invitation-guest-extenstion.ts +8 -4
- package/src/packlets/invitations/invitation-host-extension.ts +8 -7
- package/src/packlets/invitations/invitation-state.ts +111 -0
- package/src/packlets/invitations/invitations-handler.test.ts +16 -9
- package/src/packlets/invitations/invitations-handler.ts +23 -93
- package/src/packlets/invitations/space-invitation-protocol.test.ts +4 -3
- package/src/packlets/invitations/space-invitation-protocol.ts +4 -0
- package/src/packlets/logging/logging.test.ts +1 -2
- package/src/packlets/network/network-service.test.ts +2 -3
- package/src/packlets/services/service-context.test.ts +3 -1
- package/src/packlets/services/service-context.ts +110 -35
- package/src/packlets/services/service-host.test.ts +8 -12
- package/src/packlets/services/service-host.ts +25 -7
- package/src/packlets/services/service-registry.test.ts +1 -2
- package/src/packlets/spaces/data-space-manager.test.ts +2 -2
- package/src/packlets/spaces/data-space-manager.ts +44 -7
- package/src/packlets/spaces/data-space.ts +37 -6
- package/src/packlets/spaces/edge-feed-replicator.test.ts +252 -0
- package/src/packlets/spaces/edge-feed-replicator.ts +80 -22
- package/src/packlets/spaces/epoch-migrations.ts +2 -2
- package/src/packlets/spaces/notarization-plugin.test.ts +10 -7
- package/src/packlets/spaces/notarization-plugin.ts +196 -29
- package/src/packlets/spaces/spaces-service.test.ts +5 -9
- package/src/packlets/spaces/spaces-service.ts +6 -1
- package/src/packlets/storage/storage.ts +0 -1
- package/src/packlets/system/system-service.test.ts +1 -2
- package/src/packlets/testing/test-builder.ts +7 -4
- package/src/packlets/worker/worker-runtime.ts +2 -2
- package/src/testing/setup.ts +11 -0
- package/src/version.ts +1 -5
- package/dist/lib/browser/chunk-CRXXOI45.mjs.map +0 -7
- package/dist/lib/node/chunk-PZ3JJJ3K.cjs.map +0 -7
- package/dist/types/src/packlets/identity/authenticator.test.d.ts +0 -2
- package/dist/types/src/packlets/identity/authenticator.test.d.ts.map +0 -1
- package/dist/types/src/packlets/services/automerge-host.test.d.ts +0 -2
- package/dist/types/src/packlets/services/automerge-host.test.d.ts.map +0 -1
- package/src/packlets/services/automerge-host.test.ts +0 -60
|
@@ -5,15 +5,19 @@
|
|
|
5
5
|
import { decode as decodeCbor, encode as encodeCbor } from 'cbor-x';
|
|
6
6
|
|
|
7
7
|
import { Event, Mutex, scheduleMicroTask } from '@dxos/async';
|
|
8
|
-
import {
|
|
8
|
+
import { Context, Resource } from '@dxos/context';
|
|
9
9
|
import { type EdgeConnection } from '@dxos/edge-client';
|
|
10
|
+
import { EdgeConnectionClosedError, EdgeIdentityChangedError } from '@dxos/edge-client';
|
|
10
11
|
import { type FeedWrapper } from '@dxos/feed-store';
|
|
11
12
|
import { invariant } from '@dxos/invariant';
|
|
12
13
|
import { PublicKey, type SpaceId } from '@dxos/keys';
|
|
13
|
-
import { log } from '@dxos/log';
|
|
14
|
+
import { log, logInfo } from '@dxos/log';
|
|
14
15
|
import { EdgeService } from '@dxos/protocols';
|
|
15
16
|
import { buf } from '@dxos/protocols/buf';
|
|
16
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
MessageSchema as RouterMessageSchema,
|
|
19
|
+
type Message as RouterMessage,
|
|
20
|
+
} from '@dxos/protocols/buf/dxos/edge/messenger_pb';
|
|
17
21
|
import type { FeedBlock, ProtocolMessage } from '@dxos/protocols/feed-replication';
|
|
18
22
|
import { ComplexMap, arrayToBuffer, bufferToArray, defaultMap, rangeFromTo } from '@dxos/util';
|
|
19
23
|
|
|
@@ -24,7 +28,10 @@ export type EdgeFeedReplicatorParams = {
|
|
|
24
28
|
|
|
25
29
|
export class EdgeFeedReplicator extends Resource {
|
|
26
30
|
private readonly _messenger: EdgeConnection;
|
|
31
|
+
|
|
32
|
+
@logInfo
|
|
27
33
|
private readonly _spaceId: SpaceId;
|
|
34
|
+
|
|
28
35
|
private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<any>>(PublicKey.hash);
|
|
29
36
|
|
|
30
37
|
private _connectionCtx?: Context = undefined;
|
|
@@ -46,9 +53,10 @@ export class EdgeFeedReplicator extends Resource {
|
|
|
46
53
|
}
|
|
47
54
|
|
|
48
55
|
protected override async _open(): Promise<void> {
|
|
56
|
+
log('open');
|
|
49
57
|
// TODO: handle reconnects
|
|
50
58
|
this._ctx.onDispose(
|
|
51
|
-
this._messenger.addListener(
|
|
59
|
+
this._messenger.addListener((message: RouterMessage) => {
|
|
52
60
|
if (!message.serviceId) {
|
|
53
61
|
return;
|
|
54
62
|
}
|
|
@@ -64,21 +72,40 @@ export class EdgeFeedReplicator extends Resource {
|
|
|
64
72
|
}
|
|
65
73
|
|
|
66
74
|
const payload = decodeCbor(message.payload!.value) as ProtocolMessage;
|
|
67
|
-
log
|
|
75
|
+
log('receive', { from: message.source, feedKey: payload.feedKey, type: payload.type });
|
|
68
76
|
this._onMessage(payload);
|
|
69
77
|
}),
|
|
70
78
|
);
|
|
71
79
|
|
|
72
|
-
this.
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
80
|
+
this._messenger.connected.on(this._ctx, async () => {
|
|
81
|
+
await this._resetConnection();
|
|
82
|
+
this._startReplication();
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
if (this._messenger.isConnected) {
|
|
86
|
+
this._startReplication();
|
|
76
87
|
}
|
|
77
88
|
}
|
|
78
89
|
|
|
79
90
|
protected override async _close(): Promise<void> {
|
|
80
|
-
|
|
91
|
+
log('close');
|
|
92
|
+
await this._resetConnection();
|
|
93
|
+
}
|
|
81
94
|
|
|
95
|
+
private _startReplication() {
|
|
96
|
+
this._connected = true;
|
|
97
|
+
const connectionCtx = this._createConnectionContext();
|
|
98
|
+
this._connectionCtx = connectionCtx;
|
|
99
|
+
log('connection context created');
|
|
100
|
+
scheduleMicroTask(connectionCtx, async () => {
|
|
101
|
+
for (const feed of this._feeds.values()) {
|
|
102
|
+
await this._replicateFeed(connectionCtx, feed);
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private async _resetConnection() {
|
|
108
|
+
log('resetConnection');
|
|
82
109
|
this._connected = false;
|
|
83
110
|
await this._connectionCtx?.dispose();
|
|
84
111
|
this._connectionCtx = undefined;
|
|
@@ -86,11 +113,11 @@ export class EdgeFeedReplicator extends Resource {
|
|
|
86
113
|
}
|
|
87
114
|
|
|
88
115
|
async addFeed(feed: FeedWrapper<any>) {
|
|
89
|
-
log.info('addFeed', { key: feed.key });
|
|
116
|
+
log.info('addFeed', { key: feed.key, connected: this._connected, hasConnectionCtx: !!this._connectionCtx });
|
|
90
117
|
this._feeds.set(feed.key, feed);
|
|
91
118
|
|
|
92
|
-
if (this._connected) {
|
|
93
|
-
await this._replicateFeed(feed);
|
|
119
|
+
if (this._connected && this._connectionCtx) {
|
|
120
|
+
await this._replicateFeed(this._connectionCtx, feed);
|
|
94
121
|
}
|
|
95
122
|
}
|
|
96
123
|
|
|
@@ -98,25 +125,32 @@ export class EdgeFeedReplicator extends Resource {
|
|
|
98
125
|
return defaultMap(this._pushMutex, key, () => new Mutex());
|
|
99
126
|
}
|
|
100
127
|
|
|
101
|
-
private async _replicateFeed(feed: FeedWrapper<any>) {
|
|
102
|
-
|
|
103
|
-
|
|
128
|
+
private async _replicateFeed(ctx: Context, feed: FeedWrapper<any>) {
|
|
129
|
+
log('replicateFeed', { key: feed.key });
|
|
104
130
|
await this._sendMessage({
|
|
105
131
|
type: 'get-metadata',
|
|
106
132
|
feedKey: feed.key.toHex(),
|
|
107
133
|
});
|
|
108
134
|
|
|
109
|
-
Event.wrap(feed.core as any, 'append').on(
|
|
135
|
+
Event.wrap(feed.core as any, 'append').on(ctx, async () => {
|
|
110
136
|
await this._pushBlocksIfNeeded(feed);
|
|
111
137
|
});
|
|
112
138
|
}
|
|
113
139
|
|
|
114
140
|
private async _sendMessage(message: ProtocolMessage) {
|
|
115
|
-
|
|
141
|
+
if (!this._connectionCtx) {
|
|
142
|
+
log.info('message dropped because connection was disposed');
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const logPayload =
|
|
147
|
+
message.type === 'data' ? { feedKey: message.feedKey, blocks: message.blocks.map((b) => b.index) } : { message };
|
|
148
|
+
log.info('sending message', logPayload);
|
|
116
149
|
|
|
117
150
|
invariant(message.feedKey);
|
|
118
151
|
const payloadValue = bufferToArray(encodeCbor(message));
|
|
119
152
|
|
|
153
|
+
log('send', { type: message.type });
|
|
120
154
|
await this._messenger.send(
|
|
121
155
|
buf.create(RouterMessageSchema, {
|
|
122
156
|
source: {
|
|
@@ -130,11 +164,15 @@ export class EdgeFeedReplicator extends Resource {
|
|
|
130
164
|
}
|
|
131
165
|
|
|
132
166
|
private _onMessage(message: ProtocolMessage) {
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
167
|
+
if (!this._connectionCtx) {
|
|
168
|
+
log.warn('received message after connection context was disposed');
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
scheduleMicroTask(this._connectionCtx, async () => {
|
|
136
172
|
switch (message.type) {
|
|
137
173
|
case 'metadata': {
|
|
174
|
+
log.info('received metadata', { message });
|
|
175
|
+
|
|
138
176
|
const feedKey = PublicKey.fromHex(message.feedKey);
|
|
139
177
|
const feed = this._feeds.get(feedKey);
|
|
140
178
|
if (!feed) {
|
|
@@ -160,6 +198,8 @@ export class EdgeFeedReplicator extends Resource {
|
|
|
160
198
|
}
|
|
161
199
|
|
|
162
200
|
case 'data': {
|
|
201
|
+
log.info('received data', { feed: message.feedKey, blocks: message.blocks.map((b) => b.index) });
|
|
202
|
+
|
|
163
203
|
const feedKey = PublicKey.fromHex(message.feedKey);
|
|
164
204
|
const feed = this._feeds.get(feedKey);
|
|
165
205
|
if (!feed) {
|
|
@@ -223,9 +263,10 @@ export class EdgeFeedReplicator extends Resource {
|
|
|
223
263
|
}
|
|
224
264
|
|
|
225
265
|
private async _pushBlocksIfNeeded(feed: FeedWrapper<any>) {
|
|
226
|
-
using
|
|
266
|
+
using _ = await this._getPushMutex(feed.key).acquire();
|
|
227
267
|
|
|
228
268
|
if (!this._remoteLength.has(feed.key)) {
|
|
269
|
+
log('blocks not pushed because remote length is unknown');
|
|
229
270
|
return;
|
|
230
271
|
}
|
|
231
272
|
|
|
@@ -234,6 +275,23 @@ export class EdgeFeedReplicator extends Resource {
|
|
|
234
275
|
await this._pushBlocks(feed, remoteLength, feed.length);
|
|
235
276
|
}
|
|
236
277
|
}
|
|
278
|
+
|
|
279
|
+
private _createConnectionContext() {
|
|
280
|
+
const connectionCtx = new Context({
|
|
281
|
+
onError: async (err: any) => {
|
|
282
|
+
if (connectionCtx !== this._connectionCtx) {
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
if (err instanceof EdgeIdentityChangedError || err instanceof EdgeConnectionClosedError) {
|
|
286
|
+
log('resetting on reconnect');
|
|
287
|
+
await this._resetConnection();
|
|
288
|
+
} else {
|
|
289
|
+
this._ctx.raise(err);
|
|
290
|
+
}
|
|
291
|
+
},
|
|
292
|
+
});
|
|
293
|
+
return connectionCtx;
|
|
294
|
+
}
|
|
237
295
|
}
|
|
238
296
|
|
|
239
297
|
// hypercore requires buffers
|
|
@@ -4,13 +4,13 @@
|
|
|
4
4
|
|
|
5
5
|
import type { AutomergeUrl } from '@dxos/automerge/automerge-repo';
|
|
6
6
|
import { type Context } from '@dxos/context';
|
|
7
|
+
import { migrateDocument } from '@dxos/echo-db';
|
|
7
8
|
import {
|
|
8
9
|
convertLegacyReferences,
|
|
9
10
|
convertLegacySpaceRootDoc,
|
|
10
11
|
findInlineObjectOfType,
|
|
11
|
-
migrateDocument,
|
|
12
12
|
type EchoHost,
|
|
13
|
-
} from '@dxos/echo-
|
|
13
|
+
} from '@dxos/echo-pipeline';
|
|
14
14
|
import { SpaceDocVersion, type SpaceDoc } from '@dxos/echo-protocol';
|
|
15
15
|
import { TYPE_PROPERTIES } from '@dxos/echo-schema';
|
|
16
16
|
import { invariant } from '@dxos/invariant';
|
|
@@ -2,27 +2,28 @@
|
|
|
2
2
|
// Copyright 2023 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { expect } from '
|
|
5
|
+
import { onTestFinished, describe, expect, test } from 'vitest';
|
|
6
6
|
|
|
7
7
|
import { Context } from '@dxos/context';
|
|
8
8
|
import { CredentialGenerator } from '@dxos/credentials';
|
|
9
9
|
import { MockFeedWriter } from '@dxos/feed-store/testing';
|
|
10
10
|
import { Keyring } from '@dxos/keyring';
|
|
11
|
+
import { SpaceId } from '@dxos/keys';
|
|
11
12
|
import { log } from '@dxos/log';
|
|
12
13
|
import { AdmittedFeed, type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
|
|
13
14
|
import { TestBuilder, type TestConnection, TestPeer } from '@dxos/teleport/testing';
|
|
14
|
-
import { afterTest, describe, test } from '@dxos/test';
|
|
15
15
|
|
|
16
|
-
import { NotarizationPlugin } from './notarization-plugin';
|
|
16
|
+
import { NotarizationPlugin, type NotarizationPluginParams } from './notarization-plugin';
|
|
17
17
|
|
|
18
18
|
class TestAgent extends TestPeer {
|
|
19
19
|
private readonly _ctx = new Context();
|
|
20
20
|
|
|
21
21
|
feed = new MockFeedWriter<Credential>();
|
|
22
|
-
notarizationPlugin
|
|
22
|
+
notarizationPlugin: NotarizationPlugin;
|
|
23
23
|
|
|
24
|
-
constructor() {
|
|
24
|
+
constructor(params: NotarizationPluginParams) {
|
|
25
25
|
super();
|
|
26
|
+
this.notarizationPlugin = new NotarizationPlugin(params);
|
|
26
27
|
this.feed.written.on(this._ctx, async ([credential]) => {
|
|
27
28
|
log('written to feed', { credential });
|
|
28
29
|
await this.notarizationPlugin.processCredential(credential);
|
|
@@ -49,10 +50,12 @@ class TestAgent extends TestPeer {
|
|
|
49
50
|
describe('NotarizationPlugin', () => {
|
|
50
51
|
test('notarize single credential', async () => {
|
|
51
52
|
const testBuilder = new TestBuilder();
|
|
52
|
-
|
|
53
|
+
onTestFinished(() => testBuilder.destroy());
|
|
54
|
+
|
|
55
|
+
const params = { spaceId: SpaceId.random() };
|
|
53
56
|
|
|
54
57
|
// peer0 is there to test retries.
|
|
55
|
-
const [_peer0, peer1, peer2] = await testBuilder.createPeers({ factory: () => new TestAgent() });
|
|
58
|
+
const [_peer0, peer1, peer2] = await testBuilder.createPeers({ factory: () => new TestAgent(params) });
|
|
56
59
|
peer1.enableWriting();
|
|
57
60
|
|
|
58
61
|
peer1.feed.written.on(async ([credential]) => {
|
|
@@ -2,14 +2,18 @@
|
|
|
2
2
|
// Copyright 2023 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { DeferredTask, Event, scheduleTask, sleep, TimeoutError, Trigger } from '@dxos/async';
|
|
6
|
-
import { Context, rejectOnDispose } from '@dxos/context';
|
|
7
|
-
import { type CredentialProcessor } from '@dxos/credentials';
|
|
5
|
+
import { DeferredTask, Event, scheduleTask, sleep, TimeoutError, Trigger, scheduleMicroTask } from '@dxos/async';
|
|
6
|
+
import { type Context, rejectOnDispose, Resource } from '@dxos/context';
|
|
7
|
+
import { type CredentialProcessor, verifyCredential } from '@dxos/credentials';
|
|
8
|
+
import { type EdgeHttpClient } from '@dxos/edge-client';
|
|
8
9
|
import { type FeedWriter } from '@dxos/feed-store';
|
|
9
10
|
import { invariant } from '@dxos/invariant';
|
|
10
11
|
import { PublicKey } from '@dxos/keys';
|
|
11
|
-
import {
|
|
12
|
+
import { type SpaceId } from '@dxos/keys';
|
|
13
|
+
import { logInfo, log } from '@dxos/log';
|
|
14
|
+
import { EdgeCallFailedError } from '@dxos/protocols';
|
|
12
15
|
import { schema } from '@dxos/protocols/proto';
|
|
16
|
+
import { type Runtime } from '@dxos/protocols/proto/dxos/config';
|
|
13
17
|
import { type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
|
|
14
18
|
import { type NotarizationService, type NotarizeRequest } from '@dxos/protocols/proto/dxos/mesh/teleport/notarization';
|
|
15
19
|
import { type ExtensionContext, RpcExtension } from '@dxos/teleport';
|
|
@@ -21,8 +25,21 @@ const DEFAULT_SUCCESS_DELAY = 1_000;
|
|
|
21
25
|
|
|
22
26
|
const DEFAULT_NOTARIZE_TIMEOUT = 10_000;
|
|
23
27
|
|
|
28
|
+
const DEFAULT_ACTIVE_EDGE_POLLING_INTERVAL = 3_000;
|
|
29
|
+
|
|
30
|
+
const MAX_EDGE_RETRIES = 2;
|
|
31
|
+
|
|
24
32
|
const WRITER_NOT_SET_ERROR_CODE = 'WRITER_NOT_SET';
|
|
25
33
|
|
|
34
|
+
const credentialCodec = schema.getCodecForType('dxos.halo.credentials.Credential');
|
|
35
|
+
|
|
36
|
+
export type NotarizationPluginParams = {
|
|
37
|
+
spaceId: SpaceId;
|
|
38
|
+
edgeClient?: EdgeHttpClient;
|
|
39
|
+
edgeFeatures?: Runtime.Client.EdgeFeatures;
|
|
40
|
+
activeEdgePollingInterval?: number;
|
|
41
|
+
};
|
|
42
|
+
|
|
26
43
|
export type NotarizeParams = {
|
|
27
44
|
/**
|
|
28
45
|
* For cancellation.
|
|
@@ -53,13 +70,17 @@ export type NotarizeParams = {
|
|
|
53
70
|
* @default {@link DEFAULT_SUCCESS_DELAY}
|
|
54
71
|
*/
|
|
55
72
|
successDelay?: number;
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* A random amount of time before making or retrying an edge request to help prevent large bursts of requests.
|
|
76
|
+
*/
|
|
77
|
+
edgeRetryJitter?: number;
|
|
56
78
|
};
|
|
57
79
|
|
|
58
80
|
/**
|
|
59
81
|
* See NotarizationService proto.
|
|
60
82
|
*/
|
|
61
|
-
export class NotarizationPlugin implements CredentialProcessor {
|
|
62
|
-
private readonly _ctx = new Context();
|
|
83
|
+
export class NotarizationPlugin extends Resource implements CredentialProcessor {
|
|
63
84
|
private readonly _extensionOpened = new Event();
|
|
64
85
|
|
|
65
86
|
private _writer: FeedWriter<Credential> | undefined;
|
|
@@ -67,13 +88,54 @@ export class NotarizationPlugin implements CredentialProcessor {
|
|
|
67
88
|
private readonly _processedCredentials = new ComplexSet<PublicKey>(PublicKey.hash);
|
|
68
89
|
private readonly _processCredentialsTriggers = new ComplexMap<PublicKey, Trigger>(PublicKey.hash);
|
|
69
90
|
|
|
91
|
+
private _activeEdgePollingIntervalHandle: any | undefined = undefined;
|
|
92
|
+
private readonly _activeEdgePollingInterval: number;
|
|
93
|
+
|
|
94
|
+
@logInfo
|
|
95
|
+
private readonly _spaceId: SpaceId;
|
|
96
|
+
|
|
97
|
+
private readonly _edgeClient: EdgeHttpClient | undefined;
|
|
98
|
+
|
|
99
|
+
constructor(params: NotarizationPluginParams) {
|
|
100
|
+
super();
|
|
101
|
+
this._spaceId = params.spaceId;
|
|
102
|
+
this._activeEdgePollingInterval = params.activeEdgePollingInterval ?? DEFAULT_ACTIVE_EDGE_POLLING_INTERVAL;
|
|
103
|
+
if (params.edgeClient && params.edgeFeatures?.feedReplicator) {
|
|
104
|
+
this._edgeClient = params.edgeClient;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
setActiveEdgePollingEnabled(enabled: boolean) {
|
|
109
|
+
invariant(this.isOpen);
|
|
110
|
+
const client = this._edgeClient;
|
|
111
|
+
invariant(client);
|
|
112
|
+
if (enabled && !this._activeEdgePollingIntervalHandle) {
|
|
113
|
+
this._activeEdgePollingIntervalHandle = setInterval(() => {
|
|
114
|
+
if (this._writer) {
|
|
115
|
+
this._notarizePendingEdgeCredentials(client, this._writer);
|
|
116
|
+
}
|
|
117
|
+
}, this._activeEdgePollingInterval);
|
|
118
|
+
} else if (!enabled && this._activeEdgePollingIntervalHandle) {
|
|
119
|
+
clearInterval(this._activeEdgePollingIntervalHandle);
|
|
120
|
+
this._activeEdgePollingIntervalHandle = undefined;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
70
124
|
get hasWriter() {
|
|
71
125
|
return !!this._writer;
|
|
72
126
|
}
|
|
73
127
|
|
|
74
|
-
async
|
|
128
|
+
protected override async _open() {
|
|
129
|
+
if (this._edgeClient && this._writer) {
|
|
130
|
+
this._notarizePendingEdgeCredentials(this._edgeClient, this._writer);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
75
133
|
|
|
76
|
-
async
|
|
134
|
+
protected override async _close() {
|
|
135
|
+
if (this._activeEdgePollingIntervalHandle) {
|
|
136
|
+
clearInterval(this._activeEdgePollingIntervalHandle);
|
|
137
|
+
this._activeEdgePollingIntervalHandle = undefined;
|
|
138
|
+
}
|
|
77
139
|
await this._ctx.dispose();
|
|
78
140
|
}
|
|
79
141
|
|
|
@@ -86,6 +148,7 @@ export class NotarizationPlugin implements CredentialProcessor {
|
|
|
86
148
|
timeout = DEFAULT_NOTARIZE_TIMEOUT,
|
|
87
149
|
retryTimeout = DEFAULT_RETRY_TIMEOUT,
|
|
88
150
|
successDelay = DEFAULT_SUCCESS_DELAY,
|
|
151
|
+
edgeRetryJitter,
|
|
89
152
|
}: NotarizeParams) {
|
|
90
153
|
log('notarize', { credentials });
|
|
91
154
|
invariant(
|
|
@@ -103,24 +166,35 @@ export class NotarizationPlugin implements CredentialProcessor {
|
|
|
103
166
|
});
|
|
104
167
|
opCtx?.onDispose(() => ctx.dispose());
|
|
105
168
|
|
|
106
|
-
// Timeout/
|
|
107
169
|
if (timeout !== 0) {
|
|
108
|
-
|
|
109
|
-
ctx,
|
|
110
|
-
() => {
|
|
111
|
-
log.warn('Notarization timeout', {
|
|
112
|
-
timeout,
|
|
113
|
-
peers: Array.from(this._extensions).map((extension) => extension.remotePeerId),
|
|
114
|
-
});
|
|
115
|
-
void ctx.dispose();
|
|
116
|
-
errors.throw(new TimeoutError(timeout, 'Notarization timed out'));
|
|
117
|
-
},
|
|
118
|
-
timeout,
|
|
119
|
-
);
|
|
170
|
+
this._scheduleTimeout(ctx, errors, timeout);
|
|
120
171
|
}
|
|
121
172
|
|
|
122
173
|
const allNotarized = Promise.all(credentials.map((credential) => this._waitUntilProcessed(credential.id!)));
|
|
123
174
|
|
|
175
|
+
this._tryNotarizeCredentialsWithPeers(ctx, credentials, { retryTimeout, successDelay });
|
|
176
|
+
|
|
177
|
+
if (this._edgeClient) {
|
|
178
|
+
this._tryNotarizeCredentialsWithEdge(ctx, this._edgeClient, credentials, {
|
|
179
|
+
retryTimeout,
|
|
180
|
+
successDelay,
|
|
181
|
+
jitter: edgeRetryJitter,
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
try {
|
|
186
|
+
await Promise.race([rejectOnDispose(ctx), allNotarized, errors.wait()]);
|
|
187
|
+
log('done');
|
|
188
|
+
} finally {
|
|
189
|
+
await ctx.dispose();
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
private _tryNotarizeCredentialsWithPeers(
|
|
194
|
+
ctx: Context,
|
|
195
|
+
credentials: Credential[],
|
|
196
|
+
{ retryTimeout, successDelay }: NotarizationTimeouts,
|
|
197
|
+
) {
|
|
124
198
|
const peersTried = new Set<NotarizationTeleportExtension>();
|
|
125
199
|
|
|
126
200
|
// Repeatable task that tries to notarize credentials with one of the available peers.
|
|
@@ -145,6 +219,7 @@ export class NotarizationPlugin implements CredentialProcessor {
|
|
|
145
219
|
credentials: credentials.filter((credential) => !this._processedCredentials.has(credential.id!)),
|
|
146
220
|
});
|
|
147
221
|
log('success');
|
|
222
|
+
|
|
148
223
|
await sleep(successDelay); // wait before trying with a new peer
|
|
149
224
|
} catch (err: any) {
|
|
150
225
|
if (!ctx.disposed && !err.message.includes(WRITER_NOT_SET_ERROR_CODE)) {
|
|
@@ -156,13 +231,31 @@ export class NotarizationPlugin implements CredentialProcessor {
|
|
|
156
231
|
|
|
157
232
|
notarizeTask.schedule();
|
|
158
233
|
this._extensionOpened.on(ctx, () => notarizeTask.schedule());
|
|
234
|
+
}
|
|
159
235
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
236
|
+
private _tryNotarizeCredentialsWithEdge(
|
|
237
|
+
ctx: Context,
|
|
238
|
+
client: EdgeHttpClient,
|
|
239
|
+
credentials: Credential[],
|
|
240
|
+
timeouts: NotarizationTimeouts & { jitter?: number },
|
|
241
|
+
) {
|
|
242
|
+
const encodedCredentials = credentials.map((credential) => {
|
|
243
|
+
const binary = credentialCodec.encode(credential);
|
|
244
|
+
return Buffer.from(binary).toString('base64');
|
|
245
|
+
});
|
|
246
|
+
scheduleTask(ctx, async () => {
|
|
247
|
+
try {
|
|
248
|
+
await client.notarizeCredentials(
|
|
249
|
+
this._spaceId,
|
|
250
|
+
{ credentials: encodedCredentials },
|
|
251
|
+
{ retry: { count: MAX_EDGE_RETRIES, timeout: timeouts.retryTimeout, jitter: timeouts.jitter } },
|
|
252
|
+
);
|
|
253
|
+
|
|
254
|
+
log('edge notarization success');
|
|
255
|
+
} catch (error: any) {
|
|
256
|
+
handleEdgeError(error);
|
|
257
|
+
}
|
|
258
|
+
});
|
|
166
259
|
}
|
|
167
260
|
|
|
168
261
|
/**
|
|
@@ -180,6 +273,44 @@ export class NotarizationPlugin implements CredentialProcessor {
|
|
|
180
273
|
setWriter(writer: FeedWriter<Credential>) {
|
|
181
274
|
invariant(!this._writer, 'Writer already set.');
|
|
182
275
|
this._writer = writer;
|
|
276
|
+
if (this._edgeClient) {
|
|
277
|
+
this._notarizePendingEdgeCredentials(this._edgeClient, writer);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* The method is used only for adding agent feeds to spaces.
|
|
283
|
+
* When an agent is created we can admit them into all the existing spaces. In case the operation fails
|
|
284
|
+
* this method will fix it on the next space open.
|
|
285
|
+
* Given how rarely this happens there's no need to poll the endpoint.
|
|
286
|
+
*/
|
|
287
|
+
private _notarizePendingEdgeCredentials(client: EdgeHttpClient, writer: FeedWriter<Credential>) {
|
|
288
|
+
scheduleMicroTask(this._ctx, async () => {
|
|
289
|
+
try {
|
|
290
|
+
const response = await client.getCredentialsForNotarization(this._spaceId, {
|
|
291
|
+
retry: { count: MAX_EDGE_RETRIES },
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
const credentials = response.awaitingNotarization.credentials;
|
|
295
|
+
if (!credentials.length) {
|
|
296
|
+
log('edge did not return credentials for notarization');
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
log('got edge credentials for notarization', { count: credentials.length });
|
|
301
|
+
|
|
302
|
+
const decodedCredentials = credentials.map((credential) => {
|
|
303
|
+
const binary = Buffer.from(credential, 'base64');
|
|
304
|
+
return credentialCodec.decode(binary);
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
await this._notarizeCredentials(writer, decodedCredentials);
|
|
308
|
+
|
|
309
|
+
log.info('notarized edge credentials', { count: decodedCredentials.length });
|
|
310
|
+
} catch (error: any) {
|
|
311
|
+
handleEdgeError(error);
|
|
312
|
+
}
|
|
313
|
+
});
|
|
183
314
|
}
|
|
184
315
|
|
|
185
316
|
private async _waitUntilProcessed(id: PublicKey) {
|
|
@@ -196,12 +327,20 @@ export class NotarizationPlugin implements CredentialProcessor {
|
|
|
196
327
|
if (!this._writer) {
|
|
197
328
|
throw new Error(WRITER_NOT_SET_ERROR_CODE);
|
|
198
329
|
}
|
|
199
|
-
|
|
330
|
+
await this._notarizeCredentials(this._writer, request.credentials ?? []);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
private async _notarizeCredentials(writer: FeedWriter<Credential>, credentials: Credential[]) {
|
|
334
|
+
for (const credential of credentials) {
|
|
200
335
|
invariant(credential.id, 'Credential must have an id');
|
|
201
336
|
if (this._processedCredentials.has(credential.id)) {
|
|
202
337
|
continue;
|
|
203
338
|
}
|
|
204
|
-
await
|
|
339
|
+
const verificationResult = await verifyCredential(credential);
|
|
340
|
+
if (verificationResult.kind === 'fail') {
|
|
341
|
+
throw new Error(`Credential verification failed: ${verificationResult.errors.join('\n')}.`);
|
|
342
|
+
}
|
|
343
|
+
await writer.write(credential);
|
|
205
344
|
}
|
|
206
345
|
}
|
|
207
346
|
|
|
@@ -220,8 +359,31 @@ export class NotarizationPlugin implements CredentialProcessor {
|
|
|
220
359
|
});
|
|
221
360
|
return extension;
|
|
222
361
|
}
|
|
362
|
+
|
|
363
|
+
private _scheduleTimeout(ctx: Context, errors: Trigger, timeout: number) {
|
|
364
|
+
scheduleTask(
|
|
365
|
+
ctx,
|
|
366
|
+
() => {
|
|
367
|
+
log.warn('Notarization timeout', {
|
|
368
|
+
timeout,
|
|
369
|
+
peers: Array.from(this._extensions).map((extension) => extension.remotePeerId),
|
|
370
|
+
});
|
|
371
|
+
void ctx.dispose();
|
|
372
|
+
errors.throw(new TimeoutError(timeout, 'Notarization timed out'));
|
|
373
|
+
},
|
|
374
|
+
timeout,
|
|
375
|
+
);
|
|
376
|
+
}
|
|
223
377
|
}
|
|
224
378
|
|
|
379
|
+
const handleEdgeError = (error: any) => {
|
|
380
|
+
if (!(error instanceof EdgeCallFailedError) || error.errorData) {
|
|
381
|
+
log.catch(error);
|
|
382
|
+
} else {
|
|
383
|
+
log.info('Edge notarization failure', { reason: error.reason });
|
|
384
|
+
}
|
|
385
|
+
};
|
|
386
|
+
|
|
225
387
|
export type NotarizationTeleportExtensionParams = {
|
|
226
388
|
onOpen: () => Promise<void>;
|
|
227
389
|
onClose: () => Promise<void>;
|
|
@@ -261,6 +423,11 @@ export class NotarizationTeleportExtension extends RpcExtension<Services, Servic
|
|
|
261
423
|
}
|
|
262
424
|
}
|
|
263
425
|
|
|
426
|
+
type NotarizationTimeouts = {
|
|
427
|
+
retryTimeout: number;
|
|
428
|
+
successDelay: number;
|
|
429
|
+
};
|
|
430
|
+
|
|
264
431
|
type Services = {
|
|
265
432
|
NotarizationService: NotarizationService;
|
|
266
433
|
};
|
|
@@ -2,21 +2,17 @@
|
|
|
2
2
|
// Copyright 2023 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import
|
|
6
|
-
import chaiAsPromised from 'chai-as-promised';
|
|
5
|
+
import { afterEach, onTestFinished, beforeEach, describe, expect, test } from 'vitest';
|
|
7
6
|
|
|
8
7
|
import { Trigger } from '@dxos/async';
|
|
9
8
|
import { Context } from '@dxos/context';
|
|
10
9
|
import { PublicKey } from '@dxos/keys';
|
|
11
10
|
import { type Space, type SpacesService } from '@dxos/protocols/proto/dxos/client/services';
|
|
12
|
-
import { afterEach, afterTest, beforeEach, describe, test } from '@dxos/test';
|
|
13
11
|
|
|
14
12
|
import { SpacesServiceImpl } from './spaces-service';
|
|
15
13
|
import { type ServiceContext } from '../services';
|
|
16
14
|
import { createServiceContext } from '../testing';
|
|
17
15
|
|
|
18
|
-
chai.use(chaiAsPromised);
|
|
19
|
-
|
|
20
16
|
describe('SpacesService', () => {
|
|
21
17
|
let serviceContext: ServiceContext;
|
|
22
18
|
let spacesService: SpacesService;
|
|
@@ -36,7 +32,7 @@ describe('SpacesService', () => {
|
|
|
36
32
|
|
|
37
33
|
describe('createSpace', () => {
|
|
38
34
|
test('fails if no identity is available', async () => {
|
|
39
|
-
await expect(spacesService.createSpace()).
|
|
35
|
+
await expect(spacesService.createSpace()).rejects.toBeInstanceOf(Error);
|
|
40
36
|
});
|
|
41
37
|
|
|
42
38
|
test('creates a new space', async () => {
|
|
@@ -56,7 +52,7 @@ describe('SpacesService', () => {
|
|
|
56
52
|
query.subscribe(({ spaces }) => {
|
|
57
53
|
result.wake(spaces);
|
|
58
54
|
});
|
|
59
|
-
|
|
55
|
+
onTestFinished(() => query.close());
|
|
60
56
|
expect(await result.wait()).to.be.length(0);
|
|
61
57
|
});
|
|
62
58
|
|
|
@@ -73,7 +69,7 @@ describe('SpacesService', () => {
|
|
|
73
69
|
query.subscribe(({ spaces }) => {
|
|
74
70
|
result.wake(spaces);
|
|
75
71
|
});
|
|
76
|
-
|
|
72
|
+
onTestFinished(() => query.close());
|
|
77
73
|
|
|
78
74
|
const spaces = await result.wait();
|
|
79
75
|
expect(spaces).to.be.length(3);
|
|
@@ -87,7 +83,7 @@ describe('SpacesService', () => {
|
|
|
87
83
|
query.subscribe(({ spaces }) => {
|
|
88
84
|
result.wake(spaces);
|
|
89
85
|
});
|
|
90
|
-
|
|
86
|
+
onTestFinished(() => query.close());
|
|
91
87
|
expect(await result.wait()).to.be.length(0);
|
|
92
88
|
|
|
93
89
|
result.reset();
|
|
@@ -60,7 +60,7 @@ export class SpacesServiceImpl implements SpacesService {
|
|
|
60
60
|
return this._serializeSpace(space);
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
async updateSpace({ spaceKey, state }: UpdateSpaceRequest) {
|
|
63
|
+
async updateSpace({ spaceKey, state, edgeReplication }: UpdateSpaceRequest) {
|
|
64
64
|
const dataSpaceManager = await this._getDataSpaceManager();
|
|
65
65
|
const space = dataSpaceManager.spaces.get(spaceKey) ?? raise(new SpaceNotFoundError(spaceKey));
|
|
66
66
|
|
|
@@ -77,6 +77,10 @@ export class SpacesServiceImpl implements SpacesService {
|
|
|
77
77
|
throw new ApiError('Invalid space state');
|
|
78
78
|
}
|
|
79
79
|
}
|
|
80
|
+
|
|
81
|
+
if (edgeReplication !== undefined) {
|
|
82
|
+
await dataSpaceManager.setSpaceEdgeReplicationSetting(spaceKey, edgeReplication);
|
|
83
|
+
}
|
|
80
84
|
}
|
|
81
85
|
|
|
82
86
|
async updateMemberRole(request: UpdateMemberRoleRequest): Promise<void> {
|
|
@@ -308,6 +312,7 @@ export class SpacesServiceImpl implements SpacesService {
|
|
|
308
312
|
creator: space.inner.spaceState.creator?.key,
|
|
309
313
|
cache: space.cache,
|
|
310
314
|
metrics: space.metrics,
|
|
315
|
+
edgeReplication: space.getEdgeReplicationSetting(),
|
|
311
316
|
};
|
|
312
317
|
}
|
|
313
318
|
|