@dxos/client-services 0.8.4-main.72ec0f3 → 0.8.4-main.74a063c4e0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/dist/lib/browser/{chunk-HJH6BNTN.mjs → chunk-3LSLNVKQ.mjs} +2102 -1870
  2. package/dist/lib/browser/chunk-3LSLNVKQ.mjs.map +7 -0
  3. package/dist/lib/browser/chunk-NQSC7HOE.mjs +22 -0
  4. package/dist/lib/browser/chunk-NQSC7HOE.mjs.map +7 -0
  5. package/dist/lib/browser/chunk-QCWEHHJW.mjs +24 -0
  6. package/dist/lib/browser/chunk-QCWEHHJW.mjs.map +7 -0
  7. package/dist/lib/browser/index.mjs +451 -67
  8. package/dist/lib/browser/index.mjs.map +4 -4
  9. package/dist/lib/browser/meta.json +1 -1
  10. package/dist/lib/browser/packlets/diagnostics/browser-diagnostics-broadcast.mjs +93 -0
  11. package/dist/lib/browser/packlets/diagnostics/browser-diagnostics-broadcast.mjs.map +7 -0
  12. package/dist/lib/browser/packlets/diagnostics/diagnostics-broadcast.mjs +11 -0
  13. package/dist/lib/browser/packlets/diagnostics/diagnostics-broadcast.mjs.map +7 -0
  14. package/dist/lib/browser/packlets/locks/browser.mjs +126 -0
  15. package/dist/lib/browser/packlets/locks/browser.mjs.map +7 -0
  16. package/dist/lib/browser/packlets/locks/node.mjs +66 -0
  17. package/dist/lib/browser/packlets/locks/node.mjs.map +7 -0
  18. package/dist/lib/browser/testing/index.mjs +36 -17
  19. package/dist/lib/browser/testing/index.mjs.map +3 -3
  20. package/dist/lib/node-esm/chunk-2SZHAWBN.mjs +24 -0
  21. package/dist/lib/node-esm/chunk-2SZHAWBN.mjs.map +7 -0
  22. package/dist/lib/node-esm/{chunk-ONQM6RQH.mjs → chunk-5S7PIHLS.mjs} +1942 -1579
  23. package/dist/lib/node-esm/chunk-5S7PIHLS.mjs.map +7 -0
  24. package/dist/lib/node-esm/chunk-PKEGMOQ4.mjs +22 -0
  25. package/dist/lib/node-esm/chunk-PKEGMOQ4.mjs.map +7 -0
  26. package/dist/lib/node-esm/index.mjs +451 -67
  27. package/dist/lib/node-esm/index.mjs.map +4 -4
  28. package/dist/lib/node-esm/meta.json +1 -1
  29. package/dist/lib/node-esm/packlets/diagnostics/browser-diagnostics-broadcast.mjs +93 -0
  30. package/dist/lib/node-esm/packlets/diagnostics/browser-diagnostics-broadcast.mjs.map +7 -0
  31. package/dist/lib/node-esm/packlets/diagnostics/diagnostics-broadcast.mjs +11 -0
  32. package/dist/lib/node-esm/packlets/diagnostics/diagnostics-broadcast.mjs.map +7 -0
  33. package/dist/lib/node-esm/packlets/locks/browser.mjs +126 -0
  34. package/dist/lib/node-esm/packlets/locks/browser.mjs.map +7 -0
  35. package/dist/lib/node-esm/packlets/locks/node.mjs +66 -0
  36. package/dist/lib/node-esm/packlets/locks/node.mjs.map +7 -0
  37. package/dist/lib/node-esm/testing/index.mjs +36 -17
  38. package/dist/lib/node-esm/testing/index.mjs.map +3 -3
  39. package/dist/types/src/index.d.ts +1 -0
  40. package/dist/types/src/index.d.ts.map +1 -1
  41. package/dist/types/src/packlets/agents/edge-agent-manager.d.ts +3 -2
  42. package/dist/types/src/packlets/agents/edge-agent-manager.d.ts.map +1 -1
  43. package/dist/types/src/packlets/agents/edge-agent-service.d.ts.map +1 -1
  44. package/dist/types/src/packlets/devtools/devtools.d.ts +2 -2
  45. package/dist/types/src/packlets/devtools/devtools.d.ts.map +1 -1
  46. package/dist/types/src/packlets/diagnostics/index.d.ts +1 -1
  47. package/dist/types/src/packlets/diagnostics/index.d.ts.map +1 -1
  48. package/dist/types/src/packlets/identity/authenticator.d.ts +2 -2
  49. package/dist/types/src/packlets/identity/authenticator.d.ts.map +1 -1
  50. package/dist/types/src/packlets/identity/contacts-service.d.ts.map +1 -1
  51. package/dist/types/src/packlets/identity/identity-manager.d.ts +6 -6
  52. package/dist/types/src/packlets/identity/identity-manager.d.ts.map +1 -1
  53. package/dist/types/src/packlets/identity/identity-recovery-manager.d.ts +7 -6
  54. package/dist/types/src/packlets/identity/identity-recovery-manager.d.ts.map +1 -1
  55. package/dist/types/src/packlets/identity/identity-service.d.ts +1 -6
  56. package/dist/types/src/packlets/identity/identity-service.d.ts.map +1 -1
  57. package/dist/types/src/packlets/identity/identity.d.ts +8 -11
  58. package/dist/types/src/packlets/identity/identity.d.ts.map +1 -1
  59. package/dist/types/src/packlets/invitations/device-invitation-protocol.d.ts +4 -4
  60. package/dist/types/src/packlets/invitations/device-invitation-protocol.d.ts.map +1 -1
  61. package/dist/types/src/packlets/invitations/edge-invitation-handler.d.ts.map +1 -1
  62. package/dist/types/src/packlets/invitations/invitation-guest-extenstion.d.ts.map +1 -1
  63. package/dist/types/src/packlets/invitations/invitation-host-extension.d.ts.map +1 -1
  64. package/dist/types/src/packlets/invitations/invitation-protocol.d.ts +2 -3
  65. package/dist/types/src/packlets/invitations/invitation-protocol.d.ts.map +1 -1
  66. package/dist/types/src/packlets/invitations/invitations-handler.d.ts +4 -4
  67. package/dist/types/src/packlets/invitations/invitations-handler.d.ts.map +1 -1
  68. package/dist/types/src/packlets/invitations/invitations-manager.d.ts +3 -3
  69. package/dist/types/src/packlets/invitations/invitations-manager.d.ts.map +1 -1
  70. package/dist/types/src/packlets/invitations/space-invitation-protocol.d.ts +2 -2
  71. package/dist/types/src/packlets/invitations/space-invitation-protocol.d.ts.map +1 -1
  72. package/dist/types/src/packlets/locks/index.d.ts +1 -1
  73. package/dist/types/src/packlets/locks/index.d.ts.map +1 -1
  74. package/dist/types/src/packlets/logging/logging-service.d.ts +4 -0
  75. package/dist/types/src/packlets/logging/logging-service.d.ts.map +1 -1
  76. package/dist/types/src/packlets/network/network-service.d.ts.map +1 -1
  77. package/dist/types/src/packlets/services/client-rpc-server.d.ts +2 -2
  78. package/dist/types/src/packlets/services/client-rpc-server.d.ts.map +1 -1
  79. package/dist/types/src/packlets/services/feed-syncer.d.ts +59 -0
  80. package/dist/types/src/packlets/services/feed-syncer.d.ts.map +1 -0
  81. package/dist/types/src/packlets/services/feed-syncer.test.d.ts +2 -0
  82. package/dist/types/src/packlets/services/feed-syncer.test.d.ts.map +1 -0
  83. package/dist/types/src/packlets/services/platform.d.ts.map +1 -1
  84. package/dist/types/src/packlets/services/service-context.d.ts +13 -8
  85. package/dist/types/src/packlets/services/service-context.d.ts.map +1 -1
  86. package/dist/types/src/packlets/services/service-host.d.ts +20 -6
  87. package/dist/types/src/packlets/services/service-host.d.ts.map +1 -1
  88. package/dist/types/src/packlets/space-export/space-archive-reader.d.ts +9 -1
  89. package/dist/types/src/packlets/space-export/space-archive-reader.d.ts.map +1 -1
  90. package/dist/types/src/packlets/space-export/space-archive-writer.d.ts +6 -0
  91. package/dist/types/src/packlets/space-export/space-archive-writer.d.ts.map +1 -1
  92. package/dist/types/src/packlets/space-export/space-archive.test.d.ts +2 -0
  93. package/dist/types/src/packlets/space-export/space-archive.test.d.ts.map +1 -0
  94. package/dist/types/src/packlets/spaces/data-space-manager.d.ts +27 -15
  95. package/dist/types/src/packlets/spaces/data-space-manager.d.ts.map +1 -1
  96. package/dist/types/src/packlets/spaces/data-space.d.ts +24 -8
  97. package/dist/types/src/packlets/spaces/data-space.d.ts.map +1 -1
  98. package/dist/types/src/packlets/spaces/edge-feed-replicator.d.ts +2 -2
  99. package/dist/types/src/packlets/spaces/edge-feed-replicator.d.ts.map +1 -1
  100. package/dist/types/src/packlets/spaces/genesis.d.ts +2 -1
  101. package/dist/types/src/packlets/spaces/genesis.d.ts.map +1 -1
  102. package/dist/types/src/packlets/spaces/notarization-plugin.d.ts +6 -6
  103. package/dist/types/src/packlets/spaces/notarization-plugin.d.ts.map +1 -1
  104. package/dist/types/src/packlets/spaces/spaces-service.d.ts +2 -2
  105. package/dist/types/src/packlets/spaces/spaces-service.d.ts.map +1 -1
  106. package/dist/types/src/packlets/testing/invitation-utils.d.ts +6 -3
  107. package/dist/types/src/packlets/testing/invitation-utils.d.ts.map +1 -1
  108. package/dist/types/src/packlets/testing/test-builder.d.ts +6 -5
  109. package/dist/types/src/packlets/testing/test-builder.d.ts.map +1 -1
  110. package/dist/types/src/packlets/worker/worker-runtime.d.ts +31 -4
  111. package/dist/types/src/packlets/worker/worker-runtime.d.ts.map +1 -1
  112. package/dist/types/src/packlets/worker/worker-session.d.ts +2 -2
  113. package/dist/types/src/packlets/worker/worker-session.d.ts.map +1 -1
  114. package/dist/types/src/version.d.ts +1 -1
  115. package/dist/types/src/version.d.ts.map +1 -1
  116. package/dist/types/tsconfig.tsbuildinfo +1 -1
  117. package/package.json +70 -48
  118. package/src/index.ts +1 -0
  119. package/src/packlets/agents/edge-agent-manager.ts +8 -5
  120. package/src/packlets/agents/edge-agent-service.ts +2 -1
  121. package/src/packlets/devices/devices-service.test.ts +0 -1
  122. package/src/packlets/devtools/devtools.ts +2 -3
  123. package/src/packlets/diagnostics/index.ts +1 -1
  124. package/src/packlets/identity/authenticator.ts +2 -2
  125. package/src/packlets/identity/contacts-service.ts +0 -1
  126. package/src/packlets/identity/identity-manager.test.ts +5 -5
  127. package/src/packlets/identity/identity-manager.ts +21 -18
  128. package/src/packlets/identity/identity-recovery-manager.ts +22 -18
  129. package/src/packlets/identity/identity-service.test.ts +6 -27
  130. package/src/packlets/identity/identity-service.ts +5 -76
  131. package/src/packlets/identity/identity.test.ts +2 -2
  132. package/src/packlets/identity/identity.ts +9 -32
  133. package/src/packlets/invitations/device-invitation-protocol.ts +5 -6
  134. package/src/packlets/invitations/edge-invitation-handler.ts +4 -3
  135. package/src/packlets/invitations/invitation-guest-extenstion.ts +6 -4
  136. package/src/packlets/invitations/invitation-host-extension.ts +6 -4
  137. package/src/packlets/invitations/invitation-protocol.ts +2 -3
  138. package/src/packlets/invitations/invitations-handler.test.ts +4 -5
  139. package/src/packlets/invitations/invitations-handler.ts +10 -10
  140. package/src/packlets/invitations/invitations-manager.ts +37 -14
  141. package/src/packlets/invitations/invitations-service.ts +4 -4
  142. package/src/packlets/invitations/space-invitation-protocol.test.ts +17 -16
  143. package/src/packlets/invitations/space-invitation-protocol.ts +10 -15
  144. package/src/packlets/locks/index.ts +1 -1
  145. package/src/packlets/logging/logging-service.ts +4 -0
  146. package/src/packlets/network/network-service.test.ts +0 -1
  147. package/src/packlets/network/network-service.ts +5 -4
  148. package/src/packlets/services/client-rpc-server.ts +4 -4
  149. package/src/packlets/services/feed-syncer.test.ts +340 -0
  150. package/src/packlets/services/feed-syncer.ts +337 -0
  151. package/src/packlets/services/platform.ts +7 -1
  152. package/src/packlets/services/service-context.test.ts +3 -2
  153. package/src/packlets/services/service-context.ts +129 -44
  154. package/src/packlets/services/service-host.test.ts +8 -8
  155. package/src/packlets/services/service-host.ts +63 -22
  156. package/src/packlets/services/service-registry.test.ts +0 -1
  157. package/src/packlets/space-export/space-archive-reader.ts +64 -3
  158. package/src/packlets/space-export/space-archive-writer.ts +39 -2
  159. package/src/packlets/space-export/space-archive.test.ts +287 -0
  160. package/src/packlets/spaces/data-space-manager.test.ts +79 -13
  161. package/src/packlets/spaces/data-space-manager.ts +97 -107
  162. package/src/packlets/spaces/data-space.ts +52 -29
  163. package/src/packlets/spaces/edge-feed-replicator.test.ts +1 -1
  164. package/src/packlets/spaces/edge-feed-replicator.ts +10 -9
  165. package/src/packlets/spaces/epoch-migrations.ts +5 -5
  166. package/src/packlets/spaces/genesis.ts +6 -1
  167. package/src/packlets/spaces/notarization-plugin.test.ts +2 -2
  168. package/src/packlets/spaces/notarization-plugin.ts +10 -9
  169. package/src/packlets/spaces/spaces-service.test.ts +9 -7
  170. package/src/packlets/spaces/spaces-service.ts +40 -16
  171. package/src/packlets/storage/storage.ts +4 -4
  172. package/src/packlets/testing/invitation-utils.ts +10 -6
  173. package/src/packlets/testing/test-builder.ts +36 -10
  174. package/src/packlets/worker/worker-runtime.ts +150 -13
  175. package/src/packlets/worker/worker-session.ts +8 -8
  176. package/src/version.ts +1 -1
  177. package/dist/lib/browser/chunk-HJH6BNTN.mjs.map +0 -7
  178. package/dist/lib/node-esm/chunk-ONQM6RQH.mjs.map +0 -7
  179. package/dist/types/src/packlets/identity/default-space-state-machine.d.ts +0 -19
  180. package/dist/types/src/packlets/identity/default-space-state-machine.d.ts.map +0 -1
  181. package/src/packlets/identity/default-space-state-machine.ts +0 -44
@@ -2,11 +2,15 @@
2
2
  // Copyright 2021 DXOS.org
3
3
  //
4
4
 
5
+ import * as SqlClient from '@effect/sql/SqlClient';
6
+ import * as Effect from 'effect/Effect';
7
+
5
8
  import { Event, synchronized } from '@dxos/async';
6
9
  import { type ClientServices, clientServiceBundle } from '@dxos/client-protocol';
7
10
  import { type Config } from '@dxos/config';
8
11
  import { Context } from '@dxos/context';
9
12
  import { EdgeClient, type EdgeConnection, EdgeHttpClient, createStubEdgeIdentity } from '@dxos/edge-client';
13
+ import { RuntimeProvider } from '@dxos/effect';
10
14
  import { invariant } from '@dxos/invariant';
11
15
  import { PublicKey } from '@dxos/keys';
12
16
  import { type LevelDB } from '@dxos/kv-store';
@@ -21,6 +25,8 @@ import {
21
25
  import { trace } from '@dxos/protocols';
22
26
  import { SystemStatus } from '@dxos/protocols/proto/dxos/client/services';
23
27
  import { type Storage } from '@dxos/random-access-storage';
28
+ import * as SqlExport from '@dxos/sql-sqlite/SqlExport';
29
+ import type * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
24
30
  import { TRACE_PROCESSOR, trace as Trace } from '@dxos/tracing';
25
31
  import { WebsocketRpcClient } from '@dxos/websocket-rpc';
26
32
 
@@ -41,11 +47,10 @@ import { NetworkServiceImpl } from '../network';
41
47
  import { SpacesServiceImpl } from '../spaces';
42
48
  import { createLevel, createStorageObjects } from '../storage';
43
49
  import { SystemServiceImpl } from '../system';
44
-
45
- import { ServiceContext, type ServiceContextRuntimeParams } from './service-context';
50
+ import { ServiceContext, type ServiceContextRuntimeProps } from './service-context';
46
51
  import { ServiceRegistry } from './service-registry';
47
52
 
48
- export type ClientServicesHostParams = {
53
+ export type ClientServicesHostProps = {
49
54
  /**
50
55
  * Can be omitted if `initialize` is later called.
51
56
  */
@@ -57,7 +62,8 @@ export type ClientServicesHostParams = {
57
62
  level?: LevelDB;
58
63
  lockKey?: string;
59
64
  callbacks?: ClientServicesHostCallbacks;
60
- runtimeParams?: ServiceContextRuntimeParams;
65
+ runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlExport.SqlExport | SqlTransaction.SqlTransaction>;
66
+ runtimeProps?: ServiceContextRuntimeProps;
61
67
  };
62
68
 
63
69
  export type ClientServicesHostCallbacks = {
@@ -95,7 +101,10 @@ export class ClientServicesHost {
95
101
  private _edgeHttpClient?: EdgeHttpClient = undefined;
96
102
 
97
103
  private _serviceContext!: ServiceContext;
98
- private readonly _runtimeParams: ServiceContextRuntimeParams;
104
+ private readonly _runtime: RuntimeProvider.RuntimeProvider<
105
+ SqlClient.SqlClient | SqlExport.SqlExport | SqlTransaction.SqlTransaction
106
+ >;
107
+ private readonly _runtimeProps: ServiceContextRuntimeProps;
99
108
  private diagnosticsBroadcastHandler: CollectDiagnosticsBroadcastHandler;
100
109
 
101
110
  @Trace.info()
@@ -116,20 +125,14 @@ export class ClientServicesHost {
116
125
  // TODO(wittjosiah): Turn this on by default.
117
126
  lockKey,
118
127
  callbacks,
119
- runtimeParams,
120
- }: ClientServicesHostParams = {}) {
128
+ runtime,
129
+ runtimeProps,
130
+ }: ClientServicesHostProps) {
121
131
  this._storage = storage;
122
132
  this._level = level;
123
133
  this._callbacks = callbacks;
124
- this._runtimeParams = runtimeParams ?? {};
125
-
126
- if (this._runtimeParams.disableP2pReplication === undefined) {
127
- this._runtimeParams.disableP2pReplication = config?.get('runtime.client.disableP2pReplication', false);
128
- }
129
-
130
- if (this._runtimeParams.enableVectorIndexing === undefined) {
131
- this._runtimeParams.enableVectorIndexing = config?.get('runtime.client.enableVectorIndexing', false);
132
- }
134
+ this._runtime = runtime;
135
+ this._runtimeProps = runtimeProps ?? {};
133
136
 
134
137
  if (config) {
135
138
  this.initialize({ config, transportFactory, signalManager });
@@ -143,7 +146,7 @@ export class ClientServicesHost {
143
146
  void this.open(new Context());
144
147
  }
145
148
  },
146
- onRelease: () => this.close(),
149
+ onRelease: () => this.close(Context.default()),
147
150
  });
148
151
  }
149
152
 
@@ -200,6 +203,30 @@ export class ClientServicesHost {
200
203
  return this._serviceRegistry.services;
201
204
  }
202
205
 
206
+ /**
207
+ * Debugging util.
208
+ */
209
+ async exportSqliteDatabase(): Promise<Uint8Array> {
210
+ return await RuntimeProvider.runPromise(this._runtime)(
211
+ Effect.gen(function* () {
212
+ const sql = yield* SqlExport.SqlExport;
213
+ return yield* sql.export;
214
+ }),
215
+ );
216
+ }
217
+
218
+ /**
219
+ * Debugging util.
220
+ */
221
+ async runSqliteQuery(query: string, params?: any[]): Promise<readonly Record<string, unknown>[]> {
222
+ return await RuntimeProvider.runPromise(this._runtime)(
223
+ Effect.gen(function* () {
224
+ const sql = yield* SqlClient.SqlClient;
225
+ return yield* sql`${sql.unsafe(query, params)}`;
226
+ }),
227
+ );
228
+ }
229
+
203
230
  /**
204
231
  * Initialize the service host with the config.
205
232
  * Config can also be provided in the constructor.
@@ -210,6 +237,13 @@ export class ClientServicesHost {
210
237
  log('initializing...');
211
238
 
212
239
  if (config) {
240
+ if (this._runtimeProps.disableP2pReplication === undefined) {
241
+ this._runtimeProps.disableP2pReplication = config?.get('runtime.client.disableP2pReplication', false);
242
+ }
243
+ if (this._runtimeProps.enableVectorIndexing === undefined) {
244
+ this._runtimeProps.enableVectorIndexing = config?.get('runtime.client.enableVectorIndexing', false);
245
+ }
246
+
213
247
  invariant(!this._config, 'config already set');
214
248
  this._config = config;
215
249
  if (!this._storage) {
@@ -224,8 +258,9 @@ export class ClientServicesHost {
224
258
 
225
259
  const endpoint = config?.get('runtime.services.edge.url');
226
260
  if (endpoint) {
227
- this._edgeConnection = new EdgeClient(createStubEdgeIdentity(), { socketEndpoint: endpoint });
228
- this._edgeHttpClient = new EdgeHttpClient(endpoint);
261
+ const clientTag = config?.get('runtime.app.env.DX_EDGE_CLIENT_TAG');
262
+ this._edgeConnection = new EdgeClient(createStubEdgeIdentity(), { socketEndpoint: endpoint, clientTag });
263
+ this._edgeHttpClient = new EdgeHttpClient(endpoint, { clientTag });
229
264
  }
230
265
 
231
266
  const {
@@ -291,7 +326,8 @@ export class ClientServicesHost {
291
326
  this._signalManager,
292
327
  this._edgeConnection,
293
328
  this._edgeHttpClient,
294
- this._runtimeParams,
329
+ this._runtime,
330
+ this._runtimeProps,
295
331
  this._config.get('runtime.client.edgeFeatures'),
296
332
  );
297
333
 
@@ -309,7 +345,6 @@ export class ClientServicesHost {
309
345
  this._serviceContext.identityManager,
310
346
  this._serviceContext.recoveryManager,
311
347
  this._serviceContext.keyring,
312
- () => this._serviceContext.dataSpaceManager!,
313
348
  (params) => this._createIdentity(params),
314
349
  (profile) => this._serviceContext.broadcastProfileUpdate(profile),
315
350
  );
@@ -335,6 +370,7 @@ export class ClientServicesHost {
335
370
 
336
371
  DataService: this._serviceContext.echoHost.dataService,
337
372
  QueryService: this._serviceContext.echoHost.queryService,
373
+ QueueService: this._serviceContext.echoHost.queuesService,
338
374
 
339
375
  NetworkService: new NetworkServiceImpl(
340
376
  this._serviceContext.networkManager,
@@ -355,8 +391,13 @@ export class ClientServicesHost {
355
391
  EdgeAgentService: new EdgeAgentServiceImpl(agentManagerProvider, this._edgeConnection),
356
392
  });
357
393
 
394
+ log('service-host: opening service context...');
358
395
  await this._serviceContext.open(ctx);
396
+ log('service-host: service context opened');
397
+
398
+ log('service-host: opening identity service...');
359
399
  await identityService.open();
400
+ log('service-host: identity service opened');
360
401
 
361
402
  const devtoolsProxy = this._config?.get('runtime.client.devtoolsProxy');
362
403
  if (devtoolsProxy) {
@@ -380,7 +421,7 @@ export class ClientServicesHost {
380
421
 
381
422
  @synchronized
382
423
  @Trace.span()
383
- async close(): Promise<void> {
424
+ async close(ctx: Context): Promise<void> {
384
425
  if (!this._open) {
385
426
  return;
386
427
  }
@@ -15,7 +15,6 @@ import { createLinkedPorts, createProtoRpcPeer, createServiceBundle } from '@dxo
15
15
 
16
16
  import { SystemServiceImpl } from '../system';
17
17
  import { createServiceContext } from '../testing';
18
-
19
18
  import { ServiceRegistry } from './service-registry';
20
19
 
21
20
  // TODO(burdon): Create TestService (that doesn't require peers).
@@ -6,12 +6,26 @@ import type { DocumentId } from '@automerge/automerge-repo';
6
6
 
7
7
  import { assertArgument, failedInvariant, invariant } from '@dxos/invariant';
8
8
  import { log } from '@dxos/log';
9
- import { SpaceArchiveFileStructure, type SpaceArchiveMetadata } from '@dxos/protocols';
9
+ import {
10
+ type FeedArchiveBlock,
11
+ type FeedArchiveMetadata,
12
+ SpaceArchiveFileStructure,
13
+ type SpaceArchiveMetadata,
14
+ } from '@dxos/protocols';
10
15
  import type { SpaceArchive } from '@dxos/protocols/proto/dxos/client/services';
11
16
 
17
+ /**
18
+ * Extracted feed data from the archive.
19
+ */
20
+ export type ExtractedFeed = {
21
+ metadata: FeedArchiveMetadata;
22
+ blocks: FeedArchiveBlock[];
23
+ };
24
+
12
25
  export type ExtractedSpaceArchive = {
13
26
  metadata: SpaceArchiveMetadata;
14
27
  documents: Record<DocumentId, Uint8Array>;
28
+ feeds: Record<string, ExtractedFeed>;
15
29
  };
16
30
 
17
31
  export const extractSpaceArchive = async (archive: SpaceArchive): Promise<ExtractedSpaceArchive> => {
@@ -29,6 +43,53 @@ export const extractSpaceArchive = async (archive: SpaceArchive): Promise<Extrac
29
43
  documents[documentId] = entry.content ?? failedInvariant();
30
44
  }
31
45
 
32
- log.info('extracted space archive', { metadata, documents });
33
- return { metadata, documents };
46
+ const feeds: Record<string, ExtractedFeed> = {};
47
+ const feedsPrefix = `${SpaceArchiveFileStructure.feeds}/`;
48
+ const feedEntries = entries.filter((entry) => entry.fileName.startsWith(feedsPrefix));
49
+
50
+ const feedMetadataByFeedId = new Map<string, FeedArchiveMetadata>();
51
+ const feedBlocksByFeedId = new Map<string, Map<number, FeedArchiveBlock[]>>();
52
+
53
+ for (const entry of feedEntries) {
54
+ const relativePath = entry.fileName.slice(feedsPrefix.length);
55
+ const pathParts = relativePath.split('/');
56
+ if (pathParts.length !== 2) {
57
+ continue;
58
+ }
59
+
60
+ const [feedId, fileName] = pathParts;
61
+ invariant(feedId, 'Feed ID is required');
62
+ invariant(fileName, 'File name is required');
63
+
64
+ if (fileName === SpaceArchiveFileStructure.feedMetadata) {
65
+ const feedMetadata = JSON.parse(entry.getContentAsText()) as FeedArchiveMetadata;
66
+ feedMetadataByFeedId.set(feedId, feedMetadata);
67
+ } else if (fileName.startsWith(SpaceArchiveFileStructure.feedBlocksPrefix) && fileName.endsWith('.json')) {
68
+ const chunkIndexStr = fileName.slice(SpaceArchiveFileStructure.feedBlocksPrefix.length).replace(/\.json$/, '');
69
+ const chunkIndex = parseInt(chunkIndexStr, 10);
70
+ invariant(!isNaN(chunkIndex), `Invalid chunk index: ${chunkIndexStr}`);
71
+
72
+ const blocks = JSON.parse(entry.getContentAsText()) as FeedArchiveBlock[];
73
+ if (!feedBlocksByFeedId.has(feedId)) {
74
+ feedBlocksByFeedId.set(feedId, new Map());
75
+ }
76
+ feedBlocksByFeedId.get(feedId)!.set(chunkIndex, blocks);
77
+ }
78
+ }
79
+
80
+ for (const [feedId, feedMetadata] of feedMetadataByFeedId) {
81
+ const blockChunks = feedBlocksByFeedId.get(feedId) ?? new Map<number, FeedArchiveBlock[]>();
82
+ const sortedChunkIndices = Array.from(blockChunks.keys()).sort((a, b) => a - b);
83
+ const allBlocks: FeedArchiveBlock[] = [];
84
+ for (const chunkIndex of sortedChunkIndices) {
85
+ allBlocks.push(...blockChunks.get(chunkIndex)!);
86
+ }
87
+ feeds[feedId] = {
88
+ metadata: feedMetadata,
89
+ blocks: allBlocks,
90
+ };
91
+ }
92
+
93
+ log('extracted space archive', { metadata, documents, feedCount: Object.keys(feeds).length });
94
+ return { metadata, documents, feeds };
34
95
  };
@@ -7,8 +7,16 @@ import type * as tar from '@obsidize/tar-browserify';
7
7
  import { type Context, Resource } from '@dxos/context';
8
8
  import { assertArgument, assertState } from '@dxos/invariant';
9
9
  import type { IdentityDid, SpaceId } from '@dxos/keys';
10
- import { SpaceArchiveFileStructure, type SpaceArchiveMetadata, SpaceArchiveVersion } from '@dxos/protocols';
10
+ import {
11
+ FEED_ARCHIVE_BLOCKS_PER_CHUNK,
12
+ type FeedArchiveBlock,
13
+ type FeedArchiveMetadata,
14
+ SpaceArchiveFileStructure,
15
+ type SpaceArchiveMetadata,
16
+ SpaceArchiveVersion,
17
+ } from '@dxos/protocols';
11
18
  import type { SpaceArchive } from '@dxos/protocols/proto/dxos/client/services';
19
+ import { createFilename } from '@dxos/util';
12
20
 
13
21
  export type SpaceArchiveBeginProps = {
14
22
  spaceId?: SpaceId;
@@ -56,9 +64,38 @@ export class SpaceArchiveWriter extends Resource {
56
64
  this._archive.addBinaryFile(`${SpaceArchiveFileStructure.documents}/${documentId}.bin`, data);
57
65
  }
58
66
 
67
+ /**
68
+ * Writes a feed with its metadata and blocks to the archive.
69
+ * Blocks are written in chunks of {@link FEED_ARCHIVE_BLOCKS_PER_CHUNK}.
70
+ */
71
+ async writeFeed(feedId: string, namespace: string, blocks: FeedArchiveBlock[]): Promise<void> {
72
+ assertArgument(feedId, 'feedId', 'Feed ID is required');
73
+ assertArgument(namespace, 'namespace', 'Namespace is required');
74
+ assertState(this._archive, 'Not open');
75
+
76
+ const feedPath = `${SpaceArchiveFileStructure.feeds}/${feedId}`;
77
+
78
+ const metadata: FeedArchiveMetadata = {
79
+ id: feedId,
80
+ namespace,
81
+ };
82
+ this._archive.addTextFile(`${feedPath}/${SpaceArchiveFileStructure.feedMetadata}`, JSON.stringify(metadata));
83
+
84
+ for (let chunkIndex = 0; chunkIndex * FEED_ARCHIVE_BLOCKS_PER_CHUNK < blocks.length; chunkIndex++) {
85
+ const start = chunkIndex * FEED_ARCHIVE_BLOCKS_PER_CHUNK;
86
+ const end = Math.min(start + FEED_ARCHIVE_BLOCKS_PER_CHUNK, blocks.length);
87
+ const chunk = blocks.slice(start, end);
88
+ this._archive.addTextFile(
89
+ `${feedPath}/${SpaceArchiveFileStructure.feedBlocksPrefix}${chunkIndex}.json`,
90
+ JSON.stringify(chunk),
91
+ );
92
+ }
93
+ }
94
+
59
95
  async finish(): Promise<SpaceArchive> {
60
96
  assertState(this._archive, 'Not open');
61
97
  assertState(this._meta, 'Not started');
98
+ assertState(this._meta.spaceId, 'No space ID set');
62
99
  assertState(this._currentRootUrl, 'No root URL set');
63
100
 
64
101
  const metadata: SpaceArchiveMetadata = {
@@ -76,7 +113,7 @@ export class SpaceArchiveWriter extends Resource {
76
113
  const binary = this._archive.toUint8Array();
77
114
 
78
115
  return {
79
- filename: `${new Date().toISOString()}-${this._meta.spaceId}.tar`,
116
+ filename: createFilename({ parts: [this._meta.spaceId], ext: 'tar' }),
80
117
  contents: binary,
81
118
  };
82
119
  }
@@ -0,0 +1,287 @@
1
+ //
2
+ // Copyright 2025 DXOS.org
3
+ //
4
+
5
+ import type { DocumentId } from '@automerge/automerge-repo';
6
+ import { describe, expect, test } from 'vitest';
7
+
8
+ import { SpaceId } from '@dxos/keys';
9
+ import {
10
+ FEED_ARCHIVE_BLOCKS_PER_CHUNK,
11
+ type FeedArchiveBlock,
12
+ SpaceArchiveFileStructure,
13
+ SpaceArchiveVersion,
14
+ } from '@dxos/protocols';
15
+
16
+ import { extractSpaceArchive } from './space-archive-reader';
17
+ import { SpaceArchiveWriter } from './space-archive-writer';
18
+
19
+ describe('SpaceArchive', () => {
20
+ describe('SpaceArchiveWriter', () => {
21
+ test('writes and reads documents', async () => {
22
+ const writer = new SpaceArchiveWriter();
23
+ await writer.open();
24
+ try {
25
+ const spaceId = SpaceId.random();
26
+ await writer.begin({ spaceId });
27
+ await writer.setCurrentRootUrl('automerge:test123');
28
+ await writer.writeDocument('doc1', new Uint8Array([1, 2, 3]));
29
+ await writer.writeDocument('doc2', new Uint8Array([4, 5, 6]));
30
+
31
+ const archive = await writer.finish();
32
+
33
+ expect(archive.filename).toContain(spaceId);
34
+ expect(archive.contents).toBeInstanceOf(Uint8Array);
35
+
36
+ const extracted = await extractSpaceArchive(archive);
37
+ expect(extracted.metadata.version).toBe(SpaceArchiveVersion.V1);
38
+ expect(extracted.metadata.originalSpaceId).toBe(spaceId);
39
+ expect(extracted.metadata.echo?.currentRootUrl).toBe('automerge:test123');
40
+ expect(extracted.documents['doc1' as DocumentId]).toEqual(new Uint8Array([1, 2, 3]));
41
+ expect(extracted.documents['doc2' as DocumentId]).toEqual(new Uint8Array([4, 5, 6]));
42
+ } finally {
43
+ await writer.close();
44
+ }
45
+ });
46
+ });
47
+
48
+ describe('Feed Archive', () => {
49
+ test('writes and reads a single feed with blocks', async () => {
50
+ const writer = new SpaceArchiveWriter();
51
+ await writer.open();
52
+ try {
53
+ const spaceId = SpaceId.random();
54
+ await writer.begin({ spaceId });
55
+ await writer.setCurrentRootUrl('automerge:root');
56
+
57
+ const blocks: FeedArchiveBlock[] = [
58
+ {
59
+ actorId: 'actor1',
60
+ sequence: 0,
61
+ prevActorId: null,
62
+ prevSequence: null,
63
+ position: 0,
64
+ timestamp: 1000,
65
+ data: btoa('block0'),
66
+ },
67
+ {
68
+ actorId: 'actor1',
69
+ sequence: 1,
70
+ prevActorId: 'actor1',
71
+ prevSequence: 0,
72
+ position: 1,
73
+ timestamp: 2000,
74
+ data: btoa('block1'),
75
+ },
76
+ ];
77
+
78
+ await writer.writeFeed('feed-123', 'data', blocks);
79
+
80
+ const archive = await writer.finish();
81
+ const extracted = await extractSpaceArchive(archive);
82
+
83
+ expect(Object.keys(extracted.feeds)).toHaveLength(1);
84
+ expect(extracted.feeds['feed-123']).toBeDefined();
85
+ expect(extracted.feeds['feed-123'].metadata.id).toBe('feed-123');
86
+ expect(extracted.feeds['feed-123'].metadata.namespace).toBe('data');
87
+ expect(extracted.feeds['feed-123'].blocks).toHaveLength(2);
88
+ expect(extracted.feeds['feed-123'].blocks[0].actorId).toBe('actor1');
89
+ expect(extracted.feeds['feed-123'].blocks[0].sequence).toBe(0);
90
+ expect(extracted.feeds['feed-123'].blocks[1].sequence).toBe(1);
91
+ } finally {
92
+ await writer.close();
93
+ }
94
+ });
95
+
96
+ test('writes and reads multiple feeds', async () => {
97
+ const writer = new SpaceArchiveWriter();
98
+ await writer.open();
99
+ try {
100
+ const spaceId = SpaceId.random();
101
+ await writer.begin({ spaceId });
102
+ await writer.setCurrentRootUrl('automerge:root');
103
+
104
+ const dataBlocks: FeedArchiveBlock[] = [
105
+ {
106
+ actorId: 'actor1',
107
+ sequence: 0,
108
+ prevActorId: null,
109
+ prevSequence: null,
110
+ position: 0,
111
+ timestamp: 1000,
112
+ data: btoa('data-block'),
113
+ },
114
+ ];
115
+
116
+ const traceBlocks: FeedArchiveBlock[] = [
117
+ {
118
+ actorId: 'actor2',
119
+ sequence: 0,
120
+ prevActorId: null,
121
+ prevSequence: null,
122
+ position: 0,
123
+ timestamp: 2000,
124
+ data: btoa('trace-block'),
125
+ },
126
+ ];
127
+
128
+ await writer.writeFeed('feed-data', 'data', dataBlocks);
129
+ await writer.writeFeed('feed-trace', 'trace', traceBlocks);
130
+
131
+ const archive = await writer.finish();
132
+ const extracted = await extractSpaceArchive(archive);
133
+
134
+ expect(Object.keys(extracted.feeds)).toHaveLength(2);
135
+ expect(extracted.feeds['feed-data'].metadata.namespace).toBe('data');
136
+ expect(extracted.feeds['feed-trace'].metadata.namespace).toBe('trace');
137
+ } finally {
138
+ await writer.close();
139
+ }
140
+ });
141
+
142
+ test('writes blocks in chunks when exceeding chunk size', async () => {
143
+ const writer = new SpaceArchiveWriter();
144
+ await writer.open();
145
+ try {
146
+ const spaceId = SpaceId.random();
147
+ await writer.begin({ spaceId });
148
+ await writer.setCurrentRootUrl('automerge:root');
149
+
150
+ const numBlocks = FEED_ARCHIVE_BLOCKS_PER_CHUNK + 50;
151
+ const blocks: FeedArchiveBlock[] = [];
152
+ for (let i = 0; i < numBlocks; i++) {
153
+ blocks.push({
154
+ actorId: 'actor1',
155
+ sequence: i,
156
+ prevActorId: i > 0 ? 'actor1' : null,
157
+ prevSequence: i > 0 ? i - 1 : null,
158
+ position: i,
159
+ timestamp: 1000 + i,
160
+ data: btoa(`block-${i}`),
161
+ });
162
+ }
163
+
164
+ await writer.writeFeed('large-feed', 'data', blocks);
165
+
166
+ const archive = await writer.finish();
167
+ const extracted = await extractSpaceArchive(archive);
168
+
169
+ expect(extracted.feeds['large-feed'].blocks).toHaveLength(numBlocks);
170
+ expect(extracted.feeds['large-feed'].blocks[0].sequence).toBe(0);
171
+ expect(extracted.feeds['large-feed'].blocks[numBlocks - 1].sequence).toBe(numBlocks - 1);
172
+ } finally {
173
+ await writer.close();
174
+ }
175
+ });
176
+
177
+ test('handles empty feeds', async () => {
178
+ const writer = new SpaceArchiveWriter();
179
+ await writer.open();
180
+ try {
181
+ const spaceId = SpaceId.random();
182
+ await writer.begin({ spaceId });
183
+ await writer.setCurrentRootUrl('automerge:root');
184
+
185
+ await writer.writeFeed('empty-feed', 'data', []);
186
+
187
+ const archive = await writer.finish();
188
+ const extracted = await extractSpaceArchive(archive);
189
+
190
+ expect(extracted.feeds['empty-feed']).toBeDefined();
191
+ expect(extracted.feeds['empty-feed'].metadata.id).toBe('empty-feed');
192
+ expect(extracted.feeds['empty-feed'].blocks).toHaveLength(0);
193
+ } finally {
194
+ await writer.close();
195
+ }
196
+ });
197
+
198
+ test('preserves block data encoding', async () => {
199
+ const writer = new SpaceArchiveWriter();
200
+ await writer.open();
201
+ try {
202
+ const spaceId = SpaceId.random();
203
+ await writer.begin({ spaceId });
204
+ await writer.setCurrentRootUrl('automerge:root');
205
+
206
+ const originalData = new Uint8Array([0, 1, 2, 255, 128, 64]);
207
+ const base64Data = btoa(String.fromCharCode(...originalData));
208
+
209
+ const blocks: FeedArchiveBlock[] = [
210
+ {
211
+ actorId: 'actor1',
212
+ sequence: 0,
213
+ prevActorId: null,
214
+ prevSequence: null,
215
+ position: null,
216
+ timestamp: 1000,
217
+ data: base64Data,
218
+ },
219
+ ];
220
+
221
+ await writer.writeFeed('binary-feed', 'data', blocks);
222
+
223
+ const archive = await writer.finish();
224
+ const extracted = await extractSpaceArchive(archive);
225
+
226
+ const extractedData = extracted.feeds['binary-feed'].blocks[0].data;
227
+ expect(extractedData).toBe(base64Data);
228
+
229
+ const decoded = new Uint8Array(
230
+ atob(extractedData)
231
+ .split('')
232
+ .map((char) => char.charCodeAt(0)),
233
+ );
234
+ expect(decoded).toEqual(originalData);
235
+ } finally {
236
+ await writer.close();
237
+ }
238
+ });
239
+
240
+ test('combines documents and feeds in archive', async () => {
241
+ const writer = new SpaceArchiveWriter();
242
+ await writer.open();
243
+ try {
244
+ const spaceId = SpaceId.random();
245
+ await writer.begin({ spaceId });
246
+ await writer.setCurrentRootUrl('automerge:root');
247
+
248
+ await writer.writeDocument('doc1', new Uint8Array([1, 2, 3]));
249
+ await writer.writeFeed('feed1', 'data', [
250
+ {
251
+ actorId: 'actor1',
252
+ sequence: 0,
253
+ prevActorId: null,
254
+ prevSequence: null,
255
+ position: 0,
256
+ timestamp: 1000,
257
+ data: btoa('test'),
258
+ },
259
+ ]);
260
+
261
+ const archive = await writer.finish();
262
+ const extracted = await extractSpaceArchive(archive);
263
+
264
+ expect(Object.keys(extracted.documents)).toHaveLength(1);
265
+ expect(Object.keys(extracted.feeds)).toHaveLength(1);
266
+ expect(extracted.documents['doc1' as DocumentId]).toEqual(new Uint8Array([1, 2, 3]));
267
+ expect(extracted.feeds['feed1'].blocks).toHaveLength(1);
268
+ } finally {
269
+ await writer.close();
270
+ }
271
+ });
272
+ });
273
+
274
+ describe('File Structure', () => {
275
+ test('file structure constants are correct', () => {
276
+ expect(SpaceArchiveFileStructure.metadata).toBe('metadata.json');
277
+ expect(SpaceArchiveFileStructure.documents).toBe('documents');
278
+ expect(SpaceArchiveFileStructure.feeds).toBe('feeds');
279
+ expect(SpaceArchiveFileStructure.feedMetadata).toBe('metadata.json');
280
+ expect(SpaceArchiveFileStructure.feedBlocksPrefix).toBe('blocks-');
281
+ });
282
+
283
+ test('blocks per chunk constant is set', () => {
284
+ expect(FEED_ARCHIVE_BLOCKS_PER_CHUNK).toBe(100);
285
+ });
286
+ });
287
+ });