@dxos/client-services 0.8.4-main.3eb6e50203 → 0.8.4-main.3fbcb4aa9b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/dist/lib/browser/{chunk-675YEKTQ.mjs → chunk-OBWND4JJ.mjs} +1701 -3078
  2. package/dist/lib/browser/chunk-OBWND4JJ.mjs.map +7 -0
  3. package/dist/lib/browser/{chunk-NQSC7HOE.mjs → chunk-XJRPB3GA.mjs} +1 -1
  4. package/dist/lib/browser/index.mjs +106 -194
  5. package/dist/lib/browser/index.mjs.map +3 -3
  6. package/dist/lib/browser/meta.json +1 -1
  7. package/dist/lib/browser/packlets/diagnostics/browser-diagnostics-broadcast.mjs +1 -6
  8. package/dist/lib/browser/packlets/diagnostics/browser-diagnostics-broadcast.mjs.map +2 -2
  9. package/dist/lib/browser/packlets/diagnostics/diagnostics-broadcast.mjs +1 -1
  10. package/dist/lib/browser/packlets/locks/browser.mjs +9 -49
  11. package/dist/lib/browser/packlets/locks/browser.mjs.map +2 -2
  12. package/dist/lib/browser/packlets/locks/node.mjs +4 -22
  13. package/dist/lib/browser/packlets/locks/node.mjs.map +2 -2
  14. package/dist/lib/browser/testing/index.mjs +9 -22
  15. package/dist/lib/browser/testing/index.mjs.map +3 -3
  16. package/dist/lib/node-esm/{chunk-PKEGMOQ4.mjs → chunk-2DT3MZRL.mjs} +1 -1
  17. package/dist/lib/node-esm/{chunk-RUUAV2BD.mjs → chunk-PB2SRGWV.mjs} +1701 -3078
  18. package/dist/lib/node-esm/chunk-PB2SRGWV.mjs.map +7 -0
  19. package/dist/lib/node-esm/index.mjs +106 -194
  20. package/dist/lib/node-esm/index.mjs.map +3 -3
  21. package/dist/lib/node-esm/meta.json +1 -1
  22. package/dist/lib/node-esm/packlets/diagnostics/browser-diagnostics-broadcast.mjs +1 -6
  23. package/dist/lib/node-esm/packlets/diagnostics/browser-diagnostics-broadcast.mjs.map +2 -2
  24. package/dist/lib/node-esm/packlets/diagnostics/diagnostics-broadcast.mjs +1 -1
  25. package/dist/lib/node-esm/packlets/locks/browser.mjs +9 -49
  26. package/dist/lib/node-esm/packlets/locks/browser.mjs.map +2 -2
  27. package/dist/lib/node-esm/packlets/locks/node.mjs +4 -22
  28. package/dist/lib/node-esm/packlets/locks/node.mjs.map +2 -2
  29. package/dist/lib/node-esm/testing/index.mjs +9 -22
  30. package/dist/lib/node-esm/testing/index.mjs.map +3 -3
  31. package/dist/types/src/index.d.ts +1 -0
  32. package/dist/types/src/index.d.ts.map +1 -1
  33. package/dist/types/src/packlets/agents/edge-agent-manager.d.ts +3 -2
  34. package/dist/types/src/packlets/agents/edge-agent-manager.d.ts.map +1 -1
  35. package/dist/types/src/packlets/agents/edge-agent-service.d.ts +2 -1
  36. package/dist/types/src/packlets/agents/edge-agent-service.d.ts.map +1 -1
  37. package/dist/types/src/packlets/devices/devices-service.d.ts.map +1 -1
  38. package/dist/types/src/packlets/devtools/devtools.d.ts.map +1 -1
  39. package/dist/types/src/packlets/devtools/feeds.d.ts.map +1 -1
  40. package/dist/types/src/packlets/devtools/keys.d.ts.map +1 -1
  41. package/dist/types/src/packlets/devtools/metadata.d.ts.map +1 -1
  42. package/dist/types/src/packlets/devtools/network.d.ts.map +1 -1
  43. package/dist/types/src/packlets/devtools/spaces.d.ts.map +1 -1
  44. package/dist/types/src/packlets/diagnostics/browser-diagnostics-broadcast.d.ts.map +1 -1
  45. package/dist/types/src/packlets/diagnostics/diagnostics-broadcast.d.ts.map +1 -1
  46. package/dist/types/src/packlets/diagnostics/diagnostics-collector.d.ts.map +1 -1
  47. package/dist/types/src/packlets/diagnostics/diagnostics.d.ts +2 -3
  48. package/dist/types/src/packlets/diagnostics/diagnostics.d.ts.map +1 -1
  49. package/dist/types/src/packlets/identity/authenticator.d.ts.map +1 -1
  50. package/dist/types/src/packlets/identity/contacts-service.d.ts.map +1 -1
  51. package/dist/types/src/packlets/identity/identity-manager.d.ts +3 -3
  52. package/dist/types/src/packlets/identity/identity-manager.d.ts.map +1 -1
  53. package/dist/types/src/packlets/identity/identity-recovery-manager.d.ts +6 -5
  54. package/dist/types/src/packlets/identity/identity-recovery-manager.d.ts.map +1 -1
  55. package/dist/types/src/packlets/identity/identity-service.d.ts +6 -10
  56. package/dist/types/src/packlets/identity/identity-service.d.ts.map +1 -1
  57. package/dist/types/src/packlets/identity/identity.d.ts +6 -9
  58. package/dist/types/src/packlets/identity/identity.d.ts.map +1 -1
  59. package/dist/types/src/packlets/invitations/device-invitation-protocol.d.ts +2 -1
  60. package/dist/types/src/packlets/invitations/device-invitation-protocol.d.ts.map +1 -1
  61. package/dist/types/src/packlets/invitations/edge-invitation-handler.d.ts +1 -1
  62. package/dist/types/src/packlets/invitations/edge-invitation-handler.d.ts.map +1 -1
  63. package/dist/types/src/packlets/invitations/invitation-guest-extenstion.d.ts.map +1 -1
  64. package/dist/types/src/packlets/invitations/invitation-host-extension.d.ts.map +1 -1
  65. package/dist/types/src/packlets/invitations/invitation-protocol.d.ts +5 -1
  66. package/dist/types/src/packlets/invitations/invitation-protocol.d.ts.map +1 -1
  67. package/dist/types/src/packlets/invitations/invitation-state.d.ts.map +1 -1
  68. package/dist/types/src/packlets/invitations/invitation-topology.d.ts.map +1 -1
  69. package/dist/types/src/packlets/invitations/invitations-handler.d.ts.map +1 -1
  70. package/dist/types/src/packlets/invitations/invitations-manager.d.ts +3 -3
  71. package/dist/types/src/packlets/invitations/invitations-manager.d.ts.map +1 -1
  72. package/dist/types/src/packlets/invitations/invitations-service.d.ts +3 -3
  73. package/dist/types/src/packlets/invitations/invitations-service.d.ts.map +1 -1
  74. package/dist/types/src/packlets/invitations/space-invitation-protocol.d.ts +2 -1
  75. package/dist/types/src/packlets/invitations/space-invitation-protocol.d.ts.map +1 -1
  76. package/dist/types/src/packlets/invitations/utils.d.ts.map +1 -1
  77. package/dist/types/src/packlets/locks/browser.d.ts.map +1 -1
  78. package/dist/types/src/packlets/locks/node.d.ts.map +1 -1
  79. package/dist/types/src/packlets/logging/logging-service.d.ts.map +1 -1
  80. package/dist/types/src/packlets/network/network-service.d.ts +5 -4
  81. package/dist/types/src/packlets/network/network-service.d.ts.map +1 -1
  82. package/dist/types/src/packlets/services/client-rpc-server.d.ts +3 -3
  83. package/dist/types/src/packlets/services/client-rpc-server.d.ts.map +1 -1
  84. package/dist/types/src/packlets/services/feed-syncer.d.ts +22 -4
  85. package/dist/types/src/packlets/services/feed-syncer.d.ts.map +1 -1
  86. package/dist/types/src/packlets/services/feed-syncer.test.d.ts +2 -0
  87. package/dist/types/src/packlets/services/feed-syncer.test.d.ts.map +1 -0
  88. package/dist/types/src/packlets/services/service-context.d.ts +2 -4
  89. package/dist/types/src/packlets/services/service-context.d.ts.map +1 -1
  90. package/dist/types/src/packlets/services/service-host.d.ts +2 -3
  91. package/dist/types/src/packlets/services/service-host.d.ts.map +1 -1
  92. package/dist/types/src/packlets/services/service-registry.d.ts.map +1 -1
  93. package/dist/types/src/packlets/services/util.d.ts.map +1 -1
  94. package/dist/types/src/packlets/space-export/archive-format.d.ts +9 -0
  95. package/dist/types/src/packlets/space-export/archive-format.d.ts.map +1 -0
  96. package/dist/types/src/packlets/space-export/index.d.ts +4 -1
  97. package/dist/types/src/packlets/space-export/index.d.ts.map +1 -1
  98. package/dist/types/src/packlets/space-export/serialized-space-reader.d.ts +23 -0
  99. package/dist/types/src/packlets/space-export/serialized-space-reader.d.ts.map +1 -0
  100. package/dist/types/src/packlets/space-export/serialized-space-writer.d.ts +36 -0
  101. package/dist/types/src/packlets/space-export/serialized-space-writer.d.ts.map +1 -0
  102. package/dist/types/src/packlets/space-export/space-archive-reader.d.ts +9 -1
  103. package/dist/types/src/packlets/space-export/space-archive-reader.d.ts.map +1 -1
  104. package/dist/types/src/packlets/space-export/space-archive-writer.d.ts +7 -1
  105. package/dist/types/src/packlets/space-export/space-archive-writer.d.ts.map +1 -1
  106. package/dist/types/src/packlets/space-export/space-archive.test.d.ts +2 -0
  107. package/dist/types/src/packlets/space-export/space-archive.test.d.ts.map +1 -0
  108. package/dist/types/src/packlets/spaces/automerge-space-state.d.ts.map +1 -1
  109. package/dist/types/src/packlets/spaces/data-space-manager.d.ts +18 -12
  110. package/dist/types/src/packlets/spaces/data-space-manager.d.ts.map +1 -1
  111. package/dist/types/src/packlets/spaces/data-space.d.ts +24 -7
  112. package/dist/types/src/packlets/spaces/data-space.d.ts.map +1 -1
  113. package/dist/types/src/packlets/spaces/edge-feed-replicator.d.ts.map +1 -1
  114. package/dist/types/src/packlets/spaces/epoch-migrations.d.ts.map +1 -1
  115. package/dist/types/src/packlets/spaces/genesis.d.ts +2 -1
  116. package/dist/types/src/packlets/spaces/genesis.d.ts.map +1 -1
  117. package/dist/types/src/packlets/spaces/notarization-plugin.d.ts +1 -4
  118. package/dist/types/src/packlets/spaces/notarization-plugin.d.ts.map +1 -1
  119. package/dist/types/src/packlets/spaces/spaces-service.d.ts +10 -7
  120. package/dist/types/src/packlets/spaces/spaces-service.d.ts.map +1 -1
  121. package/dist/types/src/packlets/storage/level.d.ts.map +1 -1
  122. package/dist/types/src/packlets/storage/profile-archive.d.ts.map +1 -1
  123. package/dist/types/src/packlets/storage/storage.d.ts.map +1 -1
  124. package/dist/types/src/packlets/storage/util.d.ts.map +1 -1
  125. package/dist/types/src/packlets/system/system-service.d.ts +1 -1
  126. package/dist/types/src/packlets/system/system-service.d.ts.map +1 -1
  127. package/dist/types/src/packlets/testing/credential-utils.d.ts.map +1 -1
  128. package/dist/types/src/packlets/testing/invitation-utils.d.ts.map +1 -1
  129. package/dist/types/src/packlets/testing/test-builder.d.ts.map +1 -1
  130. package/dist/types/src/packlets/worker/worker-runtime.d.ts +11 -1
  131. package/dist/types/src/packlets/worker/worker-runtime.d.ts.map +1 -1
  132. package/dist/types/src/packlets/worker/worker-session.d.ts +0 -2
  133. package/dist/types/src/packlets/worker/worker-session.d.ts.map +1 -1
  134. package/dist/types/src/testing/setup.d.ts.map +1 -1
  135. package/dist/types/src/version.d.ts +1 -1
  136. package/dist/types/tsconfig.tsbuildinfo +1 -1
  137. package/package.json +42 -49
  138. package/src/index.ts +1 -0
  139. package/src/packlets/agents/edge-agent-manager.ts +8 -5
  140. package/src/packlets/agents/edge-agent-service.ts +4 -2
  141. package/src/packlets/devices/devices-service.test.ts +0 -1
  142. package/src/packlets/devtools/devtools.ts +0 -1
  143. package/src/packlets/diagnostics/diagnostics.ts +1 -2
  144. package/src/packlets/identity/contacts-service.ts +0 -1
  145. package/src/packlets/identity/identity-manager.test.ts +5 -5
  146. package/src/packlets/identity/identity-manager.ts +18 -17
  147. package/src/packlets/identity/identity-recovery-manager.ts +20 -16
  148. package/src/packlets/identity/identity-service.test.ts +6 -27
  149. package/src/packlets/identity/identity-service.ts +13 -81
  150. package/src/packlets/identity/identity.test.ts +2 -2
  151. package/src/packlets/identity/identity.ts +9 -32
  152. package/src/packlets/invitations/device-invitation-protocol.ts +3 -2
  153. package/src/packlets/invitations/edge-invitation-handler.ts +9 -5
  154. package/src/packlets/invitations/invitation-host-extension.ts +7 -10
  155. package/src/packlets/invitations/invitation-protocol.ts +5 -1
  156. package/src/packlets/invitations/invitation-state.ts +1 -15
  157. package/src/packlets/invitations/invitations-handler.test.ts +4 -5
  158. package/src/packlets/invitations/invitations-handler.ts +67 -15
  159. package/src/packlets/invitations/invitations-manager.ts +40 -15
  160. package/src/packlets/invitations/invitations-service.ts +9 -9
  161. package/src/packlets/invitations/space-invitation-protocol.test.ts +17 -16
  162. package/src/packlets/invitations/space-invitation-protocol.ts +4 -3
  163. package/src/packlets/logging/logging-service.ts +15 -15
  164. package/src/packlets/network/network-service.test.ts +0 -1
  165. package/src/packlets/network/network-service.ts +10 -8
  166. package/src/packlets/services/client-rpc-server.ts +15 -12
  167. package/src/packlets/services/feed-syncer.test.ts +340 -0
  168. package/src/packlets/services/feed-syncer.ts +175 -25
  169. package/src/packlets/services/service-context.test.ts +3 -2
  170. package/src/packlets/services/service-context.ts +98 -41
  171. package/src/packlets/services/service-host.test.ts +8 -8
  172. package/src/packlets/services/service-host.ts +17 -27
  173. package/src/packlets/services/service-registry.test.ts +0 -1
  174. package/src/packlets/space-export/archive-format.ts +42 -0
  175. package/src/packlets/space-export/index.ts +4 -1
  176. package/src/packlets/space-export/serialized-space-reader.ts +111 -0
  177. package/src/packlets/space-export/serialized-space-writer.ts +252 -0
  178. package/src/packlets/space-export/space-archive-reader.ts +64 -3
  179. package/src/packlets/space-export/space-archive-writer.ts +38 -2
  180. package/src/packlets/space-export/space-archive.test.ts +461 -0
  181. package/src/packlets/spaces/data-space-manager.test.ts +79 -13
  182. package/src/packlets/spaces/data-space-manager.ts +75 -111
  183. package/src/packlets/spaces/data-space.ts +49 -29
  184. package/src/packlets/spaces/edge-feed-replicator.test.ts +1 -1
  185. package/src/packlets/spaces/edge-feed-replicator.ts +9 -7
  186. package/src/packlets/spaces/epoch-migrations.ts +3 -3
  187. package/src/packlets/spaces/genesis.ts +6 -1
  188. package/src/packlets/spaces/notarization-plugin.ts +2 -1
  189. package/src/packlets/spaces/spaces-service.test.ts +18 -11
  190. package/src/packlets/spaces/spaces-service.ts +113 -17
  191. package/src/packlets/testing/invitation-utils.ts +3 -2
  192. package/src/packlets/worker/worker-runtime.ts +39 -6
  193. package/src/packlets/worker/worker-session.ts +8 -14
  194. package/src/version.ts +1 -1
  195. package/dist/lib/browser/chunk-675YEKTQ.mjs.map +0 -7
  196. package/dist/lib/node-esm/chunk-RUUAV2BD.mjs.map +0 -7
  197. package/dist/types/src/packlets/identity/default-space-state-machine.d.ts +0 -19
  198. package/dist/types/src/packlets/identity/default-space-state-machine.d.ts.map +0 -1
  199. package/src/packlets/identity/default-space-state-machine.ts +0 -44
  200. /package/dist/lib/browser/{chunk-NQSC7HOE.mjs.map → chunk-XJRPB3GA.mjs.map} +0 -0
  201. /package/dist/lib/node-esm/{chunk-PKEGMOQ4.mjs.map → chunk-2DT3MZRL.mjs.map} +0 -0
@@ -8,11 +8,13 @@ import * as Effect from 'effect/Effect';
8
8
  import * as Schema from 'effect/Schema';
9
9
 
10
10
  import { AsyncTask, scheduleTask } from '@dxos/async';
11
- import { Resource } from '@dxos/context';
11
+ import { Context, Resource } from '@dxos/context';
12
12
  import { type EdgeConnection, MessageSchema } from '@dxos/edge-client';
13
13
  import { RuntimeProvider } from '@dxos/effect';
14
14
  import { type FeedStore, SyncClient } from '@dxos/feed';
15
- import { type SpaceId } from '@dxos/keys';
15
+ import { invariant } from '@dxos/invariant';
16
+ import { SpaceId } from '@dxos/keys';
17
+ import { log } from '@dxos/log';
16
18
  import { FeedProtocol } from '@dxos/protocols';
17
19
  import { EdgeService } from '@dxos/protocols';
18
20
  import { createBuf } from '@dxos/protocols/buf';
@@ -24,7 +26,9 @@ const encoder = new Encoder({ tagUint8Array: false, useRecords: false });
24
26
 
25
27
  const DEFAULT_MESSAGE_BLOCKS_LIMIT = 50;
26
28
  const DEFAULT_SYNC_CONCURRENCY = 5;
27
- const DEFAULT_POLLING_INTERVAL = 10_000;
29
+ const DEFAULT_POLLING_INTERVAL = 5_000;
30
+ const DEFAULT_POLL_REQUEST_THROTTLE_MS = 250;
31
+ const MAX_BLOCKING_SYNC_ITERATIONS = 100;
28
32
 
29
33
  interface FeedSyncerOptions {
30
34
  runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransaction.SqlTransaction>;
@@ -34,10 +38,9 @@ interface FeedSyncerOptions {
34
38
  getSpaceIds: () => SpaceId[];
35
39
 
36
40
  /**
37
- * Namespace to sync.
41
+ * Namespaces to sync.
38
42
  */
39
- // TODO(dmaretskyi): Syncing only one namespace is supported.
40
- syncNamespace: string;
43
+ syncNamespaces: string[];
41
44
 
42
45
  /**
43
46
  * Maximum number of blocks to sync in a single message.
@@ -56,13 +59,20 @@ interface FeedSyncerOptions {
56
59
  * @default 10 seconds
57
60
  */
58
61
  pollingInterval?: number;
62
+
63
+ /**
64
+ * Minimum delay between externally requested best-effort polls.
65
+ * @default 250 ms
66
+ */
67
+ pollRequestThrottleMs?: number;
59
68
  }
60
69
 
61
70
  export class FeedSyncer extends Resource {
62
- readonly #syncNamespace: string;
71
+ readonly #syncNamespaces: string[];
63
72
  readonly #messageBlocksLimit: number;
64
73
  readonly #syncConcurrency: number;
65
74
  readonly #pollingInterval: number;
75
+ readonly #pollRequestThrottleMs: number;
66
76
 
67
77
  readonly #runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransaction.SqlTransaction>;
68
78
  readonly #feedStore: FeedStore;
@@ -73,6 +83,8 @@ export class FeedSyncer extends Resource {
73
83
  #spacesToPoll = new Set<SpaceId>();
74
84
  /** Last time full poll was completed. */
75
85
  #lastFullPoll: number | null = null;
86
+ #throttledPollScheduled = false;
87
+ #lastRequestedPollAt: number | null = null;
76
88
 
77
89
  constructor(options: FeedSyncerOptions) {
78
90
  super();
@@ -85,10 +97,11 @@ export class FeedSyncer extends Resource {
85
97
  sendMessage: this.#sendMessage.bind(this),
86
98
  });
87
99
  this.#getSpaceIds = options.getSpaceIds;
88
- this.#syncNamespace = options.syncNamespace;
100
+ this.#syncNamespaces = options.syncNamespaces;
89
101
  this.#messageBlocksLimit = options.messageBlocksLimit ?? DEFAULT_MESSAGE_BLOCKS_LIMIT;
90
102
  this.#syncConcurrency = options.syncConcurrency ?? DEFAULT_SYNC_CONCURRENCY;
91
103
  this.#pollingInterval = options.pollingInterval ?? DEFAULT_POLLING_INTERVAL;
104
+ this.#pollRequestThrottleMs = options.pollRequestThrottleMs ?? DEFAULT_POLL_REQUEST_THROTTLE_MS;
92
105
  }
93
106
 
94
107
  protected override async _open(): Promise<void> {
@@ -101,14 +114,28 @@ export class FeedSyncer extends Resource {
101
114
  if (service !== EdgeService.QUEUE_REPLICATOR) {
102
115
  return;
103
116
  }
117
+ log('feed sync edge ingress', {
118
+ serviceId: msg.serviceId,
119
+ payloadByteLength: msg.payload?.value?.byteLength,
120
+ });
104
121
  const handleMessageEffect = Effect.gen(this, function* () {
105
122
  const decoded = yield* Effect.try({
106
123
  try: () => cborXdecode(msg.payload!.value),
107
124
  catch: (error) => new Error(`Failed to decode feed sync message: ${error}`),
108
125
  });
109
- const payload = yield* Schema.decodeUnknown(FeedProtocol.ProtocolMessage)(decoded);
126
+ const payload = yield* Schema.validate(FeedProtocol.ProtocolMessage)(decoded);
110
127
  yield* this.#syncClient.handleMessage(payload);
111
- });
128
+ }).pipe(
129
+ Effect.tapError((cause) =>
130
+ Effect.sync(() =>
131
+ log('feed sync edge message handling failed', {
132
+ serviceId: msg.serviceId,
133
+ payloadByteLength: msg.payload?.value?.byteLength,
134
+ cause: cause instanceof Error ? cause.message : String(cause),
135
+ }),
136
+ ),
137
+ ),
138
+ );
112
139
 
113
140
  void RuntimeProvider.runPromise(this.#runtime)(handleMessageEffect);
114
141
  }),
@@ -116,7 +143,12 @@ export class FeedSyncer extends Resource {
116
143
 
117
144
  this._ctx.onDispose(
118
145
  // NOTE: This will fire immediately if the connection is already open.
119
- this.#edgeClient.onReconnected(async () => {}),
146
+ this.#edgeClient.onReconnected(async () => {
147
+ log('feed sync edge reconnected', {
148
+ peerKey: this.#edgeClient.peerKey,
149
+ identityKey: this.#edgeClient.identityKey,
150
+ });
151
+ }),
120
152
  );
121
153
 
122
154
  this.#feedStore.onNewBlocks.on(this._ctx, () => {
@@ -127,6 +159,7 @@ export class FeedSyncer extends Resource {
127
159
  await this.#pushTask.open();
128
160
 
129
161
  this.#resetSpacesToPoll();
162
+ this.#pollTask.schedule();
130
163
  }
131
164
 
132
165
  protected override async _close(): Promise<void> {
@@ -134,6 +167,87 @@ export class FeedSyncer extends Resource {
134
167
  await this.#pushTask.close();
135
168
  }
136
169
 
170
+ /**
171
+ * Schedules a best-effort pull without blocking the caller.
172
+ */
173
+ schedulePoll(): void {
174
+ this.#resetSpacesToPoll();
175
+ if (this.#throttledPollScheduled) {
176
+ return;
177
+ }
178
+
179
+ const now = Date.now();
180
+ const delay =
181
+ this.#lastRequestedPollAt == null
182
+ ? 0
183
+ : Math.max(this.#pollRequestThrottleMs - (now - this.#lastRequestedPollAt), 0);
184
+ this.#throttledPollScheduled = true;
185
+ scheduleTask(
186
+ this._ctx,
187
+ () => {
188
+ this.#throttledPollScheduled = false;
189
+ this.#lastRequestedPollAt = Date.now();
190
+ this.#pollTask.schedule();
191
+ },
192
+ delay,
193
+ );
194
+ }
195
+
196
+ /**
197
+ * Performs queue sync and blocks until there are no pending sync batches.
198
+ */
199
+ async syncBlocking(
200
+ ctx: Context,
201
+ {
202
+ spaceId,
203
+ subspaceTag,
204
+ shouldPush = true,
205
+ shouldPull = true,
206
+ }: {
207
+ spaceId: SpaceId;
208
+ subspaceTag: string;
209
+ shouldPush?: boolean;
210
+ shouldPull?: boolean;
211
+ },
212
+ ): Promise<void> {
213
+ invariant(SpaceId.isValid(spaceId));
214
+ invariant(FeedProtocol.isWellKnownNamespace(subspaceTag));
215
+ if (!shouldPush && !shouldPull) {
216
+ return;
217
+ }
218
+
219
+ await RuntimeProvider.runPromise(this.#runtime)(
220
+ Effect.gen(this, function* () {
221
+ let done = false;
222
+ let iterations = 0;
223
+ while (!done) {
224
+ done = true;
225
+ if (shouldPull) {
226
+ const pullResult = yield* this.#syncClient.pull(ctx, {
227
+ spaceId,
228
+ feedNamespace: subspaceTag,
229
+ limit: this.#messageBlocksLimit,
230
+ });
231
+ done &&= pullResult.done;
232
+ }
233
+
234
+ if (shouldPush) {
235
+ const pushResult = yield* this.#syncClient.push(ctx, {
236
+ spaceId,
237
+ feedNamespace: subspaceTag,
238
+ limit: this.#messageBlocksLimit,
239
+ });
240
+ done &&= pushResult.done;
241
+ }
242
+ iterations++;
243
+ if (iterations > MAX_BLOCKING_SYNC_ITERATIONS) {
244
+ throw new Error('Blocking sync exceeded max iterations.');
245
+ }
246
+ }
247
+ }),
248
+ );
249
+ }
250
+
137
251
  #resetSpacesToPoll(): void {
138
252
  this.#spacesToPoll.clear();
139
253
  this.#getSpaceIds().forEach((spaceId) => {
@@ -142,20 +256,44 @@ export class FeedSyncer extends Resource {
142
256
  this.#lastFullPoll = Date.now();
143
257
  }
144
258
 
145
- #sendMessage(message: FeedProtocol.QueryRequest | FeedProtocol.AppendRequest): Effect.Effect<void, unknown, never> {
259
+ #sendMessage(
260
+ ctx: Context,
261
+ message: FeedProtocol.QueryRequest | FeedProtocol.AppendRequest,
262
+ ): Effect.Effect<void, unknown, never> {
146
263
  return Effect.gen(this, function* () {
147
264
  const encoded = encoder.encode(message);
265
+ const serviceId = this.#getTargetServiceId(message);
266
+ const rpcTag = 'blocks' in message ? 'AppendRequest' : 'QueryRequest';
267
+ log('feed sync edge rpc outgoing', {
268
+ tag: rpcTag,
269
+ serviceId,
270
+ payloadByteLength: encoded.byteLength,
271
+ spaceId: message.spaceId,
272
+ feedNamespace: message.feedNamespace,
273
+ requestId: message.requestId,
274
+ });
148
275
  yield* Effect.tryPromise(async () =>
149
276
  this.#edgeClient.send(
277
+ ctx,
150
278
  createBuf(MessageSchema, {
151
279
  source: {
152
280
  identityKey: this.#edgeClient.identityKey,
153
281
  peerKey: this.#edgeClient.peerKey,
154
282
  },
155
- serviceId: this.#getTargetServiceId(message),
283
+ serviceId,
156
284
  payload: { value: bufferToArray(encoded) },
157
285
  }),
158
286
  ),
287
+ ).pipe(
288
+ Effect.tapError((cause) =>
289
+ Effect.sync(() =>
290
+ log('feed sync edge send failed', {
291
+ serviceId,
292
+ tag: rpcTag,
293
+ cause: cause instanceof Error ? cause.message : String(cause),
294
+ }),
295
+ ),
296
+ ),
159
297
  );
160
298
  });
161
299
  }
@@ -173,12 +311,18 @@ export class FeedSyncer extends Resource {
173
311
  this.#spacesToPoll,
174
312
  (spaceId) =>
175
313
  Effect.gen(this, function* () {
176
- const { done } = yield* this.#syncClient.pull({
177
- spaceId,
178
- feedNamespace: this.#syncNamespace,
179
- limit: this.#messageBlocksLimit,
180
- });
181
- if (done) {
314
+ let doneForAllNamespaces = true;
315
+ for (const feedNamespace of this.#syncNamespaces) {
316
+ const { done } = yield* this.#syncClient.pull(this._ctx, {
317
+ spaceId,
318
+ feedNamespace,
319
+ limit: this.#messageBlocksLimit,
320
+ });
321
+ if (!done) {
322
+ doneForAllNamespaces = false;
323
+ }
324
+ }
325
+ if (doneForAllNamespaces) {
182
326
  this.#spacesToPoll.delete(spaceId);
183
327
  }
184
328
  }),
@@ -210,12 +354,18 @@ export class FeedSyncer extends Resource {
210
354
  this.#getSpaceIds(),
211
355
  (spaceId) =>
212
356
  Effect.gen(this, function* () {
213
- const { done } = yield* this.#syncClient.push({
214
- spaceId,
215
- feedNamespace: this.#syncNamespace,
216
- limit: this.#messageBlocksLimit,
217
- });
218
- if (!done) {
357
+ let doneForAllNamespaces = true;
358
+ for (const feedNamespace of this.#syncNamespaces) {
359
+ const { done } = yield* this.#syncClient.push(this._ctx, {
360
+ spaceId,
361
+ feedNamespace,
362
+ limit: this.#messageBlocksLimit,
363
+ });
364
+ if (!done) {
365
+ doneForAllNamespaces = false;
366
+ }
367
+ }
368
+ if (!doneForAllNamespaces) {
219
369
  // Keep pushing until all blocks are pushed.
220
370
  this.#pushTask.schedule();
221
371
  }
@@ -4,6 +4,7 @@
4
4
 
5
5
  import { describe, test } from 'vitest';
6
6
 
7
+ import { Context } from '@dxos/context';
7
8
  import { MemorySignalManager, MemorySignalManagerContext } from '@dxos/messaging';
8
9
  import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
9
10
  import { openAndClose } from '@dxos/test-utils';
@@ -19,7 +20,7 @@ describe('services/ServiceContext', () => {
19
20
  const device2 = await createOpenServiceContext(networkContext);
20
21
  await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
21
22
 
22
- const space1 = await device1.dataSpaceManager!.createSpace();
23
+ const space1 = await device1.dataSpaceManager!.createSpace(new Context());
23
24
  await device2.dataSpaceManager!.waitUntilSpaceReady(space1!.key);
24
25
  const space2 = await device2.dataSpaceManager!.spaces.get(space1.key);
25
26
  await space2!.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.timeframe);
@@ -35,7 +36,7 @@ describe('services/ServiceContext', () => {
35
36
 
36
37
  const identity2 = await createOpenServiceContext(networkContext);
37
38
  await identity2.createIdentity();
38
- const space1 = await identity2.dataSpaceManager!.createSpace();
39
+ const space1 = await identity2.dataSpaceManager!.createSpace(new Context());
39
40
  await Promise.all(
40
41
  performInvitation({
41
42
  host: identity2,
@@ -22,12 +22,12 @@ import { type RuntimeProvider } from '@dxos/effect';
22
22
  import { FeedFactory, FeedStore } from '@dxos/feed-store';
23
23
  import { invariant } from '@dxos/invariant';
24
24
  import { Keyring } from '@dxos/keyring';
25
- import { PublicKey } from '@dxos/keys';
25
+ import { type SpaceId } from '@dxos/keys';
26
26
  import { type LevelDB } from '@dxos/kv-store';
27
27
  import { log } from '@dxos/log';
28
28
  import { type SignalManager } from '@dxos/messaging';
29
29
  import { type SwarmNetworkManager } from '@dxos/network-manager';
30
- import { InvalidStorageVersionError, STORAGE_VERSION, trace } from '@dxos/protocols';
30
+ import { InvalidStorageVersionError, STORAGE_VERSION } from '@dxos/protocols';
31
31
  import { FeedProtocol } from '@dxos/protocols';
32
32
  import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
33
33
  import { type Runtime } from '@dxos/protocols/proto/dxos/config';
@@ -42,6 +42,7 @@ import { safeInstanceof } from '@dxos/util';
42
42
  import { EdgeAgentManager } from '../agents';
43
43
  import {
44
44
  type CreateIdentityOptions,
45
+ type Identity,
45
46
  IdentityManager,
46
47
  type IdentityManagerProps,
47
48
  type JoinIdentityProps,
@@ -56,7 +57,6 @@ import {
56
57
  SpaceInvitationProtocol,
57
58
  } from '../invitations';
58
59
  import { DataSpaceManager, type DataSpaceManagerRuntimeProps, type SigningContext } from '../spaces';
59
-
60
60
  import { FeedSyncer } from './feed-syncer';
61
61
 
62
62
  export type ServiceContextRuntimeProps = Pick<
@@ -67,7 +67,6 @@ export type ServiceContextRuntimeProps = Pick<
67
67
  invitationConnectionDefaultProps?: InvitationConnectionProps;
68
68
  disableP2pReplication?: boolean;
69
69
  enableVectorIndexing?: boolean;
70
- enableLocalQueues?: boolean;
71
70
  };
72
71
  /**
73
72
  * Shared backend for all client services.
@@ -75,7 +74,7 @@ export type ServiceContextRuntimeProps = Pick<
75
74
  // TODO(burdon): Rename/break-up into smaller components. And/or make members private.
76
75
  // TODO(dmaretskyi): Gets duplicated in CJS build between normal and testing bundles.
77
76
  @safeInstanceof('dxos.client-services.ServiceContext')
78
- @Trace.resource()
77
+ @Trace.resource({ lifecycle: true })
79
78
  export class ServiceContext extends Resource {
80
79
  private readonly _edgeIdentityUpdateMutex = new Mutex();
81
80
 
@@ -105,8 +104,6 @@ export class ServiceContext extends Resource {
105
104
 
106
105
  private _deviceSpaceSync?: CredentialProcessor;
107
106
 
108
- private readonly _instanceId = PublicKey.random().toHex();
109
-
110
107
  constructor(
111
108
  public readonly storage: Storage,
112
109
  public readonly level: LevelDB,
@@ -170,7 +167,15 @@ export class ServiceContext extends Resource {
170
167
  peerIdProvider: () => this.identityManager.identity?.deviceKey?.toHex(),
171
168
  getSpaceKeyByRootDocumentId: (documentId) => this.spaceManager.findSpaceByRootDocumentId(documentId)?.key,
172
169
  runtime: this._runtime,
173
- localQueues: this._runtimeProps?.enableLocalQueues,
170
+ useSubduction: this._edgeFeatures?.subductionReplicator,
171
+ syncQueue: async (ctx, request) => {
172
+ return this._feedSyncer?.syncBlocking(ctx, {
173
+ spaceId: request.spaceId as SpaceId,
174
+ subspaceTag: request.subspaceTag,
175
+ shouldPush: request.shouldPush,
176
+ shouldPull: request.shouldPull,
177
+ });
178
+ },
174
179
  });
175
180
 
176
181
  this.invitations = new InvitationsHandler(
@@ -213,49 +218,80 @@ export class ServiceContext extends Resource {
213
218
  edgeClient: this._edgeConnection,
214
219
  peerId: this.identityManager.identity?.deviceKey?.toHex() ?? '',
215
220
  getSpaceIds: () => this.echoHost!.spaceIds,
216
- syncNamespace: FeedProtocol.WellKnownNamespaces.data,
221
+ syncNamespaces: [FeedProtocol.WellKnownNamespaces.data, FeedProtocol.WellKnownNamespaces.trace],
217
222
  });
218
223
  }
219
224
  }
220
225
 
221
- @Trace.span()
226
+ @Trace.span({ op: 'lifecycle' })
222
227
  protected override async _open(ctx: Context): Promise<void> {
223
228
  await this._checkStorageVersion();
224
229
 
225
230
  log('opening...');
226
- log.trace('dxos.sdk.service-context.open', trace.begin({ id: this._instanceId }));
227
231
 
232
+ log('opening identityManager...');
228
233
  await this.identityManager.open(ctx);
234
+ log('identityManager opened', { hasIdentity: !!this.identityManager.identity });
235
+
236
+ log('setting network identity...');
237
+ await this._setNetworkIdentity({ identity: this.identityManager.identity });
238
+ log('network identity set');
229
239
 
230
- await this._setNetworkIdentity();
240
+ log('opening edge connection...');
241
+ await this._edgeConnection?.open(ctx);
242
+ log('edge connection opened');
231
243
 
232
- await this._edgeConnection?.open();
233
- await this.signalManager.open();
244
+ log('opening signal manager...');
245
+ await this.signalManager.open(ctx);
246
+ log('signal manager opened');
247
+
248
+ log('opening network manager...');
234
249
  await this.networkManager.open();
250
+ log('network manager opened');
235
251
 
252
+ log('opening echo host...');
236
253
  await this.echoHost.open(ctx);
254
+ log('echo host opened');
237
255
 
238
256
  if (this._meshReplicator) {
239
- await this.echoHost.addReplicator(this._meshReplicator);
257
+ log('adding mesh replicator...');
258
+ await this.echoHost.addReplicator(ctx, this._meshReplicator);
259
+ log('mesh replicator added');
240
260
  }
241
261
  if (this._echoEdgeReplicator) {
242
- await this.echoHost.addReplicator(this._echoEdgeReplicator);
262
+ log('adding edge replicator...');
263
+ await this.echoHost.addReplicator(ctx, this._echoEdgeReplicator);
264
+ log('edge replicator added');
243
265
  }
244
266
 
267
+ log('loading metadata store...');
245
268
  await this.metadataStore.load();
269
+ log('metadata store loaded');
270
+
271
+ log('opening space manager...');
246
272
  await this.spaceManager.open();
273
+ log('space manager opened');
247
274
 
248
275
  if (this.identityManager.identity) {
249
- await this.identityManager.identity.joinNetwork();
276
+ log('joining network...');
277
+ await this.identityManager.identity.joinNetwork(ctx);
278
+ log('network joined');
279
+
280
+ log('initializing spaces...(calling _initialize)');
250
281
  await this._initialize(ctx);
282
+ log('spaces initialized');
283
+ } else {
284
+ log('no identity, skipping network join and space initialization');
251
285
  }
252
286
 
253
- await this._feedSyncer?.open();
287
+ log('opening feed syncer...');
288
+ await this._feedSyncer?.open(ctx);
289
+ log('feed syncer opened');
254
290
 
255
- const loadedInvitations = await this.invitationsManager.loadPersistentInvitations();
291
+ log('loading persistent invitations...');
292
+ const loadedInvitations = await this.invitationsManager.loadPersistentInvitations(ctx);
256
293
  log('loaded persistent invitations', { count: loadedInvitations.invitations?.length });
257
294
 
258
- log.trace('dxos.sdk.service-context.open', trace.end({ id: this._instanceId }));
259
295
  log('opened');
260
296
  }
261
297
 
@@ -267,13 +303,13 @@ export class ServiceContext extends Resource {
267
303
  if (this._deviceSpaceSync && this.identityManager.identity) {
268
304
  await this.identityManager.identity.space.spaceState.removeCredentialProcessor(this._deviceSpaceSync);
269
305
  }
270
- await this.dataSpaceManager?.close();
306
+ await this.dataSpaceManager?.close(ctx);
271
307
  await this.edgeAgentManager?.close();
272
- await this.identityManager.close();
308
+ await this.identityManager.close(ctx);
273
309
  await this.spaceManager.close();
274
310
  await this.echoHost.close(ctx);
275
311
 
276
- await this.networkManager.close();
312
+ await this.networkManager.close(ctx);
277
313
  await this.signalManager.close();
278
314
  await this._edgeConnection?.close();
279
315
  await this.feedStore.close();
@@ -282,11 +318,12 @@ export class ServiceContext extends Resource {
282
318
  log('closed');
283
319
  }
284
320
 
285
- async createIdentity(params: CreateIdentityOptions = {}) {
286
- const identity = await this.identityManager.createIdentity(params);
287
- await this._setNetworkIdentity();
288
- await identity.joinNetwork();
289
- await this._initialize(new Context());
321
+ async createIdentity(params: CreateIdentityOptions = {}, ctx?: Context) {
322
+ ctx ??= this._ctx;
323
+ const identity = await this.identityManager.createIdentity(params, ctx);
324
+ await this._setNetworkIdentity({ identity });
325
+ await identity.joinNetwork(ctx);
326
+ await this._initialize(ctx);
290
327
  return identity;
291
328
  }
292
329
 
@@ -310,11 +347,11 @@ export class ServiceContext extends Resource {
310
347
  }
311
348
 
312
349
  private async _acceptIdentity(params: JoinIdentityProps) {
313
- const { identity, identityRecord } = await this.identityManager.prepareIdentity(params);
314
- await this._setNetworkIdentity({ deviceCredential: params.authorizedDeviceCredential! });
315
- await identity.joinNetwork();
350
+ const { identity, identityRecord } = await this.identityManager.prepareIdentity(params, this._ctx);
351
+ await this._setNetworkIdentity({ deviceCredential: params.authorizedDeviceCredential!, identity });
352
+ await identity.joinNetwork(this._ctx);
316
353
  await this.identityManager.acceptIdentity(identity, identityRecord, params.deviceProfile);
317
- await this._initialize(new Context());
354
+ await this._initialize(this._ctx);
318
355
  return identity;
319
356
  }
320
357
 
@@ -329,7 +366,7 @@ export class ServiceContext extends Resource {
329
366
  // Called when identity is created.
330
367
  @Trace.span()
331
368
  private async _initialize(ctx: Context): Promise<void> {
332
- log('initializing spaces...');
369
+ log('_initialize: start');
333
370
  const identity = this.identityManager.identity ?? failUndefined();
334
371
  const signingContext: SigningContext = {
335
372
  credentialSigner: identity.getIdentityCredentialSigner(),
@@ -341,6 +378,7 @@ export class ServiceContext extends Resource {
341
378
  },
342
379
  };
343
380
 
381
+ log('_initialize: creating DataSpaceManager');
344
382
  this.dataSpaceManager = new DataSpaceManager({
345
383
  spaceManager: this.spaceManager,
346
384
  metadataStore: this.metadataStore,
@@ -356,7 +394,9 @@ export class ServiceContext extends Resource {
356
394
  runtimeProps: this._runtimeProps as DataSpaceManagerRuntimeProps,
357
395
  edgeFeatures: this._edgeFeatures,
358
396
  });
359
- await this.dataSpaceManager.open();
397
+ log('_initialize: opening DataSpaceManager...');
398
+ await this.dataSpaceManager.open(ctx);
399
+ log('_initialize: DataSpaceManager opened');
360
400
 
361
401
  this.edgeAgentManager = new EdgeAgentManager(
362
402
  this._edgeFeatures,
@@ -364,13 +404,16 @@ export class ServiceContext extends Resource {
364
404
  this.dataSpaceManager,
365
405
  identity,
366
406
  );
367
- await this.edgeAgentManager.open();
407
+ log('_initialize: opening EdgeAgentManager...');
408
+ await this.edgeAgentManager.open(ctx);
409
+ log('_initialize: EdgeAgentManager opened');
368
410
 
369
411
  this._handlerFactories.set(Invitation.Kind.SPACE, (invitation) => {
370
412
  invariant(this.dataSpaceManager, 'dataSpaceManager not initialized yet');
371
413
  return new SpaceInvitationProtocol(this.dataSpaceManager, signingContext, this.keyring, invitation.spaceKey);
372
414
  });
373
415
  this.initialized.wake();
416
+ log('_initialize: initialized.wake() called');
374
417
 
375
418
  this._deviceSpaceSync = {
376
419
  processCredential: async (credential: Credential) => {
@@ -393,9 +436,10 @@ export class ServiceContext extends Resource {
393
436
 
394
437
  try {
395
438
  log('accepting space recorded in halo', { details: assertion });
396
- await this.dataSpaceManager.acceptSpace({
439
+ await this.dataSpaceManager.acceptSpace(this._ctx, {
397
440
  spaceKey: assertion.spaceKey,
398
441
  genesisFeedKey: assertion.genesisFeedKey,
442
+ tags: assertion.tags,
399
443
  });
400
444
  } catch (err) {
401
445
  log.catch(err);
@@ -406,33 +450,42 @@ export class ServiceContext extends Resource {
406
450
  await identity.space.spaceState.addCredentialProcessor(this._deviceSpaceSync);
407
451
  }
408
452
 
409
- private async _setNetworkIdentity(params?: { deviceCredential: Credential }): Promise<void> {
453
+ private async _setNetworkIdentity(params?: { deviceCredential?: Credential; identity?: Identity }): Promise<void> {
454
+ log('_setNetworkIdentity: acquiring mutex...');
410
455
  using _ = await this._edgeIdentityUpdateMutex.acquire();
456
+ log('_setNetworkIdentity: mutex acquired');
411
457
 
412
458
  let edgeIdentity: EdgeIdentity;
413
- const identity = this.identityManager.identity;
459
+ const identity = params?.identity;
414
460
  if (identity) {
415
- log('setting identity on edge connection', {
461
+ log('_setNetworkIdentity: has identity', {
416
462
  identity: identity.identityKey.toHex(),
417
- swarms: this.networkManager.topics,
463
+ hasDeviceCredential: !!params?.deviceCredential,
418
464
  });
419
465
 
420
466
  if (params?.deviceCredential) {
467
+ log('_setNetworkIdentity: creating chain edge identity with device credential...');
421
468
  edgeIdentity = await createChainEdgeIdentity(
422
469
  identity.signer,
423
470
  identity.identityKey,
424
471
  identity.deviceKey,
425
- params?.deviceCredential && { credential: params.deviceCredential },
472
+ { credential: params.deviceCredential },
426
473
  [], // TODO(dmaretskyi): Service access credentials.
427
474
  );
475
+ log('_setNetworkIdentity: chain edge identity created');
428
476
  } else {
477
+ log('_setNetworkIdentity: waiting for identity.ready()...');
429
478
  // TODO: throw here or from identity if device chain can't be loaded, to avoid indefinite hangup
430
479
  await warnAfterTimeout(10_000, 'Waiting for identity to be ready for edge connection', async () => {
431
480
  await identity.ready();
432
481
  });
482
+ log('_setNetworkIdentity: identity.ready() resolved', {
483
+ hasDeviceCredentialChain: !!identity.deviceCredentialChain,
484
+ });
433
485
 
434
486
  invariant(identity.deviceCredentialChain);
435
487
 
488
+ log('_setNetworkIdentity: creating chain edge identity...');
436
489
  edgeIdentity = await createChainEdgeIdentity(
437
490
  identity.signer,
438
491
  identity.identityKey,
@@ -440,9 +493,12 @@ export class ServiceContext extends Resource {
440
493
  identity.deviceCredentialChain,
441
494
  [], // TODO(dmaretskyi): Service access credentials.
442
495
  );
496
+ log('_setNetworkIdentity: chain edge identity created');
443
497
  }
444
498
  } else {
499
+ log('_setNetworkIdentity: no identity, creating ephemeral edge identity...');
445
500
  edgeIdentity = await createEphemeralEdgeIdentity();
501
+ log('_setNetworkIdentity: ephemeral edge identity created');
446
502
  }
447
503
 
448
504
  this._edgeConnection?.setIdentity(edgeIdentity);
@@ -451,5 +507,6 @@ export class ServiceContext extends Resource {
451
507
  identityKey: edgeIdentity.identityKey,
452
508
  peerKey: edgeIdentity.peerKey,
453
509
  });
510
+ log('_setNetworkIdentity: done');
454
511
  }
455
512
  }