@dxos/client-services 0.8.4-main.bc674ce → 0.8.4-main.c351d160a8

package/src/packlets/services/feed-syncer.ts

@@ -0,0 +1,330 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import type * as SqlClient from '@effect/sql/SqlClient';
+import { Encoder, decode as cborXdecode } from 'cbor-x';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { AsyncTask, scheduleTask } from '@dxos/async';
+import { Resource } from '@dxos/context';
+import { type EdgeConnection, MessageSchema } from '@dxos/edge-client';
+import { RuntimeProvider } from '@dxos/effect';
+import { type FeedStore, SyncClient } from '@dxos/feed';
+import { invariant } from '@dxos/invariant';
+import { SpaceId } from '@dxos/keys';
+import { FeedProtocol } from '@dxos/protocols';
+import { EdgeService } from '@dxos/protocols';
+import { createBuf } from '@dxos/protocols/buf';
+import { type Message as RouterMessage } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
+import type { SqlTransaction } from '@dxos/sql-sqlite';
+import { bufferToArray } from '@dxos/util';
+
+const encoder = new Encoder({ tagUint8Array: false, useRecords: false });
+
+const DEFAULT_MESSAGE_BLOCKS_LIMIT = 50;
+const DEFAULT_SYNC_CONCURRENCY = 5;
+const DEFAULT_POLLING_INTERVAL = 5_000;
+const DEFAULT_POLL_REQUEST_THROTTLE_MS = 250;
+const MAX_BLOCKING_SYNC_ITERATIONS = 100;
+
+interface FeedSyncerOptions {
+  runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransaction.SqlTransaction>;
+  feedStore: FeedStore;
+  edgeClient: EdgeConnection;
+  peerId: string;
+  getSpaceIds: () => SpaceId[];
+
+  /**
+   * Namespaces to sync.
+   */
+  syncNamespaces: string[];
+
+  /**
+   * Maximum number of blocks to sync in a single message.
+   * @default 50
+   */
+  messageBlocksLimit?: number;
+
+  /**
+   * Maximum number of spaces to sync concurrently.
+   * @default 5
+   */
+  syncConcurrency?: number;
+
+  /**
+   * Interval between full polls.
+   * @default 10 seconds
+   */
+  pollingInterval?: number;
+
+  /**
+   * Minimum delay between externally requested best-effort polls.
+   * @default 250 ms
+   */
+  pollRequestThrottleMs?: number;
+}
+
+export class FeedSyncer extends Resource {
+  readonly #syncNamespaces: string[];
+  readonly #messageBlocksLimit: number;
+  readonly #syncConcurrency: number;
+  readonly #pollingInterval: number;
+  readonly #pollRequestThrottleMs: number;
+
+  readonly #runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransaction.SqlTransaction>;
+  readonly #feedStore: FeedStore;
+  readonly #edgeClient: EdgeConnection;
+  readonly #syncClient: SyncClient;
+  readonly #getSpaceIds: () => SpaceId[];
+
+  #spacesToPoll = new Set<SpaceId>();
+  /** Last time full poll was completed. */
+  #lastFullPoll: number | null = null;
+  #throttledPollScheduled = false;
+  #lastRequestedPollAt: number | null = null;
+
+  constructor(options: FeedSyncerOptions) {
+    super();
+    this.#runtime = options.runtime;
+    this.#feedStore = options.feedStore;
+    this.#edgeClient = options.edgeClient;
+    this.#syncClient = new SyncClient({
+      peerId: options.peerId,
+      feedStore: options.feedStore,
+      sendMessage: this.#sendMessage.bind(this),
+    });
+    this.#getSpaceIds = options.getSpaceIds;
+    this.#syncNamespaces = options.syncNamespaces;
+    this.#messageBlocksLimit = options.messageBlocksLimit ?? DEFAULT_MESSAGE_BLOCKS_LIMIT;
+    this.#syncConcurrency = options.syncConcurrency ?? DEFAULT_SYNC_CONCURRENCY;
+    this.#pollingInterval = options.pollingInterval ?? DEFAULT_POLLING_INTERVAL;
+    this.#pollRequestThrottleMs = options.pollRequestThrottleMs ?? DEFAULT_POLL_REQUEST_THROTTLE_MS;
+  }
+
+  protected override async _open(): Promise<void> {
+    this._ctx.onDispose(
+      this.#edgeClient.onMessage((msg: RouterMessage) => {
+        if (!msg.serviceId) {
+          return;
+        }
+        const service = msg.serviceId.split(':')[0];
+        if (service !== EdgeService.QUEUE_REPLICATOR) {
+          return;
+        }
+        const handleMessageEffect = Effect.gen(this, function* () {
+          const decoded = yield* Effect.try({
+            try: () => cborXdecode(msg.payload!.value),
+            catch: (error) => new Error(`Failed to decode feed sync message: ${error}`),
+          });
+          const payload = yield* Schema.validate(FeedProtocol.ProtocolMessage)(decoded);
+          yield* this.#syncClient.handleMessage(payload);
+        });
+
+        void RuntimeProvider.runPromise(this.#runtime)(handleMessageEffect);
+      }),
+    );
+
+    this._ctx.onDispose(
+      // NOTE: This will fire immediately if the connection is already open.
+      this.#edgeClient.onReconnected(async () => {}),
+    );
+
+    this.#feedStore.onNewBlocks.on(this._ctx, () => {
+      this.#pushTask.schedule();
+    });
+
+    await this.#pollTask.open();
+    await this.#pushTask.open();
+
+    this.#resetSpacesToPoll();
+    this.#pollTask.schedule();
+  }
+
+  protected override async _close(): Promise<void> {
+    await this.#pollTask.close();
+    await this.#pushTask.close();
+  }
+
+  /**
+   * Schedules a best-effort pull without blocking the caller.
+   */
+  schedulePoll(): void {
+    this.#resetSpacesToPoll();
+    if (this.#throttledPollScheduled) {
+      return;
+    }
+
+    const now = Date.now();
+    const delay =
+      this.#lastRequestedPollAt == null
+        ? 0
+        : Math.max(this.#pollRequestThrottleMs - (now - this.#lastRequestedPollAt), 0);
+    this.#throttledPollScheduled = true;
+    scheduleTask(
+      this._ctx,
+      () => {
+        this.#throttledPollScheduled = false;
+        this.#lastRequestedPollAt = Date.now();
+        this.#pollTask.schedule();
+      },
+      delay,
+    );
+  }
+
+  /**
+   * Performs queue sync and blocks until there are no pending sync batches.
+   */
+  async syncBlocking({
+    spaceId,
+    subspaceTag,
+    shouldPush = true,
+    shouldPull = true,
+  }: {
+    spaceId: SpaceId;
+    subspaceTag: string;
+    shouldPush?: boolean;
+    shouldPull?: boolean;
+  }): Promise<void> {
+    invariant(SpaceId.isValid(spaceId));
+    invariant(FeedProtocol.isWellKnownNamespace(subspaceTag));
+    if (!shouldPush && !shouldPull) {
+      return;
+    }
+
+    await RuntimeProvider.runPromise(this.#runtime)(
+      Effect.gen(this, function* () {
+        let done = false;
+        let iterations = 0;
+        while (!done) {
+          done = true;
+          if (shouldPull) {
+            const pullResult = yield* this.#syncClient.pull({
+              spaceId,
+              feedNamespace: subspaceTag,
+              limit: this.#messageBlocksLimit,
+            });
+            done &&= pullResult.done;
+          }
+
+          if (shouldPush) {
+            const pushResult = yield* this.#syncClient.push({
+              spaceId,
+              feedNamespace: subspaceTag,
+              limit: this.#messageBlocksLimit,
+            });
+            done &&= pushResult.done;
+          }
+          iterations++;
+          if (iterations > MAX_BLOCKING_SYNC_ITERATIONS) {
+            throw new Error('Blocking sync exceeded max iterations.');
+          }
+        }
+      }),
+    );
+  }
+
+  #resetSpacesToPoll(): void {
+    this.#spacesToPoll.clear();
+    this.#getSpaceIds().forEach((spaceId) => {
+      this.#spacesToPoll.add(spaceId);
+    });
+    this.#lastFullPoll = Date.now();
+  }
+
+  #sendMessage(message: FeedProtocol.QueryRequest | FeedProtocol.AppendRequest): Effect.Effect<void, unknown, never> {
+    return Effect.gen(this, function* () {
+      const encoded = encoder.encode(message);
+      yield* Effect.tryPromise(async () =>
+        this.#edgeClient.send(
+          createBuf(MessageSchema, {
+            source: {
+              identityKey: this.#edgeClient.identityKey,
+              peerKey: this.#edgeClient.peerKey,
+            },
+            serviceId: this.#getTargetServiceId(message),
+            payload: { value: bufferToArray(encoded) },
+          }),
+        ),
+      );
+    });
+  }
+
+  #getTargetServiceId(message: FeedProtocol.QueryRequest | FeedProtocol.AppendRequest): string {
+    // TODO(dmaretskyi): Perhaps in the future we will want to include the queue namespace here as well.
+    //                   This would require putting it at the top level of the message.
+    //                   For now, we let the edge router handle it.
+    return FeedProtocol.encodeServiceId(message.feedNamespace, message.spaceId as SpaceId);
+  }
+
+  readonly #pollTask = new AsyncTask(async () =>
+    Effect.gen(this, function* () {
+      yield* Effect.forEach(
+        this.#spacesToPoll,
+        (spaceId) =>
+          Effect.gen(this, function* () {
+            let doneForAllNamespaces = true;
+            for (const feedNamespace of this.#syncNamespaces) {
+              const { done } = yield* this.#syncClient.pull({
+                spaceId,
+                feedNamespace,
+                limit: this.#messageBlocksLimit,
+              });
+              if (!done) {
+                doneForAllNamespaces = false;
+              }
+            }
+            if (doneForAllNamespaces) {
+              this.#spacesToPoll.delete(spaceId);
+            }
+          }),
+        { concurrency: this.#syncConcurrency },
+      );
+
+      // If its time to do a full poll, reset the spaces to poll and schedule the next poll immediately.
+      if (this.#lastFullPoll == null || Date.now() - this.#lastFullPoll > this.#pollingInterval) {
+        this.#resetSpacesToPoll();
+        this.#pollTask.schedule();
+      } else if (this.#spacesToPoll.size > 0) {
+        // If there are some spaces still syncing, poll them immediately.
+        this.#pollTask.schedule();
+      } else {
+        // All spaces sync, and there's time before the next full poll, schedule it later.
+        this.#resetSpacesToPoll();
+        scheduleTask(
+          this._ctx,
+          () => this.#pollTask.schedule(),
+          Math.max(this.#pollingInterval - (Date.now() - (this.#lastFullPoll ?? 0)), 0),
+        );
+      }
+    }).pipe(RuntimeProvider.runPromise(this.#runtime)),
+  );
+
+  readonly #pushTask = new AsyncTask(async () =>
+    Effect.gen(this, function* () {
+      yield* Effect.forEach(
+        this.#getSpaceIds(),
+        (spaceId) =>
+          Effect.gen(this, function* () {
+            let doneForAllNamespaces = true;
+            for (const feedNamespace of this.#syncNamespaces) {
+              const { done } = yield* this.#syncClient.push({
+                spaceId,
+                feedNamespace,
+                limit: this.#messageBlocksLimit,
+              });
+              if (!done) {
+                doneForAllNamespaces = false;
+              }
+            }
+            if (!doneForAllNamespaces) {
+              // Keep pushing until all blocks are pushed.
+              this.#pushTask.schedule();
+            }
+          }),
+        { concurrency: this.#syncConcurrency },
+      );
+    }).pipe(RuntimeProvider.runPromise(this.#runtime)),
+  );
+}

package/src/packlets/services/platform.ts

@@ -14,12 +14,18 @@ export const getPlatform = (): Platform => {
         userAgent,
         uptime: Math.floor((Date.now() - window.performance.timeOrigin) / 1_000),
       };
-    } else {
+    } else if (typeof SharedWorkerGlobalScope !== 'undefined') {
       // Shared worker.
       return {
         type: Platform.PLATFORM_TYPE.SHARED_WORKER,
         uptime: Math.floor((Date.now() - performance.timeOrigin) / 1_000),
       };
+    } else {
+      // Dedicated worker.
+      return {
+        type: Platform.PLATFORM_TYPE.DEDICATED_WORKER,
+        uptime: Math.floor((Date.now() - performance.timeOrigin) / 1_000),
+      };
     }
   } else {
     // Node.

package/src/packlets/services/service-context.ts

@@ -22,12 +22,13 @@ import { type RuntimeProvider } from '@dxos/effect';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { Keyring } from '@dxos/keyring';
-import { PublicKey } from '@dxos/keys';
+import { PublicKey, type SpaceId } from '@dxos/keys';
 import { type LevelDB } from '@dxos/kv-store';
 import { log } from '@dxos/log';
 import { type SignalManager } from '@dxos/messaging';
 import { type SwarmNetworkManager } from '@dxos/network-manager';
 import { InvalidStorageVersionError, STORAGE_VERSION, trace } from '@dxos/protocols';
+import { FeedProtocol } from '@dxos/protocols';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { type Runtime } from '@dxos/protocols/proto/dxos/config';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
@@ -56,6 +57,8 @@ import {
 } from '../invitations';
 import { DataSpaceManager, type DataSpaceManagerRuntimeProps, type SigningContext } from '../spaces';
 
+import { FeedSyncer } from './feed-syncer';
+
 export type ServiceContextRuntimeProps = Pick<
   IdentityManagerProps,
   'devicePresenceOfflineTimeout' | 'devicePresenceAnnounceInterval'
@@ -64,8 +67,6 @@ export type ServiceContextRuntimeProps = Pick<
     invitationConnectionDefaultProps?: InvitationConnectionProps;
     disableP2pReplication?: boolean;
     enableVectorIndexing?: boolean;
-    enableSqlite?: boolean;
-    enableLocalQueues?: boolean;
   };
 /**
  * Shared backend for all client services.
@@ -90,6 +91,7 @@ export class ServiceContext extends Resource {
   public readonly echoHost: EchoHost;
   private readonly _meshReplicator?: MeshEchoReplicator = undefined;
   private readonly _echoEdgeReplicator?: EchoEdgeReplicator = undefined;
+  private readonly _feedSyncer?: FeedSyncer = undefined;
 
   // Initialized after identity is initialized.
   public dataSpaceManager?: DataSpaceManager;
@@ -166,12 +168,15 @@ export class ServiceContext extends Resource {
       kv: this.level,
       peerIdProvider: () => this.identityManager.identity?.deviceKey?.toHex(),
       getSpaceKeyByRootDocumentId: (documentId) => this.spaceManager.findSpaceByRootDocumentId(documentId)?.key,
-      indexing: {
-        vector: this._runtimeProps?.enableVectorIndexing,
-        sqlIndex: this._runtimeProps?.enableSqlite,
-      },
       runtime: this._runtime,
-      localQueues: this._runtimeProps?.enableLocalQueues,
+      syncQueue: async (request) => {
+        return this._feedSyncer?.syncBlocking({
+          spaceId: request.spaceId as SpaceId,
+          subspaceTag: request.subspaceTag,
+          shouldPush: request.shouldPush,
+          shouldPull: request.shouldPull,
+        });
+      },
     });
 
     this.invitations = new InvitationsHandler(
@@ -206,6 +211,17 @@ export class ServiceContext extends Resource {
         edgeHttpClient: this._edgeHttpClient,
       });
     }
+
+    if (this.echoHost.feedStore && this._edgeConnection) {
+      this._feedSyncer = new FeedSyncer({
+        runtime: this._runtime,
+        feedStore: this.echoHost.feedStore,
+        edgeClient: this._edgeConnection,
+        peerId: this.identityManager.identity?.deviceKey?.toHex() ?? '',
+        getSpaceIds: () => this.echoHost!.spaceIds,
+        syncNamespaces: [FeedProtocol.WellKnownNamespaces.data, FeedProtocol.WellKnownNamespaces.trace],
+      });
+    }
   }
 
   @Trace.span()
@@ -215,31 +231,66 @@ export class ServiceContext extends Resource {
     log('opening...');
     log.trace('dxos.sdk.service-context.open', trace.begin({ id: this._instanceId }));
 
+    log('opening identityManager...');
     await this.identityManager.open(ctx);
+    log('identityManager opened', { hasIdentity: !!this.identityManager.identity });
 
+    log('setting network identity...');
     await this._setNetworkIdentity();
+    log('network identity set');
 
+    log('opening edge connection...');
     await this._edgeConnection?.open();
+    log('edge connection opened');
+
+    log('opening signal manager...');
     await this.signalManager.open();
+    log('signal manager opened');
+
+    log('opening network manager...');
     await this.networkManager.open();
+    log('network manager opened');
 
+    log('opening echo host...');
     await this.echoHost.open(ctx);
+    log('echo host opened');
 
     if (this._meshReplicator) {
+      log('adding mesh replicator...');
       await this.echoHost.addReplicator(this._meshReplicator);
+      log('mesh replicator added');
     }
     if (this._echoEdgeReplicator) {
+      log('adding edge replicator...');
       await this.echoHost.addReplicator(this._echoEdgeReplicator);
+      log('edge replicator added');
     }
 
+    log('loading metadata store...');
     await this.metadataStore.load();
+    log('metadata store loaded');
+
+    log('opening space manager...');
     await this.spaceManager.open();
+    log('space manager opened');
 
     if (this.identityManager.identity) {
+      log('joining network...');
       await this.identityManager.identity.joinNetwork();
+      log('network joined');
+
+      log('initializing spaces...(calling _initialize)');
       await this._initialize(ctx);
+      log('spaces initialized');
+    } else {
+      log('no identity, skipping network join and space initialization');
     }
 
+    log('opening feed syncer...');
+    await this._feedSyncer?.open();
+    log('feed syncer opened');
+
+    log('loading persistent invitations...');
     const loadedInvitations = await this.invitationsManager.loadPersistentInvitations();
     log('loaded persistent invitations', { count: loadedInvitations.invitations?.length });
 
@@ -249,6 +300,9 @@ export class ServiceContext extends Resource {
 
   protected override async _close(ctx: Context): Promise<void> {
     log('closing...');
+
+    await this._feedSyncer?.close();
+
     if (this._deviceSpaceSync && this.identityManager.identity) {
       await this.identityManager.identity.space.spaceState.removeCredentialProcessor(this._deviceSpaceSync);
     }
@@ -314,7 +368,7 @@ export class ServiceContext extends Resource {
   // Called when identity is created.
   @Trace.span()
   private async _initialize(ctx: Context): Promise<void> {
-    log('initializing spaces...');
+    log('_initialize: start');
     const identity = this.identityManager.identity ?? failUndefined();
     const signingContext: SigningContext = {
       credentialSigner: identity.getIdentityCredentialSigner(),
@@ -326,6 +380,7 @@ export class ServiceContext extends Resource {
       },
     };
 
+    log('_initialize: creating DataSpaceManager');
     this.dataSpaceManager = new DataSpaceManager({
       spaceManager: this.spaceManager,
       metadataStore: this.metadataStore,
@@ -341,7 +396,9 @@ export class ServiceContext extends Resource {
       runtimeProps: this._runtimeProps as DataSpaceManagerRuntimeProps,
       edgeFeatures: this._edgeFeatures,
     });
+    log('_initialize: opening DataSpaceManager...');
     await this.dataSpaceManager.open();
+    log('_initialize: DataSpaceManager opened');
 
     this.edgeAgentManager = new EdgeAgentManager(
       this._edgeFeatures,
@@ -349,13 +406,16 @@ export class ServiceContext extends Resource {
       this.dataSpaceManager,
       identity,
     );
+    log('_initialize: opening EdgeAgentManager...');
     await this.edgeAgentManager.open();
+    log('_initialize: EdgeAgentManager opened');
 
     this._handlerFactories.set(Invitation.Kind.SPACE, (invitation) => {
       invariant(this.dataSpaceManager, 'dataSpaceManager not initialized yet');
       return new SpaceInvitationProtocol(this.dataSpaceManager, signingContext, this.keyring, invitation.spaceKey);
     });
     this.initialized.wake();
+    log('_initialize: initialized.wake() called');
 
     this._deviceSpaceSync = {
       processCredential: async (credential: Credential) => {
@@ -392,17 +452,20 @@ export class ServiceContext extends Resource {
   }
 
   private async _setNetworkIdentity(params?: { deviceCredential: Credential }): Promise<void> {
+    log('_setNetworkIdentity: acquiring mutex...');
     using _ = await this._edgeIdentityUpdateMutex.acquire();
+    log('_setNetworkIdentity: mutex acquired');
 
     let edgeIdentity: EdgeIdentity;
     const identity = this.identityManager.identity;
     if (identity) {
-      log('setting identity on edge connection', {
+      log('_setNetworkIdentity: has identity', {
         identity: identity.identityKey.toHex(),
-        swarms: this.networkManager.topics,
+        hasDeviceCredential: !!params?.deviceCredential,
       });
 
       if (params?.deviceCredential) {
+        log('_setNetworkIdentity: creating chain edge identity with device credential...');
         edgeIdentity = await createChainEdgeIdentity(
           identity.signer,
           identity.identityKey,
@@ -410,14 +473,20 @@ export class ServiceContext extends Resource {
           params?.deviceCredential && { credential: params.deviceCredential },
           [], // TODO(dmaretskyi): Service access credentials.
         );
+        log('_setNetworkIdentity: chain edge identity created');
       } else {
+        log('_setNetworkIdentity: waiting for identity.ready()...');
         // TODO: throw here or from identity if device chain can't be loaded, to avoid indefinite hangup
         await warnAfterTimeout(10_000, 'Waiting for identity to be ready for edge connection', async () => {
           await identity.ready();
         });
+        log('_setNetworkIdentity: identity.ready() resolved', {
+          hasDeviceCredentialChain: !!identity.deviceCredentialChain,
+        });
 
         invariant(identity.deviceCredentialChain);
 
+        log('_setNetworkIdentity: creating chain edge identity...');
         edgeIdentity = await createChainEdgeIdentity(
           identity.signer,
           identity.identityKey,
@@ -425,9 +494,12 @@ export class ServiceContext extends Resource {
           identity.deviceCredentialChain,
           [], // TODO(dmaretskyi): Service access credentials.
         );
+        log('_setNetworkIdentity: chain edge identity created');
       }
     } else {
+      log('_setNetworkIdentity: no identity, creating ephemeral edge identity...');
       edgeIdentity = await createEphemeralEdgeIdentity();
+      log('_setNetworkIdentity: ephemeral edge identity created');
     }
 
     this._edgeConnection?.setIdentity(edgeIdentity);
@@ -436,5 +508,6 @@ export class ServiceContext extends Resource {
       identityKey: edgeIdentity.identityKey,
       peerKey: edgeIdentity.peerKey,
     });
+    log('_setNetworkIdentity: done');
   }
 }

package/src/packlets/services/service-host.ts

@@ -244,9 +244,6 @@ export class ClientServicesHost {
       if (this._runtimeProps.enableVectorIndexing === undefined) {
         this._runtimeProps.enableVectorIndexing = config?.get('runtime.client.enableVectorIndexing', false);
       }
-      if (this._runtimeProps.enableLocalQueues === undefined) {
-        this._runtimeProps.enableLocalQueues = config?.get('runtime.client.enableLocalQueues', false);
-      }
 
       invariant(!this._config, 'config already set');
       this._config = config;
@@ -395,8 +392,13 @@ export class ClientServicesHost {
       EdgeAgentService: new EdgeAgentServiceImpl(agentManagerProvider, this._edgeConnection),
     });
 
+    log('service-host: opening service context...');
     await this._serviceContext.open(ctx);
+    log('service-host: service context opened');
+
+    log('service-host: opening identity service...');
     await identityService.open();
+    log('service-host: identity service opened');
 
     const devtoolsProxy = this._config?.get('runtime.client.devtoolsProxy');
     if (devtoolsProxy) {

package/src/packlets/spaces/data-space.ts

@@ -450,6 +450,9 @@ export class DataSpace {
 
         log('credentials notarized');
       } catch (err) {
+        if (err instanceof ContextDisposedError) {
+          return;
+        }
         log.error('error notarizing credentials for feed admission', err);
         throw err;
       }