@dxos/client-services 0.8.4-main.f9ba587 → 0.8.4-main.fcfe5033a5

package/src/packlets/services/service-host.ts

@@ -2,39 +2,43 @@
 // Copyright 2021 DXOS.org
 //
 
+import * as SqlClient from '@effect/sql/SqlClient';
+import * as Effect from 'effect/Effect';
+
 import { Event, synchronized } from '@dxos/async';
-import { clientServiceBundle, type ClientServices } from '@dxos/client-protocol';
-import { type Config } from '@dxos/config';
+import { type ClientServices, clientServiceBundle } from '@dxos/client-protocol';
+import { type Config, resolveTelemetryTag } from '@dxos/config';
 import { Context } from '@dxos/context';
-import { EdgeClient, EdgeHttpClient, createStubEdgeIdentity, type EdgeConnection } from '@dxos/edge-client';
+import { EdgeClient, type EdgeConnection, EdgeHttpClient, createStubEdgeIdentity } from '@dxos/edge-client';
+import { RuntimeProvider } from '@dxos/effect';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { type LevelDB } from '@dxos/kv-store';
 import { log } from '@dxos/log';
-import { EdgeSignalManager, WebsocketSignalManager, type SignalManager } from '@dxos/messaging';
+import { EdgeSignalManager, type SignalManager, WebsocketSignalManager } from '@dxos/messaging';
 import {
   SwarmNetworkManager,
+  type TransportFactory,
   createIceProvider,
   createRtcTransportFactory,
-  type TransportFactory,
 } from '@dxos/network-manager';
 import { trace } from '@dxos/protocols';
 import { SystemStatus } from '@dxos/protocols/proto/dxos/client/services';
 import { type Storage } from '@dxos/random-access-storage';
-import { TRACE_PROCESSOR, trace as Trace } from '@dxos/tracing';
+import * as SqlExport from '@dxos/sql-sqlite/SqlExport';
+import type * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
+import { trace as Trace } from '@dxos/tracing';
 import { WebsocketRpcClient } from '@dxos/websocket-rpc';
 
-import { ServiceContext, type ServiceContextRuntimeParams } from './service-context';
-import { ServiceRegistry } from './service-registry';
 import { EdgeAgentServiceImpl } from '../agents';
 import { DevicesServiceImpl } from '../devices';
 import { DevtoolsHostEvents, DevtoolsServiceImpl } from '../devtools';
 import {
+  type CollectDiagnosticsBroadcastHandler,
   createCollectDiagnosticsBroadcastHandler,
   createDiagnostics,
-  type CollectDiagnosticsBroadcastHandler,
 } from '../diagnostics';
-import { IdentityServiceImpl, type CreateIdentityOptions } from '../identity';
+import { type CreateIdentityOptions, IdentityServiceImpl } from '../identity';
 import { ContactsServiceImpl } from '../identity/contacts-service';
 import { InvitationsServiceImpl } from '../invitations';
 import { Lock, type ResourceLock } from '../locks';
@@ -43,8 +47,10 @@ import { NetworkServiceImpl } from '../network';
 import { SpacesServiceImpl } from '../spaces';
 import { createLevel, createStorageObjects } from '../storage';
 import { SystemServiceImpl } from '../system';
+import { ServiceContext, type ServiceContextRuntimeProps } from './service-context';
+import { ServiceRegistry } from './service-registry';
 
-export type ClientServicesHostParams = {
+export type ClientServicesHostProps = {
   /**
    * Can be omitted if `initialize` is later called.
    */
@@ -56,7 +62,8 @@ export type ClientServicesHostParams = {
   level?: LevelDB;
   lockKey?: string;
   callbacks?: ClientServicesHostCallbacks;
-  runtimeParams?: ServiceContextRuntimeParams;
+  runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlExport.SqlExport | SqlTransaction.SqlTransaction>;
+  runtimeProps?: ServiceContextRuntimeProps;
 };
 
 export type ClientServicesHostCallbacks = {
@@ -79,10 +86,9 @@ export class ClientServicesHost {
   private readonly _serviceRegistry: ServiceRegistry<ClientServices>;
   private readonly _systemService: SystemServiceImpl;
   private readonly _loggingService: LoggingServiceImpl;
-  private readonly _tracingService = TRACE_PROCESSOR.createTraceSender();
+  private readonly _statusUpdate = new Event<void>();
 
   private _config?: Config;
-  private readonly _statusUpdate = new Event<void>();
   private _signalManager?: SignalManager;
   private _networkManager?: SwarmNetworkManager;
   private _storage?: Storage;
@@ -93,7 +99,10 @@ export class ClientServicesHost {
   private _edgeHttpClient?: EdgeHttpClient = undefined;
 
   private _serviceContext!: ServiceContext;
-  private readonly _runtimeParams: ServiceContextRuntimeParams;
+  private readonly _runtime: RuntimeProvider.RuntimeProvider<
+    SqlClient.SqlClient | SqlExport.SqlExport | SqlTransaction.SqlTransaction
+  >;
+  private readonly _runtimeProps: ServiceContextRuntimeProps;
   private diagnosticsBroadcastHandler: CollectDiagnosticsBroadcastHandler;
 
   @Trace.info()
@@ -114,20 +123,14 @@ export class ClientServicesHost {
     // TODO(wittjosiah): Turn this on by default.
     lockKey,
     callbacks,
-    runtimeParams,
-  }: ClientServicesHostParams = {}) {
+    runtime,
+    runtimeProps,
+  }: ClientServicesHostProps) {
     this._storage = storage;
     this._level = level;
     this._callbacks = callbacks;
-    this._runtimeParams = runtimeParams ?? {};
-
-    if (this._runtimeParams.disableP2pReplication === undefined) {
-      this._runtimeParams.disableP2pReplication = config?.get('runtime.client.disableP2pReplication', false);
-    }
-
-    if (this._runtimeParams.enableVectorIndexing === undefined) {
-      this._runtimeParams.enableVectorIndexing = config?.get('runtime.client.enableVectorIndexing', false);
-    }
+    this._runtime = runtime;
+    this._runtimeProps = runtimeProps ?? {};
 
     if (config) {
       this.initialize({ config, transportFactory, signalManager });
@@ -141,7 +144,7 @@ export class ClientServicesHost {
             void this.open(new Context());
           }
         },
-        onRelease: () => this.close(),
+        onRelease: () => this.close(Context.default()),
       });
     }
 
@@ -170,7 +173,6 @@ export class ClientServicesHost {
 
     this._serviceRegistry = new ServiceRegistry<ClientServices>(clientServiceBundle, {
       SystemService: this._systemService,
-      TracingService: this._tracingService,
     });
   }
 
@@ -198,6 +200,30 @@ export class ClientServicesHost {
     return this._serviceRegistry.services;
   }
 
+  /**
+   * Debugging util.
+   */
+  async exportSqliteDatabase(): Promise<Uint8Array> {
+    return await RuntimeProvider.runPromise(this._runtime)(
+      Effect.gen(function* () {
+        const sql = yield* SqlExport.SqlExport;
+        return yield* sql.export;
+      }),
+    );
+  }
+
+  /**
+   * Debugging util.
+   */
+  async runSqliteQuery(query: string, params?: any[]): Promise<readonly Record<string, unknown>[]> {
+    return await RuntimeProvider.runPromise(this._runtime)(
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        return yield* sql`${sql.unsafe(query, params)}`;
+      }),
+    );
+  }
+
   /**
    * Initialize the service host with the config.
    * Config can also be provided in the constructor.
@@ -208,6 +234,13 @@ export class ClientServicesHost {
     log('initializing...');
 
     if (config) {
+      if (this._runtimeProps.disableP2pReplication === undefined) {
+        this._runtimeProps.disableP2pReplication = config?.get('runtime.client.disableP2pReplication', false);
+      }
+      if (this._runtimeProps.enableVectorIndexing === undefined) {
+        this._runtimeProps.enableVectorIndexing = config?.get('runtime.client.enableVectorIndexing', false);
+      }
+
       invariant(!this._config, 'config already set');
       this._config = config;
       if (!this._storage) {
@@ -215,14 +248,16 @@ export class ClientServicesHost {
       }
     }
 
+    // TODO(wittjosiah): This is quite noisy during tests. Make configurable? Remove?
     if (!options.signalManager) {
-      log.warn('running signaling without telemetry metadata.');
+      // log.warn('running signaling without telemetry metadata.');
     }
 
     const endpoint = config?.get('runtime.services.edge.url');
     if (endpoint) {
-      this._edgeConnection = new EdgeClient(createStubEdgeIdentity(), { socketEndpoint: endpoint });
-      this._edgeHttpClient = new EdgeHttpClient(endpoint);
+      const clientTag = resolveTelemetryTag(config);
+      this._edgeConnection = new EdgeClient(createStubEdgeIdentity(), { socketEndpoint: endpoint, clientTag });
+      this._edgeHttpClient = new EdgeHttpClient(endpoint, { clientTag });
     }
 
     const {
@@ -288,7 +323,8 @@ export class ClientServicesHost {
       this._signalManager,
       this._edgeConnection,
       this._edgeHttpClient,
-      this._runtimeParams,
+      this._runtime,
+      this._runtimeProps,
       this._config.get('runtime.client.edgeFeatures'),
     );
 
@@ -306,8 +342,7 @@ export class ClientServicesHost {
       this._serviceContext.identityManager,
       this._serviceContext.recoveryManager,
       this._serviceContext.keyring,
-      () => this._serviceContext.dataSpaceManager!,
-      (params) => this._createIdentity(params),
+      (params, ctx) => this._createIdentity(params, ctx),
       (profile) => this._serviceContext.broadcastProfileUpdate(profile),
     );
 
@@ -327,11 +362,13 @@ export class ClientServicesHost {
       SpacesService: new SpacesServiceImpl(
         this._serviceContext.identityManager,
         this._serviceContext.spaceManager,
+        this._serviceContext.echoHost,
         dataSpaceManagerProvider,
       ),
 
       DataService: this._serviceContext.echoHost.dataService,
       QueryService: this._serviceContext.echoHost.queryService,
+      QueueService: this._serviceContext.echoHost.queuesService,
 
       NetworkService: new NetworkServiceImpl(
         this._serviceContext.networkManager,
@@ -340,7 +377,6 @@ export class ClientServicesHost {
       ),
 
       LoggingService: this._loggingService,
-      TracingService: this._tracingService,
 
       // TODO(burdon): Move to new protobuf definitions.
       DevtoolsHost: new DevtoolsServiceImpl({
@@ -352,8 +388,13 @@ export class ClientServicesHost {
       EdgeAgentService: new EdgeAgentServiceImpl(agentManagerProvider, this._edgeConnection),
     });
 
+    log('service-host: opening service context...');
     await this._serviceContext.open(ctx);
+    log('service-host: service context opened');
+
+    log('service-host: opening identity service...');
     await identityService.open();
+    log('service-host: identity service opened');
 
     const devtoolsProxy = this._config?.get('runtime.client.devtoolsProxy');
     if (devtoolsProxy) {
@@ -377,7 +418,7 @@ export class ClientServicesHost {
 
   @synchronized
   @Trace.span()
-  async close(): Promise<void> {
+  async close(ctx: Context): Promise<void> {
     if (!this._open) {
       return;
     }
@@ -405,14 +446,20 @@ export class ClientServicesHost {
     this._resetting = true;
     this._statusUpdate.emit();
     await this._serviceContext?.close();
+    // Clear LevelDB contents to remove all persisted Echo/Automerge/index data.
+    try {
+      await this._level!.clear();
+    } catch (err) {
+      log.warn('failed to clear leveldb during reset', { err });
+    }
     await this._storage!.reset();
     log.info('reset');
     log.trace('dxos.sdk.client-services-host.reset', trace.end({ id: traceId }));
     await this._callbacks?.onReset?.();
   }
 
-  private async _createIdentity(params: CreateIdentityOptions) {
-    const identity = await this._serviceContext.createIdentity(params);
+  private async _createIdentity(params: CreateIdentityOptions, ctx?: Context) {
+    const identity = await this._serviceContext.createIdentity(params, ctx);
     await this._serviceContext.initialized.wait();
     return identity;
   }

package/src/packlets/services/service-registry.test.ts

@@ -13,9 +13,9 @@ import { schema } from '@dxos/protocols/proto';
 import { type SystemService, SystemStatus } from '@dxos/protocols/proto/dxos/client/services';
 import { createLinkedPorts, createProtoRpcPeer, createServiceBundle } from '@dxos/rpc';
 
-import { ServiceRegistry } from './service-registry';
 import { SystemServiceImpl } from '../system';
 import { createServiceContext } from '../testing';
+import { ServiceRegistry } from './service-registry';
 
 // TODO(burdon): Create TestService (that doesn't require peers).
 

package/src/packlets/space-export/archive-format.ts

@@ -0,0 +1,42 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { SpaceArchive } from '@dxos/protocols/proto/dxos/client/services';
+
+const JSON_EXTENSIONS = ['.dx.json', '.dx.json.gz', '.json'];
+const BINARY_EXTENSIONS = ['.tar', '.tar.gz'];
+
+/**
+ * Detect the format of a space archive.
+ *
+ * Uses filename extension as the primary signal, and falls back to sniffing
+ * the first bytes of the archive contents to distinguish JSON from tar.
+ */
+export const detectSpaceArchiveFormat = (archive: Pick<SpaceArchive, 'filename' | 'contents'>): SpaceArchive.Format => {
+  const filename = archive.filename?.toLowerCase() ?? '';
+  if (JSON_EXTENSIONS.some((ext) => filename.endsWith(ext))) {
+    return SpaceArchive.Format.JSON;
+  }
+  if (BINARY_EXTENSIONS.some((ext) => filename.endsWith(ext))) {
+    return SpaceArchive.Format.BINARY;
+  }
+
+  // Fall back to byte sniffing: JSON archives start with '{' (0x7B) or whitespace.
+  const bytes = archive.contents;
+  if (bytes && bytes.length > 0) {
+    for (let i = 0; i < Math.min(bytes.length, 16); i++) {
+      const byte = bytes[i];
+      // Skip whitespace.
+      if (byte === 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d) {
+        continue;
+      }
+      if (byte === 0x7b /* '{' */) {
+        return SpaceArchive.Format.JSON;
+      }
+      break;
+    }
+  }
+
+  return SpaceArchive.Format.BINARY;
+};

package/src/packlets/space-export/index.ts

@@ -2,5 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './space-archive-writer';
+export * from './archive-format';
+export * from './serialized-space-reader';
+export * from './serialized-space-writer';
 export * from './space-archive-reader';
+export * from './space-archive-writer';

package/src/packlets/space-export/serialized-space-reader.ts

@@ -0,0 +1,111 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Obj } from '@dxos/echo';
+import { type SerializedSpace } from '@dxos/echo-db';
+import { type DatabaseDirectory, type ObjectStructure } from '@dxos/echo-protocol';
+import { assertArgument } from '@dxos/invariant';
+import { type SpaceArchive } from '@dxos/protocols/proto/dxos/client/services';
+
+const ATTR_TYPE = '@type';
+const ATTR_META = '@meta';
+const ATTR_DELETED = '@deleted';
+const ATTR_PARENT = '@parent';
+const ATTR_RELATION_SOURCE = '@relationSource';
+const ATTR_RELATION_TARGET = '@relationTarget';
+
+const INTERNAL_KEYS: ReadonlySet<string> = new Set([
+  'id',
+  ATTR_TYPE,
+  ATTR_META,
+  ATTR_DELETED,
+  ATTR_PARENT,
+  ATTR_RELATION_SOURCE,
+  ATTR_RELATION_TARGET,
+]);
+
+/**
+ * Parse a JSON space archive.
+ *
+ * The archive contents are expected to be the UTF-8 encoding of
+ * `JSON.stringify(serializedSpace)`.
+ */
+export const readSerializedSpaceArchive = (archive: SpaceArchive): SerializedSpace => {
+  const text = new TextDecoder().decode(archive.contents);
+  const parsed = JSON.parse(text) as SerializedSpace;
+  assertArgument(typeof parsed === 'object' && parsed !== null, 'archive', 'Invalid JSON archive payload');
+  assertArgument(typeof parsed.version === 'number', 'archive', 'Missing SerializedSpace.version');
+  assertArgument(Array.isArray(parsed.objects), 'archive', 'Missing SerializedSpace.objects');
+  return parsed;
+};
+
+/**
+ * Convert an {@link Obj.JSON} back into an internal {@link ObjectStructure} suitable
+ * for embedding into a {@link DatabaseDirectory}.
+ */
+export const objJsonToObjectStructure = (obj: Obj.JSON): ObjectStructure => {
+  const data: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(obj)) {
+    if (INTERNAL_KEYS.has(key)) {
+      continue;
+    }
+    data[key] = value;
+  }
+
+  const system: NonNullable<ObjectStructure['system']> = {};
+
+  const type = obj[ATTR_TYPE];
+  if (type) {
+    system.type = { '/': type as string };
+  }
+
+  const parent = (obj as any)[ATTR_PARENT];
+  if (typeof parent === 'string') {
+    system.parent = { '/': parent };
+  }
+
+  const relationSource = (obj as any)[ATTR_RELATION_SOURCE];
+  const relationTarget = (obj as any)[ATTR_RELATION_TARGET];
+  if (typeof relationSource === 'string' || typeof relationTarget === 'string') {
+    system.kind = 'relation';
+    if (typeof relationSource === 'string') {
+      system.source = { '/': relationSource };
+    }
+    if (typeof relationTarget === 'string') {
+      system.target = { '/': relationTarget };
+    }
+  } else {
+    system.kind = 'object';
+  }
+
+  if ((obj as any)[ATTR_DELETED]) {
+    system.deleted = true;
+  }
+
+  const meta = (obj as any)[ATTR_META];
+  return {
+    system,
+    meta: {
+      keys: meta?.keys ?? [],
+      ...(meta?.tags ? { tags: meta.tags } : {}),
+    },
+    data,
+  };
+};
+
+/**
+ * Build a new {@link DatabaseDirectory} containing every object from the archive,
+ * keyed by object id. The caller is responsible for stamping the `access.spaceKey`
+ * and version fields after the document is created.
+ */
+export const buildDatabaseDirectoryFromObjects = (objects: readonly Obj.JSON[]): DatabaseDirectory => {
+  const map: Record<string, ObjectStructure> = {};
+  for (const obj of objects) {
+    map[obj.id] = objJsonToObjectStructure(obj);
+  }
+  return {
+    objects: map,
+    links: {},
+  };
+};