@dxos/client-services 0.8.4-main.b97322e → 0.8.4-main.bcb3aa67d6

package/src/packlets/services/service-host.ts

@@ -2,39 +2,43 @@
 // Copyright 2021 DXOS.org
 //
 
+import * as SqlClient from '@effect/sql/SqlClient';
+import * as Effect from 'effect/Effect';
+
 import { Event, synchronized } from '@dxos/async';
-import { clientServiceBundle, type ClientServices } from '@dxos/client-protocol';
+import { type ClientServices, clientServiceBundle } from '@dxos/client-protocol';
 import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
-import { EdgeClient, EdgeHttpClient, createStubEdgeIdentity, type EdgeConnection } from '@dxos/edge-client';
+import { EdgeClient, type EdgeConnection, EdgeHttpClient, createStubEdgeIdentity } from '@dxos/edge-client';
+import { RuntimeProvider } from '@dxos/effect';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { type LevelDB } from '@dxos/kv-store';
 import { log } from '@dxos/log';
-import { EdgeSignalManager, WebsocketSignalManager, type SignalManager } from '@dxos/messaging';
+import { EdgeSignalManager, type SignalManager, WebsocketSignalManager } from '@dxos/messaging';
 import {
   SwarmNetworkManager,
+  type TransportFactory,
   createIceProvider,
   createRtcTransportFactory,
-  type TransportFactory,
 } from '@dxos/network-manager';
 import { trace } from '@dxos/protocols';
 import { SystemStatus } from '@dxos/protocols/proto/dxos/client/services';
 import { type Storage } from '@dxos/random-access-storage';
+import * as SqlExport from '@dxos/sql-sqlite/SqlExport';
+import type * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
 import { TRACE_PROCESSOR, trace as Trace } from '@dxos/tracing';
 import { WebsocketRpcClient } from '@dxos/websocket-rpc';
 
-import { ServiceContext, type ServiceContextRuntimeParams } from './service-context';
-import { ServiceRegistry } from './service-registry';
 import { EdgeAgentServiceImpl } from '../agents';
 import { DevicesServiceImpl } from '../devices';
 import { DevtoolsHostEvents, DevtoolsServiceImpl } from '../devtools';
 import {
+  type CollectDiagnosticsBroadcastHandler,
   createCollectDiagnosticsBroadcastHandler,
   createDiagnostics,
-  type CollectDiagnosticsBroadcastHandler,
 } from '../diagnostics';
-import { IdentityServiceImpl, type CreateIdentityOptions } from '../identity';
+import { type CreateIdentityOptions, IdentityServiceImpl } from '../identity';
 import { ContactsServiceImpl } from '../identity/contacts-service';
 import { InvitationsServiceImpl } from '../invitations';
 import { Lock, type ResourceLock } from '../locks';
@@ -44,7 +48,10 @@ import { SpacesServiceImpl } from '../spaces';
 import { createLevel, createStorageObjects } from '../storage';
 import { SystemServiceImpl } from '../system';
 
-export type ClientServicesHostParams = {
+import { ServiceContext, type ServiceContextRuntimeProps } from './service-context';
+import { ServiceRegistry } from './service-registry';
+
+export type ClientServicesHostProps = {
   /**
    * Can be omitted if `initialize` is later called.
    */
@@ -56,7 +63,8 @@ export type ClientServicesHostParams = {
   level?: LevelDB;
   lockKey?: string;
   callbacks?: ClientServicesHostCallbacks;
-  runtimeParams?: ServiceContextRuntimeParams;
+  runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlExport.SqlExport | SqlTransaction.SqlTransaction>;
+  runtimeProps?: ServiceContextRuntimeProps;
 };
 
 export type ClientServicesHostCallbacks = {
@@ -94,7 +102,10 @@ export class ClientServicesHost {
   private _edgeHttpClient?: EdgeHttpClient = undefined;
 
   private _serviceContext!: ServiceContext;
-  private readonly _runtimeParams: ServiceContextRuntimeParams;
+  private readonly _runtime: RuntimeProvider.RuntimeProvider<
+    SqlClient.SqlClient | SqlExport.SqlExport | SqlTransaction.SqlTransaction
+  >;
+  private readonly _runtimeProps: ServiceContextRuntimeProps;
   private diagnosticsBroadcastHandler: CollectDiagnosticsBroadcastHandler;
 
   @Trace.info()
@@ -115,20 +126,14 @@ export class ClientServicesHost {
     // TODO(wittjosiah): Turn this on by default.
     lockKey,
     callbacks,
-    runtimeParams,
-  }: ClientServicesHostParams = {}) {
+    runtime,
+    runtimeProps,
+  }: ClientServicesHostProps) {
     this._storage = storage;
     this._level = level;
     this._callbacks = callbacks;
-    this._runtimeParams = runtimeParams ?? {};
-
-    if (this._runtimeParams.disableP2pReplication === undefined) {
-      this._runtimeParams.disableP2pReplication = config?.get('runtime.client.disableP2pReplication', false);
-    }
-
-    if (this._runtimeParams.enableVectorIndexing === undefined) {
-      this._runtimeParams.enableVectorIndexing = config?.get('runtime.client.enableVectorIndexing', false);
-    }
+    this._runtime = runtime;
+    this._runtimeProps = runtimeProps ?? {};
 
     if (config) {
       this.initialize({ config, transportFactory, signalManager });
@@ -142,7 +147,7 @@ export class ClientServicesHost {
             void this.open(new Context());
           }
         },
-        onRelease: () => this.close(),
+        onRelease: () => this.close(Context.default()),
       });
     }
 
@@ -199,6 +204,30 @@ export class ClientServicesHost {
     return this._serviceRegistry.services;
   }
 
+  /**
+   * Debugging util.
+   */
+  async exportSqliteDatabase(): Promise<Uint8Array> {
+    return await RuntimeProvider.runPromise(this._runtime)(
+      Effect.gen(function* () {
+        const sql = yield* SqlExport.SqlExport;
+        return yield* sql.export;
+      }),
+    );
+  }
+
+  /**
+   * Debugging util.
+   */
+  async runSqliteQuery(query: string, params?: any[]): Promise<readonly Record<string, unknown>[]> {
+    return await RuntimeProvider.runPromise(this._runtime)(
+      Effect.gen(function* () {
+        const sql = yield* SqlClient.SqlClient;
+        return yield* sql`${sql.unsafe(query, params)}`;
+      }),
+    );
+  }
+
   /**
    * Initialize the service host with the config.
    * Config can also be provided in the constructor.
@@ -209,6 +238,13 @@ export class ClientServicesHost {
     log('initializing...');
 
     if (config) {
+      if (this._runtimeProps.disableP2pReplication === undefined) {
+        this._runtimeProps.disableP2pReplication = config?.get('runtime.client.disableP2pReplication', false);
+      }
+      if (this._runtimeProps.enableVectorIndexing === undefined) {
+        this._runtimeProps.enableVectorIndexing = config?.get('runtime.client.enableVectorIndexing', false);
+      }
+
       invariant(!this._config, 'config already set');
       this._config = config;
       if (!this._storage) {
@@ -216,14 +252,16 @@ export class ClientServicesHost {
       }
     }
 
+    // TODO(wittjosiah): This is quite noisy during tests. Make configurable? Remove?
     if (!options.signalManager) {
-      log.warn('running signaling without telemetry metadata.');
+      // log.warn('running signaling without telemetry metadata.');
     }
 
     const endpoint = config?.get('runtime.services.edge.url');
     if (endpoint) {
+      const clientTag = config?.get('runtime.app.env.DX_EDGE_CLIENT_TAG');
       this._edgeConnection = new EdgeClient(createStubEdgeIdentity(), { socketEndpoint: endpoint });
-      this._edgeHttpClient = new EdgeHttpClient(endpoint);
+      this._edgeHttpClient = new EdgeHttpClient(endpoint, { clientTag });
     }
 
     const {
@@ -289,7 +327,8 @@ export class ClientServicesHost {
       this._signalManager,
       this._edgeConnection,
       this._edgeHttpClient,
-      this._runtimeParams,
+      this._runtime,
+      this._runtimeProps,
       this._config.get('runtime.client.edgeFeatures'),
     );
 
@@ -307,7 +346,6 @@ export class ClientServicesHost {
       this._serviceContext.identityManager,
       this._serviceContext.recoveryManager,
       this._serviceContext.keyring,
-      () => this._serviceContext.dataSpaceManager!,
       (params) => this._createIdentity(params),
       (profile) => this._serviceContext.broadcastProfileUpdate(profile),
     );
@@ -333,6 +371,7 @@ export class ClientServicesHost {
 
       DataService: this._serviceContext.echoHost.dataService,
       QueryService: this._serviceContext.echoHost.queryService,
+      QueueService: this._serviceContext.echoHost.queuesService,
 
       NetworkService: new NetworkServiceImpl(
         this._serviceContext.networkManager,
@@ -353,8 +392,13 @@ export class ClientServicesHost {
       EdgeAgentService: new EdgeAgentServiceImpl(agentManagerProvider, this._edgeConnection),
     });
 
+    log('service-host: opening service context...');
     await this._serviceContext.open(ctx);
+    log('service-host: service context opened');
+
+    log('service-host: opening identity service...');
     await identityService.open();
+    log('service-host: identity service opened');
 
     const devtoolsProxy = this._config?.get('runtime.client.devtoolsProxy');
     if (devtoolsProxy) {
@@ -378,7 +422,7 @@ export class ClientServicesHost {
 
   @synchronized
   @Trace.span()
-  async close(): Promise<void> {
+  async close(ctx: Context): Promise<void> {
     if (!this._open) {
       return;
     }
@@ -406,6 +450,12 @@ export class ClientServicesHost {
     this._resetting = true;
     this._statusUpdate.emit();
     await this._serviceContext?.close();
+    // Clear LevelDB contents to remove all persisted Echo/Automerge/index data.
+    try {
+      await this._level!.clear();
+    } catch (err) {
+      log.warn('failed to clear leveldb during reset', { err });
+    }
     await this._storage!.reset();
     log.info('reset');
     log.trace('dxos.sdk.client-services-host.reset', trace.end({ id: traceId }));

package/src/packlets/services/service-registry.test.ts

@@ -13,10 +13,11 @@ import { schema } from '@dxos/protocols/proto';
 import { type SystemService, SystemStatus } from '@dxos/protocols/proto/dxos/client/services';
 import { createLinkedPorts, createProtoRpcPeer, createServiceBundle } from '@dxos/rpc';
 
-import { ServiceRegistry } from './service-registry';
 import { SystemServiceImpl } from '../system';
 import { createServiceContext } from '../testing';
 
+import { ServiceRegistry } from './service-registry';
+
 // TODO(burdon): Create TestService (that doesn't require peers).
 
 type TestServices = {

package/src/packlets/space-export/space-archive-reader.ts

@@ -6,19 +6,33 @@ import type { DocumentId } from '@automerge/automerge-repo';
 
 import { assertArgument, failedInvariant, invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
-import { SpaceArchiveFileStructure, type SpaceArchiveMetadata } from '@dxos/protocols';
+import {
+  type FeedArchiveBlock,
+  type FeedArchiveMetadata,
+  SpaceArchiveFileStructure,
+  type SpaceArchiveMetadata,
+} from '@dxos/protocols';
 import type { SpaceArchive } from '@dxos/protocols/proto/dxos/client/services';
 
+/**
+ * Extracted feed data from the archive.
+ */
+export type ExtractedFeed = {
+  metadata: FeedArchiveMetadata;
+  blocks: FeedArchiveBlock[];
+};
+
 export type ExtractedSpaceArchive = {
   metadata: SpaceArchiveMetadata;
   documents: Record<DocumentId, Uint8Array>;
+  feeds: Record<string, ExtractedFeed>;
 };
 
 export const extractSpaceArchive = async (archive: SpaceArchive): Promise<ExtractedSpaceArchive> => {
   const { Archive } = await import('@obsidize/tar-browserify');
   const { entries } = await Archive.extract(archive.contents);
   const metadataEntry = entries.find((entry) => entry.fileName === SpaceArchiveFileStructure.metadata);
-  assertArgument(metadataEntry, 'Metadata entry not found');
+  assertArgument(metadataEntry, 'metadataEntry', 'Metadata entry not found');
   const metadata = JSON.parse(metadataEntry.getContentAsText());
   const documents: Record<DocumentId, Uint8Array> = {};
   for (const entry of entries.filter((entry) => entry.fileName.startsWith(`${SpaceArchiveFileStructure.documents}/`))) {
@@ -29,6 +43,53 @@ export const extractSpaceArchive = async (archive: SpaceArchive): Promise<Extrac
     documents[documentId] = entry.content ?? failedInvariant();
   }
 
-  log.info('extracted space archive', { metadata, documents });
-  return { metadata, documents };
+  const feeds: Record<string, ExtractedFeed> = {};
+  const feedsPrefix = `${SpaceArchiveFileStructure.feeds}/`;
+  const feedEntries = entries.filter((entry) => entry.fileName.startsWith(feedsPrefix));
+
+  const feedMetadataByFeedId = new Map<string, FeedArchiveMetadata>();
+  const feedBlocksByFeedId = new Map<string, Map<number, FeedArchiveBlock[]>>();
+
+  for (const entry of feedEntries) {
+    const relativePath = entry.fileName.slice(feedsPrefix.length);
+    const pathParts = relativePath.split('/');
+    if (pathParts.length !== 2) {
+      continue;
+    }
+
+    const [feedId, fileName] = pathParts;
+    invariant(feedId, 'Feed ID is required');
+    invariant(fileName, 'File name is required');
+
+    if (fileName === SpaceArchiveFileStructure.feedMetadata) {
+      const feedMetadata = JSON.parse(entry.getContentAsText()) as FeedArchiveMetadata;
+      feedMetadataByFeedId.set(feedId, feedMetadata);
+    } else if (fileName.startsWith(SpaceArchiveFileStructure.feedBlocksPrefix) && fileName.endsWith('.json')) {
+      const chunkIndexStr = fileName.slice(SpaceArchiveFileStructure.feedBlocksPrefix.length).replace(/\.json$/, '');
+      const chunkIndex = parseInt(chunkIndexStr, 10);
+      invariant(!isNaN(chunkIndex), `Invalid chunk index: ${chunkIndexStr}`);
+
+      const blocks = JSON.parse(entry.getContentAsText()) as FeedArchiveBlock[];
+      if (!feedBlocksByFeedId.has(feedId)) {
+        feedBlocksByFeedId.set(feedId, new Map());
+      }
+      feedBlocksByFeedId.get(feedId)!.set(chunkIndex, blocks);
+    }
+  }
+
+  for (const [feedId, feedMetadata] of feedMetadataByFeedId) {
+    const blockChunks = feedBlocksByFeedId.get(feedId) ?? new Map<number, FeedArchiveBlock[]>();
+    const sortedChunkIndices = Array.from(blockChunks.keys()).sort((a, b) => a - b);
+    const allBlocks: FeedArchiveBlock[] = [];
+    for (const chunkIndex of sortedChunkIndices) {
+      allBlocks.push(...blockChunks.get(chunkIndex)!);
+    }
+    feeds[feedId] = {
+      metadata: feedMetadata,
+      blocks: allBlocks,
+    };
+  }
+
+  log('extracted space archive', { metadata, documents, feedCount: Object.keys(feeds).length });
+  return { metadata, documents, feeds };
 };

package/src/packlets/space-export/space-archive-writer.ts

@@ -4,11 +4,19 @@
 
 import type * as tar from '@obsidize/tar-browserify';
 
-import { Resource, type Context } from '@dxos/context';
+import { type Context, Resource } from '@dxos/context';
 import { assertArgument, assertState } from '@dxos/invariant';
 import type { IdentityDid, SpaceId } from '@dxos/keys';
-import { SpaceArchiveFileStructure, SpaceArchiveVersion, type SpaceArchiveMetadata } from '@dxos/protocols';
+import {
+  FEED_ARCHIVE_BLOCKS_PER_CHUNK,
+  type FeedArchiveBlock,
+  type FeedArchiveMetadata,
+  SpaceArchiveFileStructure,
+  type SpaceArchiveMetadata,
+  SpaceArchiveVersion,
+} from '@dxos/protocols';
 import type { SpaceArchive } from '@dxos/protocols/proto/dxos/client/services';
+import { createFilename } from '@dxos/util';
 
 export type SpaceArchiveBeginProps = {
   spaceId?: SpaceId;
@@ -44,21 +52,50 @@ export class SpaceArchiveWriter extends Resource {
   }
 
   async setCurrentRootUrl(url: string): Promise<void> {
-    assertArgument(url.startsWith('automerge:'), 'Invalid root URL');
+    assertArgument(url.startsWith('automerge:'), 'url', 'Invalid root URL');
     assertState(this._tar, 'Not open');
     assertState(this._meta, 'Not started');
     this._currentRootUrl = url;
   }
 
   async writeDocument(documentId: string, data: Uint8Array): Promise<void> {
-    assertArgument(!documentId.startsWith('automerge:'), 'Invalid document ID');
+    assertArgument(!documentId.startsWith('automerge:'), 'documentId', 'Invalid document ID');
     assertState(this._archive, 'Not open');
     this._archive.addBinaryFile(`${SpaceArchiveFileStructure.documents}/${documentId}.bin`, data);
   }
 
+  /**
+   * Writes a feed with its metadata and blocks to the archive.
+   * Blocks are written in chunks of {@link FEED_ARCHIVE_BLOCKS_PER_CHUNK}.
+   */
+  async writeFeed(feedId: string, namespace: string, blocks: FeedArchiveBlock[]): Promise<void> {
+    assertArgument(feedId, 'feedId', 'Feed ID is required');
+    assertArgument(namespace, 'namespace', 'Namespace is required');
+    assertState(this._archive, 'Not open');
+
+    const feedPath = `${SpaceArchiveFileStructure.feeds}/${feedId}`;
+
+    const metadata: FeedArchiveMetadata = {
+      id: feedId,
+      namespace,
+    };
+    this._archive.addTextFile(`${feedPath}/${SpaceArchiveFileStructure.feedMetadata}`, JSON.stringify(metadata));
+
+    for (let chunkIndex = 0; chunkIndex * FEED_ARCHIVE_BLOCKS_PER_CHUNK < blocks.length; chunkIndex++) {
+      const start = chunkIndex * FEED_ARCHIVE_BLOCKS_PER_CHUNK;
+      const end = Math.min(start + FEED_ARCHIVE_BLOCKS_PER_CHUNK, blocks.length);
+      const chunk = blocks.slice(start, end);
+      this._archive.addTextFile(
+        `${feedPath}/${SpaceArchiveFileStructure.feedBlocksPrefix}${chunkIndex}.json`,
+        JSON.stringify(chunk),
+      );
+    }
+  }
+
   async finish(): Promise<SpaceArchive> {
     assertState(this._archive, 'Not open');
     assertState(this._meta, 'Not started');
+    assertState(this._meta.spaceId, 'No space ID set');
     assertState(this._currentRootUrl, 'No root URL set');
 
     const metadata: SpaceArchiveMetadata = {
@@ -76,7 +113,7 @@ export class SpaceArchiveWriter extends Resource {
     const binary = this._archive.toUint8Array();
 
     return {
-      filename: `${this._meta.spaceId}.tar`,
+      filename: createFilename({ parts: [this._meta.spaceId], ext: 'tar' }),
       contents: binary,
     };
   }