@dxos/client-services 0.5.9-main.bdf733d → 0.5.9-main.bf3bb8f

package/src/packlets/spaces/epoch-migrations.ts

@@ -0,0 +1,154 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import type { AutomergeUrl } from '@dxos/automerge/automerge-repo';
+import { type Context } from '@dxos/context';
+import {
+  convertLegacyReferences,
+  convertLegacySpaceRootDoc,
+  findInlineObjectOfType,
+  migrateDocument,
+  type EchoHost,
+} from '@dxos/echo-db';
+import { SpaceDocVersion, type SpaceDoc } from '@dxos/echo-protocol';
+import { TYPE_PROPERTIES } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+import type { PublicKey, SpaceId } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { CreateEpochRequest } from '@dxos/protocols/proto/dxos/client/services';
+
+export type MigrationContext = {
+  echoHost: EchoHost;
+
+  spaceId: SpaceId;
+  /**
+   * @deprecated Remove.
+   */
+  spaceKey: PublicKey;
+  migration: CreateEpochRequest.Migration;
+  currentRoot: string | null;
+
+  /**
+   * For set automerge root migration type.
+   */
+  newAutomergeRoot?: string;
+};
+
+export type MigrationResult = {
+  newRoot?: string;
+};
+
+const LOAD_DOC_TIMEOUT = 10_000;
+
+export const runEpochMigration = async (ctx: Context, context: MigrationContext): Promise<MigrationResult> => {
+  switch (context.migration) {
+    case CreateEpochRequest.Migration.INIT_AUTOMERGE: {
+      const document = context.echoHost.createDoc();
+      await context.echoHost.flush();
+      return { newRoot: document.url };
+    }
+    case CreateEpochRequest.Migration.PRUNE_AUTOMERGE_ROOT_HISTORY: {
+      if (!context.currentRoot) {
+        throw new Error('Space does not have an automerge root');
+      }
+      const rootHandle = await context.echoHost.loadDoc(ctx, context.currentRoot as AutomergeUrl, {
+        timeout: LOAD_DOC_TIMEOUT,
+      });
+
+      const newRoot = context.echoHost.createDoc(rootHandle.docSync());
+      await context.echoHost.flush();
+      return { newRoot: newRoot.url };
+    }
+    case CreateEpochRequest.Migration.FRAGMENT_AUTOMERGE_ROOT: {
+      log.info('Fragmenting');
+
+      const currentRootUrl = context.currentRoot;
+      const rootHandle = await context.echoHost.loadDoc<SpaceDoc>(ctx, currentRootUrl as any, {
+        timeout: LOAD_DOC_TIMEOUT,
+      });
+
+      // Find properties object.
+      const objects = Object.entries((rootHandle.docSync() as SpaceDoc).objects!);
+      const properties = findInlineObjectOfType(rootHandle.docSync() as SpaceDoc, TYPE_PROPERTIES);
+      const otherObjects = objects.filter(([key]) => key !== properties?.[0]);
+      invariant(properties, 'Properties not found');
+
+      // Create a new space doc with the properties object.
+      const newRoot = context.echoHost.createDoc({
+        ...rootHandle.docSync(),
+        objects: Object.fromEntries([properties]),
+      });
+      invariant(typeof newRoot.url === 'string' && newRoot.url.length > 0);
+
+      // Create new automerge documents for all objects.
+      const newLinks: [string, AutomergeUrl][] = [];
+      for (const [id, objData] of otherObjects) {
+        const handle = context.echoHost.createDoc<SpaceDoc>({
+          version: SpaceDocVersion.CURRENT,
+          access: {
+            spaceKey: context.spaceKey.toHex(),
+          },
+          objects: {
+            [id]: objData,
+          },
+        });
+        newLinks.push([id, handle.url]);
+      }
+      newRoot.change((doc: SpaceDoc) => {
+        doc.links ??= {};
+        for (const [id, url] of newLinks) {
+          doc.links[id] = url;
+        }
+      });
+
+      await context.echoHost.flush();
+      return {
+        newRoot: newRoot.url,
+      };
+    }
+    case CreateEpochRequest.Migration.MIGRATE_REFERENCES_TO_DXN: {
+      const currentRootUrl = context.currentRoot;
+      const rootHandle = await context.echoHost.loadDoc<SpaceDoc>(ctx, currentRootUrl as any, {
+        timeout: LOAD_DOC_TIMEOUT,
+      });
+      invariant(rootHandle.docSync(), 'Root doc not found');
+
+      const newRootContent = await convertLegacySpaceRootDoc(structuredClone(rootHandle.docSync()!));
+
+      for (const [id, url] of Object.entries(newRootContent.links ?? {})) {
+        try {
+          const handle = await context.echoHost.loadDoc(ctx, url as any, { timeout: LOAD_DOC_TIMEOUT });
+          invariant(handle.docSync());
+          const newDoc = await convertLegacyReferences(structuredClone(handle.docSync()!));
+          const migratedDoc = migrateDocument(handle.docSync(), newDoc);
+          const newHandle = context.echoHost.createDoc(migratedDoc, { preserveHistory: true });
+          newRootContent.links![id] = newHandle.url;
+        } catch (err) {
+          log.warn('Failed to migrate reference', { id, url, error: err });
+          delete newRootContent.links![id];
+        }
+      }
+
+      const migratedRoot = migrateDocument(rootHandle.docSync(), newRootContent);
+      const newRoot = context.echoHost.createDoc(migratedRoot, { preserveHistory: true });
+
+      await context.echoHost.flush();
+      return {
+        newRoot: newRoot.url,
+      };
+    }
+    // TODO(dmaretskyi): This path doesn't seem to fit here. This is not a migration.
+    case CreateEpochRequest.Migration.REPLACE_AUTOMERGE_ROOT: {
+      invariant(context.newAutomergeRoot);
+
+      // Defensive programming - it should be the responsibility of the caller to flush the new root.
+      await context.echoHost.flush();
+      return {
+        newRoot: context.newAutomergeRoot,
+      };
+    }
+  }
+
+  return {};
+};

package/src/packlets/spaces/spaces-service.ts

@@ -30,6 +30,11 @@ import {
   type UpdateSpaceRequest,
   type WriteCredentialsRequest,
   type UpdateMemberRoleRequest,
+  type AdmitContactRequest,
+  type ContactAdmission,
+  type JoinSpaceResponse,
+  type JoinBySpaceKeyRequest,
+  type CreateEpochResponse,
 } from '@dxos/protocols/proto/dxos/client/services';
 import { type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { type GossipMessage } from '@dxos/protocols/proto/dxos/mesh/teleport/gossip';
@@ -125,8 +130,18 @@ export class SpacesServiceImpl implements SpacesService {
           subscriptions.clear();
 
           for (const space of dataSpaceManager.spaces.values()) {
-            // TODO(dmaretskyi): This can skip updates and not report intermediate states. Potential race condition here.
-            subscriptions.add(space.stateUpdate.on(ctx, () => scheduler.forceTrigger()));
+            let lastState: SpaceState | undefined;
+            subscriptions.add(
+              space.stateUpdate.on(ctx, () => {
+                // Always send a separate update if the space state has changed.
+                if (space.state !== lastState) {
+                  scheduler.forceTrigger();
+                } else {
+                  scheduler.trigger();
+                }
+                lastState = space.state;
+              }),
+            );
 
             subscriptions.add(space.presence.updated.on(ctx, () => scheduler.trigger()));
             subscriptions.add(space.automergeSpaceState.onNewEpoch.on(ctx, () => scheduler.trigger()));
@@ -208,10 +223,45 @@ export class SpacesServiceImpl implements SpacesService {
     }
   }
 
-  async createEpoch({ spaceKey, migration }: CreateEpochRequest) {
+  async createEpoch({ spaceKey, migration, automergeRootUrl }: CreateEpochRequest): Promise<CreateEpochResponse> {
     const dataSpaceManager = await this._getDataSpaceManager();
     const space = dataSpaceManager.spaces.get(spaceKey) ?? raise(new SpaceNotFoundError(spaceKey));
-    await space.createEpoch({ migration });
+    const credential = await space.createEpoch({ migration, newAutomergeRoot: automergeRootUrl });
+    return { epochCredential: credential ?? undefined };
+  }
+
+  async admitContact(request: AdmitContactRequest): Promise<void> {
+    const dataSpaceManager = await this._getDataSpaceManager();
+    await dataSpaceManager.admitMember({
+      spaceKey: request.spaceKey,
+      identityKey: request.contact.identityKey,
+      role: request.role,
+    });
+  }
+
+  async joinBySpaceKey({ spaceKey }: JoinBySpaceKeyRequest): Promise<JoinSpaceResponse> {
+    const dataSpaceManager = await this._getDataSpaceManager();
+    const credential = await dataSpaceManager.requestSpaceAdmissionCredential(spaceKey);
+    return this._joinByAdmission({ credential });
+  }
+
+  private async _joinByAdmission({ credential }: ContactAdmission): Promise<JoinSpaceResponse> {
+    const assertion = getCredentialAssertion(credential);
+    invariant(assertion['@type'] === 'dxos.halo.credentials.SpaceMember', 'Invalid credential');
+    const myIdentity = this._identityManager.identity;
+    invariant(myIdentity && credential.subject.id.equals(myIdentity.identityKey));
+
+    const dataSpaceManager = await this._getDataSpaceManager();
+    let dataSpace = dataSpaceManager.spaces.get(assertion.spaceKey);
+    if (!dataSpace) {
+      dataSpace = await dataSpaceManager.acceptSpace({
+        spaceKey: assertion.spaceKey,
+        genesisFeedKey: assertion.genesisFeedKey,
+      });
+      await myIdentity.controlPipeline.writer.write({ credential: { credential } });
+    }
+
+    return { space: this._serializeSpace(dataSpace) };
   }
 
   private _serializeSpace(space: DataSpace): Space {
@@ -234,6 +284,8 @@ export class SpacesServiceImpl implements SpacesService {
         currentDataTimeframe: undefined,
         targetDataTimeframe: undefined,
         totalDataTimeframe: undefined,
+
+        spaceRootUrl: space.databaseRoot?.url,
       },
       members: Array.from(space.inner.spaceState.members.values()).map((member) => {
         const peers = space.presence.getPeersOnline().filter(({ identityKey }) => identityKey.equals(member.key));

package/src/packlets/storage/index.ts

@@ -4,3 +4,4 @@
 
 export * from './storage';
 export * from './level';
+export * from './profile-archive';

package/src/packlets/storage/profile-archive.ts

@@ -0,0 +1,111 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { cbor } from '@dxos/automerge/automerge-repo';
+import { invariant } from '@dxos/invariant';
+import type { LevelDB } from '@dxos/kv-store';
+import { log } from '@dxos/log';
+import { ProfileArchiveEntryType, type ProfileArchive } from '@dxos/protocols';
+import type { Storage } from '@dxos/random-access-storage';
+import { arrayToBuffer } from '@dxos/util';
+
+export const encodeProfileArchive = (profile: ProfileArchive): Uint8Array => cbor.encode(profile);
+
+export const decodeProfileArchive = (data: Uint8Array): ProfileArchive => cbor.decode(data);
+
+export const exportProfileData = async ({
+  storage,
+  level,
+}: {
+  storage: Storage;
+  level: LevelDB;
+}): Promise<ProfileArchive> => {
+  const archive: ProfileArchive = { storage: [], meta: { timestamp: new Date().toISOString() } };
+
+  {
+    const directory = await storage.createDirectory();
+    const files = await directory.list();
+
+    log.info('begin exporting files', { count: files.length });
+    for (const filename of files) {
+      const file = await directory.getOrCreateFile(filename);
+      const { size } = await file.stat();
+      const data = await file.read(0, size);
+      archive.storage.push({
+        type: ProfileArchiveEntryType.FILE,
+        key: filename,
+        value: data,
+      });
+    }
+    log.info('done exporting files', { count: files.length });
+  }
+
+  {
+    log.info('begin exporting kv pairs');
+    const iter = await level.iterator<Uint8Array, Uint8Array>({ keyEncoding: 'binary', valueEncoding: 'binary' });
+    let count = 0;
+    for await (const [key, value] of iter) {
+      archive.storage.push({
+        type: ProfileArchiveEntryType.KEY_VALUE,
+        key,
+        value,
+      });
+      count++;
+    }
+    log.info('done exporting kv pairs', { count });
+  }
+
+  return archive;
+};
+
+export const importProfileData = async (
+  {
+    storage,
+    level,
+  }: {
+    storage: Storage;
+    level: LevelDB;
+  },
+  archive: ProfileArchive,
+): Promise<void> => {
+  let batch = level.batch();
+
+  let count = 0;
+  for (const entry of archive.storage) {
+    switch (entry.type) {
+      case ProfileArchiveEntryType.FILE: {
+        const directory = await storage.createDirectory();
+        invariant(typeof entry.key === 'string', 'Invalid key type');
+        const file = await directory.getOrCreateFile(entry.key);
+        invariant(entry.value instanceof Uint8Array, 'Invalid value type');
+        await file.write(0, arrayToBuffer(entry.value));
+        await file.close();
+        break;
+      }
+      case ProfileArchiveEntryType.KEY_VALUE: {
+        invariant(entry.key instanceof Uint8Array, 'Invalid key type');
+        invariant(entry.value instanceof Uint8Array, 'Invalid value type');
+        batch.put(entry.key, entry.value, { keyEncoding: 'binary', valueEncoding: 'binary' });
+        break;
+      }
+      default:
+        throw new Error(`Invalid entry type: ${entry.type}`);
+    }
+
+    if (++count % 1000 === 0) {
+      // Apparently indexedDB can't handle big batches.
+      await batch.write();
+      batch = level.batch();
+
+      log.info('importing', {
+        count,
+        total: archive.storage.length,
+        progress: `${((count / archive.storage.length) * 100).toFixed()}%`,
+      });
+    }
+  }
+
+  log.info('committing changes..');
+  await batch.write();
+};

package/src/packlets/testing/test-builder.ts

@@ -19,8 +19,8 @@ import { createStorage, StorageType, type Storage } from '@dxos/random-access-st
 import { BlobStore } from '@dxos/teleport-extension-object-sync';
 
 import { InvitationsHandler, InvitationsManager, SpaceInvitationProtocol } from '../invitations';
-import { ClientServicesHost, ServiceContext } from '../services';
-import { DataSpaceManager, type SigningContext } from '../spaces';
+import { ClientServicesHost, ServiceContext, type ServiceContextRuntimeParams } from '../services';
+import { DataSpaceManager, type DataSpaceManagerRuntimeParams, type SigningContext } from '../spaces';
 
 //
 // TODO(burdon): Replace with test builder.
@@ -37,9 +37,11 @@ export const createServiceHost = (config: Config, signalManagerContext: MemorySi
 export const createServiceContext = async ({
   signalContext = new MemorySignalManagerContext(),
   storage = createStorage({ type: StorageType.RAM }),
+  runtimeParams,
 }: {
   signalContext?: MemorySignalManagerContext;
   storage?: Storage;
+  runtimeParams?: ServiceContextRuntimeParams;
 } = {}) => {
   const signalManager = new MemorySignalManager(signalContext);
   const networkManager = new SwarmNetworkManager({
@@ -51,6 +53,7 @@ export const createServiceContext = async ({
 
   return new ServiceContext(storage, level, networkManager, signalManager, {
     invitationConnectionDefaultParams: { controlHeartbeatInterval: 200 },
+    ...runtimeParams,
   });
 };
 
@@ -88,6 +91,7 @@ export class TestBuilder {
 
 export type TestPeerOpts = {
   dataStore?: StorageType;
+  dataSpaceParams?: DataSpaceManagerRuntimeParams;
 };
 
 export type TestPeerProps = {
@@ -110,8 +114,8 @@ export class TestPeer {
   private _props: TestPeerProps = {};
 
   constructor(
-    private readonly signalContext: MemorySignalManagerContext,
-    private readonly opts: TestPeerOpts = { dataStore: StorageType.RAM },
+    private readonly _signalContext: MemorySignalManagerContext,
+    private readonly _opts: TestPeerOpts = { dataStore: StorageType.RAM },
   ) {}
 
   get props() {
@@ -119,7 +123,7 @@ export class TestPeer {
   }
 
   get storage() {
-    return (this._props.storage ??= createStorage({ type: this.opts.dataStore }));
+    return (this._props.storage ??= createStorage({ type: this._opts.dataStore }));
   }
 
   get keyring() {
@@ -156,7 +160,7 @@ export class TestPeer {
 
   get networkManager() {
     return (this._props.networkManager ??= new SwarmNetworkManager({
-      signalManager: new MemorySignalManager(this.signalContext),
+      signalManager: new MemorySignalManager(this._signalContext),
       transportFactory: MemoryTransportFactory,
     }));
   }
@@ -176,10 +180,7 @@ export class TestPeer {
   }
 
   get echoHost() {
-    return (this._props.echoHost ??= new EchoHost({
-      kv: this.level,
-      storage: this.storage,
-    }));
+    return (this._props.echoHost ??= new EchoHost({ kv: this.level }));
   }
 
   get dataSpaceManager(): DataSpaceManager {
@@ -191,6 +192,7 @@ export class TestPeer {
       this.feedStore,
       this.echoHost,
       this.invitationsManager,
+      this._opts.dataSpaceParams,
     ));
   }
 

package/src/version.ts

@@ -1 +1 @@
-export const DXOS_VERSION = "0.5.9-main.bdf733d";
+export const DXOS_VERSION = "0.5.9-main.bf3bb8f";