@dxos/client-services 0.4.9-main.a108ade → 0.4.9-main.a3cfaa7

package/src/packlets/spaces/data-space-manager.test.ts

@@ -3,24 +3,16 @@
 //
 
 import { expect } from 'chai';
-import path from 'node:path';
 
 import { asyncTimeout, latch } from '@dxos/async';
-import { type SpecificCredential, createAdmissionCredentials } from '@dxos/credentials';
+import { createAdmissionCredentials } from '@dxos/credentials';
 import { AuthStatus } from '@dxos/echo-pipeline';
-import { testLocalDatabase } from '@dxos/echo-pipeline/testing';
 import { writeMessages } from '@dxos/feed-store';
-import { type PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { SpaceState } from '@dxos/protocols/proto/dxos/client/services';
-import { type SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
-import { type Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
-import { StorageType } from '@dxos/random-access-storage';
-import { afterTest, describe, openAndClose, test } from '@dxos/test';
-import { Timeframe } from '@dxos/timeframe';
-import { range } from '@dxos/util';
+import { describe, openAndClose, test } from '@dxos/test';
 
-import { TestBuilder, syncItemsLocal } from '../testing';
+import { TestBuilder } from '../testing';
 
 describe('DataSpaceManager', () => {
   test('create space', async () => {
@@ -103,8 +95,6 @@ describe('DataSpaceManager', () => {
     });
     log.break();
 
-    await syncItemsLocal(space1.dataPipeline, space2.dataPipeline);
-
     expect(space1.inner.protocol.sessions.get(peer2.identity.deviceKey)).to.exist;
     expect(space1.inner.protocol.sessions.get(peer2.identity.deviceKey)?.authStatus).to.equal(AuthStatus.SUCCESS);
     expect(space2.inner.protocol.sessions.get(peer1.identity.deviceKey)).to.exist;
@@ -157,137 +147,6 @@ describe('DataSpaceManager', () => {
     await receivedMessage();
   });
 
-  describe('Epochs', () => {
-    test
-      .skip('Epoch truncates feeds', async () => {
-        const builder = new TestBuilder();
-        afterTest(async () => builder.destroy());
-
-        const peer = builder.createPeer({
-          dataStore: typeof window === 'undefined' ? StorageType.NODE : StorageType.WEBFS,
-        });
-        await peer.createIdentity();
-        await openAndClose(peer.dataSpaceManager);
-        const space = await peer.dataSpaceManager.createSpace();
-        await space.inner.controlPipeline.state.waitUntilTimeframe(space.inner.controlPipeline.state.endTimeframe);
-
-        const feedDataPath = path.join(space.inner.dataPipeline.pipelineState!.feeds[0].key.toHex(), 'data');
-        const directory = peer.storage.createDirectory('feeds');
-        const file = directory.getOrCreateFile(feedDataPath);
-        afterTest(() => file.close());
-
-        expect((await file.stat()).size === 0).to.be.true;
-
-        for (const _ in range(10)) {
-          await testLocalDatabase(space.dataPipeline);
-        }
-
-        expect((await file.stat()).size !== 0).to.be.true;
-        await space.createEpoch();
-        expect((await file.stat()).size === 0).to.be.true;
-      })
-      .onlyEnvironments('nodejs', 'chromium', 'firefox')
-      .tag('flaky');
-
-    test('Loads only last epoch', async () => {
-      const builder = new TestBuilder();
-      afterTest(async () => builder.destroy());
-
-      const peer = builder.createPeer();
-      await peer.createIdentity();
-      const epochsNumber = 10;
-      let spaceKey: PublicKey;
-      {
-        // Create space and create epochs in it.s
-        await openAndClose(peer.dataSpaceManager);
-
-        const space = await peer.dataSpaceManager.createSpace();
-        spaceKey = space.key;
-        await space.inner.controlPipeline.state.waitUntilTimeframe(space.inner.controlPipeline.state.endTimeframe);
-
-        for (const _ of range(epochsNumber)) {
-          await testLocalDatabase(space.dataPipeline);
-          await space.createEpoch();
-        }
-        await space.close();
-        await peer.dataSpaceManager.close();
-        peer.props.dataSpaceManager = undefined;
-      }
-      {
-        // Load same space and check if it loads only last epoch.s
-        await openAndClose(peer.dataSpaceManager);
-
-        const space = peer.dataSpaceManager.spaces.get(spaceKey)!;
-
-        const epochs: number[] = [];
-        space.dataPipeline.onNewEpoch.on((epoch: SpecificCredential<Epoch>) => {
-          epochs.push(epoch.subject.assertion.number);
-        });
-        const processedFirstEpoch = space.dataPipeline.onNewEpoch.waitFor(() => true);
-
-        await space.inner.controlPipeline.state.waitUntilTimeframe(space.inner.controlPipeline.state.endTimeframe);
-
-        await processedFirstEpoch;
-        expect(epochs).to.deep.equal([epochsNumber]);
-      }
-    });
-
-    test('Items are cleared before epoch applied', async () => {
-      const builder = new TestBuilder();
-      afterTest(async () => builder.destroy());
-      const peer = builder.createPeer();
-      await peer.createIdentity();
-      await openAndClose(peer.dataSpaceManager);
-
-      // Create space and fill it with Items.
-      const space = await peer.dataSpaceManager.createSpace();
-      await space.inner.controlPipeline.state.waitUntilTimeframe(space.inner.controlPipeline.state.endTimeframe);
-      const itemsNumber = 2;
-      for (const _ of range(itemsNumber)) {
-        await testLocalDatabase(space.dataPipeline);
-      }
-
-      // Create empty Epoch and check if it clears items.
-      {
-        const processedFirstEpoch = space.dataPipeline.onNewEpoch.waitFor(
-          (epoch) => epoch.subject.assertion.number === 1,
-        );
-
-        // Empty snapshot.
-        const snapshot: SpaceSnapshot = {
-          spaceKey: space.key.asUint8Array(),
-          timeframe: space.inner.dataPipeline.pipelineState!.timeframe,
-          database: {},
-        };
-
-        const snapshotCid = await peer.snapshotStore.saveSnapshot(snapshot);
-
-        const epoch: Epoch = {
-          previousId: space.dataPipeline.currentEpoch?.id,
-          timeframe: space.inner.dataPipeline.pipelineState!.timeframe,
-          number: (space.dataPipeline.currentEpoch?.subject.assertion as Epoch).number + 1,
-          snapshotCid,
-        };
-
-        const receipt = await space.inner.controlPipeline.writer.write({
-          credential: {
-            credential: await peer.props.signingContext!.credentialSigner.createCredential({
-              subject: space.key,
-              assertion: {
-                '@type': 'dxos.halo.credentials.Epoch',
-                ...epoch,
-              },
-            }),
-          },
-        });
-        await space.inner.controlPipeline.state.waitUntilTimeframe(new Timeframe([[receipt.feedKey, receipt.seq]]));
-        await processedFirstEpoch;
-      }
-
-      expect(Array.from(space.dataPipeline.itemManager.entities.keys()).length).to.equal(0);
-    });
-  });
-
   describe('activation', () => {
     test('can activate and deactivate a space', async () => {
       const builder = new TestBuilder();
@@ -308,7 +167,6 @@ describe('DataSpaceManager', () => {
         space.stateUpdate.waitForCondition(() => space.state === SpaceState.READY),
         500,
       );
-      await testLocalDatabase(space.dataPipeline);
     });
   });
 });

package/src/packlets/spaces/data-space-manager.ts

@@ -3,15 +3,9 @@
 //
 
 import { Event, synchronized, trackLeaks } from '@dxos/async';
-import { cancelWithContext, Context } from '@dxos/context';
-import { type CredentialSigner, getCredentialAssertion } from '@dxos/credentials';
-import {
-  type AutomergeHost,
-  type DataServiceSubscriptions,
-  type MetadataStore,
-  type Space,
-  type SpaceManager,
-} from '@dxos/echo-pipeline';
+import { Context, cancelWithContext } from '@dxos/context';
+import { getCredentialAssertion, type CredentialSigner } from '@dxos/credentials';
+import { type AutomergeHost, type MetadataStore, type Space, type SpaceManager } from '@dxos/echo-pipeline';
 import { type FeedStore } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { type Keyring } from '@dxos/keyring';
@@ -80,7 +74,6 @@ export class DataSpaceManager {
   constructor(
     private readonly _spaceManager: SpaceManager,
     private readonly _metadataStore: MetadataStore,
-    private readonly _dataServiceSubscriptions: DataServiceSubscriptions,
     private readonly _keyring: Keyring,
     private readonly _signingContext: SigningContext,
     private readonly _feedStore: FeedStore<FeedMessage>,
@@ -270,10 +263,6 @@ export class DataSpaceManager {
       callbacks: {
         beforeReady: async () => {
           log('before space ready', { space: space.key });
-          await this._dataServiceSubscriptions.registerSpace(
-            space.key,
-            dataSpace.dataPipeline.databaseHost!.createDataServiceHost(),
-          );
         },
         afterReady: async () => {
           log('after space ready', { space: space.key, open: this._isOpen });
@@ -283,7 +272,6 @@ export class DataSpaceManager {
         },
         beforeClose: async () => {
           log('before space close', { space: space.key });
-          await this._dataServiceSubscriptions.unregisterSpace(space.key);
         },
       },
       cache: metadata.cache,
@@ -297,9 +285,6 @@ export class DataSpaceManager {
     if (metadata.controlTimeframe) {
       dataSpace.inner.controlPipeline.state.setTargetTimeframe(metadata.controlTimeframe);
     }
-    if (metadata.dataTimeframe) {
-      dataSpace.dataPipeline.setTargetTimeframe(metadata.dataTimeframe);
-    }
 
     this._spaces.set(metadata.key, dataSpace);
     return dataSpace;

package/src/packlets/spaces/data-space.ts

@@ -6,14 +6,7 @@ import { Event, asyncTimeout, scheduleTask, sleep, synchronized, trackLeaks } fr
 import { AUTH_TIMEOUT } from '@dxos/client-protocol';
 import { cancelWithContext, Context, ContextDisposedError } from '@dxos/context';
 import { timed, warnAfterTimeout } from '@dxos/debug';
-import {
-  type MetadataStore,
-  type Space,
-  createMappedFeedWriter,
-  type DataPipeline,
-  type CreateEpochOptions,
-  type AutomergeHost,
-} from '@dxos/echo-pipeline';
+import { type MetadataStore, type Space, createMappedFeedWriter, type AutomergeHost } from '@dxos/echo-pipeline';
 import { type FeedStore } from '@dxos/feed-store';
 import { failedInvariant, invariant } from '@dxos/invariant';
 import { type Keyring } from '@dxos/keyring';
@@ -71,11 +64,9 @@ export type DataSpaceParams = {
   automergeHost: AutomergeHost;
 };
 
-/**
- * Delete feed blocks after an epoch is created.
- */
-// TODO(dmaretskyi): Disabled till better times https://github.com/dxos/dxos/issues/3949.
-const ENABLE_FEED_PURGE = false;
+export type CreateEpochOptions = {
+  migration?: CreateEpochRequest.Migration;
+};
 
 @trackLeaks('open', 'close')
 @trace.resource()
@@ -160,10 +151,6 @@ export class DataSpace {
     return this._inner;
   }
 
-  get dataPipeline(): DataPipeline {
-    return this._inner.dataPipeline;
-  }
-
   get presence() {
     return this._presence;
   }
@@ -278,20 +265,8 @@ export class DataSpace {
 
     this._automergeSpaceState.startProcessingRootDocs();
 
-    await this._inner.initializeDataPipeline();
-
-    this.metrics.dataPipelineOpen = new Date();
-
     // Wait for the first epoch.
-    await cancelWithContext(this._ctx, this._inner.dataPipeline.ensureEpochInitialized());
-
-    log('waiting for data pipeline to reach target timeframe');
-    // Wait for the data pipeline to catch up to its desired timeframe.
-    await this._inner.dataPipeline.pipelineState!.waitUntilReachedTargetTimeframe({
-      ctx: this._ctx,
-      breakOnStall: false,
-    });
-    this.metrics.dataPipelineReady = new Date();
+    await cancelWithContext(this._ctx, this.automergeSpaceState.ensureEpochInitialized());
 
     log('data pipeline ready');
     await this._callbacks.beforeReady?.();
@@ -426,7 +401,12 @@ export class DataSpace {
       case CreateEpochRequest.Migration.NONE:
         {
           // TODO(dmaretskyi): Unify epoch construction.
-          epoch = await this.dataPipeline.createEpoch();
+          epoch = {
+            previousId: this._automergeSpaceState.lastEpoch?.id,
+            number: (this._automergeSpaceState.lastEpoch?.subject.assertion.number ?? -1) + 1,
+            timeframe: this._automergeSpaceState.lastEpoch?.subject.assertion.timeframe ?? new Timeframe(),
+            automergeRoot: this._automergeSpaceState.lastEpoch?.subject.assertion?.automergeRoot,
+          };
         }
         break;
       case CreateEpochRequest.Migration.INIT_AUTOMERGE:
@@ -476,18 +456,6 @@ export class DataSpace {
     });
 
     await this.inner.controlPipeline.state.waitUntilTimeframe(new Timeframe([[receipt.feedKey, receipt.seq]]));
-
-    // Clear feed blocks before epoch.
-    if (ENABLE_FEED_PURGE) {
-      for (const feed of this.inner.dataPipeline.pipelineState?.feeds ?? []) {
-        const indexBeforeEpoch = epoch.timeframe.get(feed.key);
-        if (indexBeforeEpoch === undefined) {
-          continue;
-        }
-
-        await feed.safeClear(0, indexBeforeEpoch + 1);
-      }
-    }
   }
 
   @synchronized

package/src/packlets/spaces/spaces-service.test.ts

@@ -24,15 +24,10 @@ describe('SpacesService', () => {
   beforeEach(async () => {
     serviceContext = createServiceContext();
     await serviceContext.open(new Context());
-    spacesService = new SpacesServiceImpl(
-      serviceContext.identityManager,
-      serviceContext.spaceManager,
-      serviceContext.dataServiceSubscriptions,
-      async () => {
-        await serviceContext.initialized.wait();
-        return serviceContext.dataSpaceManager!;
-      },
-    );
+    spacesService = new SpacesServiceImpl(serviceContext.identityManager, serviceContext.spaceManager, async () => {
+      await serviceContext.initialized.wait();
+      return serviceContext.dataSpaceManager!;
+    });
   });
 
   afterEach(async () => {

package/src/packlets/spaces/spaces-service.ts

@@ -6,18 +6,18 @@ import { EventSubscriptions, UpdateScheduler, scheduleTask } from '@dxos/async';
 import { Stream } from '@dxos/codec-protobuf';
 import { type CredentialProcessor } from '@dxos/credentials';
 import { raise } from '@dxos/debug';
-import { type DataServiceSubscriptions, type SpaceManager } from '@dxos/echo-pipeline';
+import { type SpaceManager } from '@dxos/echo-pipeline';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { ApiError, SpaceNotFoundError, encodeError } from '@dxos/protocols';
 import {
+  SpaceMember,
+  SpaceState,
   type CreateEpochRequest,
   type PostMessageRequest,
   type QueryCredentialsRequest,
   type QuerySpacesResponse,
   type Space,
-  SpaceMember,
-  SpaceState,
   type SpacesService,
   type SubscribeMessagesRequest,
   type UpdateSpaceRequest,
@@ -35,7 +35,6 @@ export class SpacesServiceImpl implements SpacesService {
   constructor(
     private readonly _identityManager: IdentityManager,
     private readonly _spaceManager: SpaceManager,
-    private readonly _dataServiceSubscriptions: DataServiceSubscriptions,
     private readonly _getDataSpaceManager: Provider<Promise<DataSpaceManager>>,
   ) {}
 
@@ -96,13 +95,10 @@ export class SpacesServiceImpl implements SpacesService {
             subscriptions.add(space.stateUpdate.on(ctx, () => scheduler.forceTrigger()));
 
             subscriptions.add(space.presence.updated.on(ctx, () => scheduler.trigger()));
-            subscriptions.add(space.dataPipeline.onNewEpoch.on(ctx, () => scheduler.trigger()));
+            subscriptions.add(space.automergeSpaceState.onNewEpoch.on(ctx, () => scheduler.trigger()));
 
             // Pipeline progress.
             subscriptions.add(space.inner.controlPipeline.state.timeframeUpdate.on(ctx, () => scheduler.trigger()));
-            if (space.dataPipeline.pipelineState) {
-              subscriptions.add(space.dataPipeline.pipelineState.timeframeUpdate.on(ctx, () => scheduler.trigger()));
-            }
           }
         };
 
@@ -190,19 +186,19 @@ export class SpacesServiceImpl implements SpacesService {
       state: space.state,
       error: space.error ? encodeError(space.error) : undefined,
       pipeline: {
-        currentEpoch: space.dataPipeline.currentEpoch,
-        appliedEpoch: space.dataPipeline.appliedEpoch,
+        currentEpoch: space.automergeSpaceState.lastEpoch,
+        appliedEpoch: space.automergeSpaceState.lastEpoch,
 
         controlFeeds: space.inner.controlPipeline.state.feeds.map((feed) => feed.key),
         currentControlTimeframe: space.inner.controlPipeline.state.timeframe,
         targetControlTimeframe: space.inner.controlPipeline.state.targetTimeframe,
         totalControlTimeframe: space.inner.controlPipeline.state.endTimeframe,
 
-        dataFeeds: space.dataPipeline.pipelineState?.feeds.map((feed) => feed.key) ?? [],
-        startDataTimeframe: space.dataPipeline.pipelineState?.startTimeframe,
-        currentDataTimeframe: space.dataPipeline.pipelineState?.timeframe,
-        targetDataTimeframe: space.dataPipeline.pipelineState?.targetTimeframe,
-        totalDataTimeframe: space.dataPipeline.pipelineState?.endTimeframe,
+        dataFeeds: undefined,
+        startDataTimeframe: undefined,
+        currentDataTimeframe: undefined,
+        targetDataTimeframe: undefined,
+        totalDataTimeframe: undefined,
       },
       members: Array.from(space.inner.spaceState.members.values()).map((member) => {
         const peers = space.presence.getPeersOnline().filter(({ identityKey }) => identityKey.equals(member.key));

package/src/packlets/testing/test-builder.ts

@@ -6,24 +6,15 @@ import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
 import { createCredentialSignerWithChain, CredentialGenerator } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
-import {
-  SnapshotStore,
-  type DataPipeline,
-  MetadataStore,
-  SpaceManager,
-  valueEncoding,
-  DataServiceSubscriptions,
-  AutomergeHost,
-} from '@dxos/echo-pipeline';
-import { testLocalDatabase } from '@dxos/echo-pipeline/testing';
+import { AutomergeHost, MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
 import { Keyring } from '@dxos/keyring';
 import { MemorySignalManager, MemorySignalManagerContext } from '@dxos/messaging';
 import { MemoryTransportFactory, NetworkManager } from '@dxos/network-manager';
-import { createStorage, type Storage, StorageType } from '@dxos/random-access-storage';
+import { createStorage, StorageType, type Storage } from '@dxos/random-access-storage';
 import { BlobStore } from '@dxos/teleport-extension-object-sync';
 
-import { ClientServicesHost, createDefaultModelFactory, ServiceContext } from '../services';
+import { ClientServicesHost, ServiceContext } from '../services';
 import { DataSpaceManager, type SigningContext } from '../spaces';
 
 //
@@ -51,8 +42,7 @@ export const createServiceContext = ({
     transportFactory: MemoryTransportFactory,
   });
 
-  const modelFactory = createDefaultModelFactory();
-  return new ServiceContext(storage, networkManager, signalManager, modelFactory);
+  return new ServiceContext(storage, networkManager, signalManager);
 };
 
 export const createPeers = async (numPeers: number) => {
@@ -72,16 +62,6 @@ export const createIdentity = async (peer: ServiceContext) => {
   return peer;
 };
 
-// TODO(burdon): Remove @dxos/client-testing.
-// TODO(burdon): Create builder and make configurable.
-/**
- * @deprecated
- */
-export const syncItemsLocal = async (db1: DataPipeline, db2: DataPipeline) => {
-  await testLocalDatabase(db1, db2);
-  await testLocalDatabase(db2, db1);
-};
-
 export class TestBuilder {
   public readonly signalContext = new MemorySignalManagerContext();
   private readonly _ctx = new Context();
@@ -171,7 +151,6 @@ export class TestPeer {
       feedStore: this.feedStore,
       networkManager: this.networkManager,
       metadataStore: this.metadataStore,
-      modelFactory: createDefaultModelFactory(),
       snapshotStore: this.snapshotStore,
       blobStore: this.blobStore,
     }));
@@ -189,7 +168,6 @@ export class TestPeer {
     return (this._props.dataSpaceManager ??= new DataSpaceManager(
       this.spaceManager,
       this.metadataStore,
-      new DataServiceSubscriptions(),
       this.keyring,
       this.identity,
       this.feedStore,

package/src/version.ts

@@ -1 +1 @@
-export const DXOS_VERSION = "0.4.9-main.a108ade";
+export const DXOS_VERSION = "0.4.9-main.a3cfaa7";