@dxos/client-services 0.1.41 → 0.1.45-next.45667d4

package/src/packlets/spaces/data-space.ts

@@ -3,31 +3,25 @@
 //
 
 import { Event, scheduleTask, trackLeaks } from '@dxos/async';
-import { SpaceState } from '@dxos/client';
-import { Context } from '@dxos/context';
+import { cancelWithContext, Context } from '@dxos/context';
 import { CredentialConsumer } from '@dxos/credentials';
 import { timed } from '@dxos/debug';
-import {
-  MetadataStore,
-  Space,
-  SigningContext,
-  createMappedFeedWriter,
-  SnapshotManager,
-  DataPipeline
-} from '@dxos/echo-pipeline';
+import { MetadataStore, Space, createMappedFeedWriter, DataPipeline } from '@dxos/echo-pipeline';
 import { CancelledError, SystemError } from '@dxos/errors';
 import { FeedStore } from '@dxos/feed-store';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { ModelFactory } from '@dxos/model-factory';
+import { SpaceState, Space as SpaceProto } from '@dxos/protocols/proto/dxos/client/services';
 import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { SpaceCache } from '@dxos/protocols/proto/dxos/echo/metadata';
 import { AdmittedFeed, Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { GossipMessage } from '@dxos/protocols/proto/dxos/mesh/teleport/gossip';
 import { Gossip, Presence } from '@dxos/teleport-extension-gossip';
 import { ComplexSet } from '@dxos/util';
 
 import { TrustedKeySetAuthVerifier } from '../identity';
+import { SigningContext } from './data-space-manager';
 import { NotarizationPlugin } from './notarization-plugin';
 
 const AUTH_TIMEOUT = 30000;
@@ -46,23 +40,19 @@ export type DataSpaceCallbacks = {
 
 export type DataSpaceParams = {
   inner: Space;
-  modelFactory: ModelFactory;
   metadataStore: MetadataStore;
-  snapshotManager: SnapshotManager;
   gossip: Gossip;
   presence: Presence;
   keyring: Keyring;
   feedStore: FeedStore<FeedMessage>;
   signingContext: SigningContext;
-  memberKey: PublicKey;
-  snapshotId?: string | undefined;
   callbacks?: DataSpaceCallbacks;
+  cache?: SpaceCache;
 };
 
 @trackLeaks('open', 'close')
 export class DataSpace {
   private readonly _ctx = new Context();
-  private readonly _dataPipeline: DataPipeline;
   private readonly _inner: Space;
   private readonly _gossip: Gossip;
   private readonly _presence: Presence;
@@ -72,12 +62,20 @@ export class DataSpace {
   private readonly _signingContext: SigningContext;
   private readonly _notarizationPluginConsumer: CredentialConsumer<NotarizationPlugin>;
   private readonly _callbacks: DataSpaceCallbacks;
+  private readonly _cache?: SpaceCache;
 
   private _state = SpaceState.CLOSED;
 
+  /**
+   * Error for _state === SpaceState.ERROR.
+   */
+  public error: Error | undefined = undefined;
+
   public readonly authVerifier: TrustedKeySetAuthVerifier;
   public readonly stateUpdate = new Event();
 
+  public metrics: SpaceProto.Metrics = {};
+
   constructor(params: DataSpaceParams) {
     this._inner = params.inner;
     this._inner.stateUpdate.on(this._ctx, () => this.stateUpdate.emit());
@@ -89,23 +87,15 @@ export class DataSpace {
     this._metadataStore = params.metadataStore;
     this._signingContext = params.signingContext;
     this._callbacks = params.callbacks ?? {};
-    this._dataPipeline = new DataPipeline({
-      modelFactory: params.modelFactory,
-      metadataStore: params.metadataStore,
-      snapshotManager: params.snapshotManager,
-      memberKey: params.memberKey,
-      spaceKey: this._inner.key,
-      feedInfoProvider: (feedKey) => this._inner.spaceState.feeds.get(feedKey),
-      snapshotId: params.snapshotId
-    });
 
     this.authVerifier = new TrustedKeySetAuthVerifier({
       trustedKeysProvider: () => new ComplexSet(PublicKey.hash, Array.from(this._inner.spaceState.members.keys())),
       update: this._inner.stateUpdate,
-      authTimeout: AUTH_TIMEOUT
+      authTimeout: AUTH_TIMEOUT,
     });
 
     this._notarizationPluginConsumer = this._inner.spaceState.registerProcessor(new NotarizationPlugin());
+    this._cache = params.cache;
   }
 
   get key() {
@@ -126,7 +116,7 @@ export class DataSpace {
   }
 
   get dataPipeline(): DataPipeline {
-    return this._dataPipeline;
+    return this._inner.dataPipeline;
   }
 
   get presence() {
@@ -137,18 +127,22 @@ export class DataSpace {
     return this._notarizationPluginConsumer.processor;
   }
 
+  get cache() {
+    return this._cache;
+  }
+
   async open() {
     await this.notarizationPlugin.open();
     await this._notarizationPluginConsumer.open();
     await this._inner.open();
     this._state = SpaceState.INACTIVE;
+    this.metrics.open = new Date();
   }
 
   async close() {
     this._state = SpaceState.CLOSED;
     await this._ctx.dispose();
 
-    await this._dataPipeline.close();
     await this.authVerifier.close();
 
     await this._inner.close();
@@ -172,6 +166,7 @@ export class DataSpace {
   initializeDataPipelineAsync() {
     scheduleTask(this._ctx, async () => {
       try {
+        this.metrics.pipelineInitBegin = new Date();
         await this.initializeDataPipeline();
       } catch (err) {
         if (err instanceof CancelledError) {
@@ -180,6 +175,11 @@ export class DataSpace {
         }
 
         log.error('Error initializing data pipeline', err);
+        this._state = SpaceState.ERROR;
+        this.error = err as Error;
+        this.stateUpdate.emit();
+      } finally {
+        this.metrics.ready = new Date();
       }
     });
   }
@@ -193,9 +193,11 @@ export class DataSpace {
     // TODO(dmaretskyi): Cancel with context.
     await this._inner.controlPipeline.state.waitUntilReachedTargetTimeframe({
       ctx: this._ctx,
-      breakOnStall: false
+      breakOnStall: false,
     });
 
+    this.metrics.controlPipelineReady = new Date();
+
     await this._createWritableFeeds();
     log('Writable feeds created');
     this.stateUpdate.emit();
@@ -203,27 +205,28 @@ export class DataSpace {
     this.notarizationPlugin.setWriter(
       createMappedFeedWriter<Credential, FeedMessage.Payload>(
         (credential) => ({
-          credential: { credential }
+          credential: { credential },
         }),
-        this._inner.controlPipeline.writer
-      )
+        this._inner.controlPipeline.writer,
+      ),
     );
 
-    await this._dataPipeline.open({
-      openPipeline: async (start) => {
-        const pipeline = await this._inner.createDataPipeline({ start });
-        await pipeline.start();
-        return pipeline;
-      }
-    });
+    await this._inner.initializeDataPipeline();
+
+    this.metrics.dataPipelineOpen = new Date();
+
+    // Wait for the first epoch.
+    await cancelWithContext(this._ctx, this._inner.dataPipeline.ensureEpochInitialized());
 
     log('waiting for data pipeline to reach target timeframe');
     // Wait for the data pipeline to catch up to its desired timeframe.
-    await this._dataPipeline.pipelineState!.waitUntilReachedTargetTimeframe({
+    await this._inner.dataPipeline.pipelineState!.waitUntilReachedTargetTimeframe({
       ctx: this._ctx,
-      breakOnStall: false
+      breakOnStall: false,
     });
 
+    this.metrics.dataPipelineReady = new Date();
+
     log('data pipeline ready');
     await this._callbacks.beforeReady?.();
 
@@ -248,9 +251,9 @@ export class DataSpace {
             spaceKey: this.key,
             deviceKey: this._signingContext.deviceKey,
             identityKey: this._signingContext.identityKey,
-            designation: AdmittedFeed.Designation.CONTROL
-          }
-        })
+            designation: AdmittedFeed.Designation.CONTROL,
+          },
+        }),
       );
     }
     if (!this.inner.dataFeedKey) {
@@ -265,9 +268,9 @@ export class DataSpace {
             spaceKey: this.key,
             deviceKey: this._signingContext.deviceKey,
             identityKey: this._signingContext.identityKey,
-            designation: AdmittedFeed.Designation.DATA
-          }
-        })
+            designation: AdmittedFeed.Designation.DATA,
+          },
+        }),
       );
     }
 
@@ -279,4 +282,27 @@ export class DataSpace {
     // Set this after credentials are notarized so that on failure we will retry.
     await this._metadataStore.setWritableFeedKeys(this.key, this.inner.controlFeedKey!, this.inner.dataFeedKey!);
   }
+
+  async createEpoch() {
+    const epoch = await this.dataPipeline.createEpoch();
+
+    await this.inner.controlPipeline.writer.write({
+      credential: {
+        credential: await this._signingContext.credentialSigner.createCredential({
+          subject: this.key,
+          assertion: {
+            '@type': 'dxos.halo.credentials.Epoch',
+            ...epoch,
+          },
+        }),
+      },
+    });
+
+    for (const feed of this.inner.dataPipeline.pipelineState?.feeds ?? []) {
+      const indexBeforeEpoch = epoch.timeframe.get(feed.key);
+      if (indexBeforeEpoch) {
+        await feed.clear(0, indexBeforeEpoch + 1);
+      }
+    }
+  }
 }

package/src/packlets/spaces/genesis.ts

@@ -0,0 +1,81 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { createCredential } from '@dxos/credentials';
+import { failUndefined } from '@dxos/debug';
+import { Space } from '@dxos/echo-pipeline';
+import { Keyring } from '@dxos/keyring';
+import { AdmittedFeed, SpaceMember } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { Timeframe } from '@dxos/timeframe';
+
+import { SigningContext } from './data-space-manager';
+
+export const spaceGenesis = async (keyring: Keyring, signingContext: SigningContext, space: Space) => {
+  // TODO(dmaretskyi): Find a way to reconcile with credential generator.
+  const credentials = [
+    await createCredential({
+      signer: keyring,
+      issuer: space.key,
+      subject: space.key,
+      assertion: {
+        '@type': 'dxos.halo.credentials.SpaceGenesis',
+        spaceKey: space.key,
+      },
+    }),
+
+    await createCredential({
+      signer: keyring,
+      issuer: space.key,
+      subject: signingContext.identityKey,
+      assertion: {
+        '@type': 'dxos.halo.credentials.SpaceMember',
+        spaceKey: space.key,
+        role: SpaceMember.Role.ADMIN,
+        profile: signingContext.profile,
+        genesisFeedKey: space.controlFeedKey ?? failUndefined(),
+      },
+    }),
+
+    await signingContext.credentialSigner.createCredential({
+      subject: space.controlFeedKey ?? failUndefined(),
+      assertion: {
+        '@type': 'dxos.halo.credentials.AdmittedFeed',
+        spaceKey: space.key,
+        identityKey: signingContext.identityKey,
+        deviceKey: signingContext.deviceKey,
+        designation: AdmittedFeed.Designation.CONTROL,
+      },
+    }),
+
+    await signingContext.credentialSigner.createCredential({
+      subject: space.dataFeedKey ?? failUndefined(),
+      assertion: {
+        '@type': 'dxos.halo.credentials.AdmittedFeed',
+        spaceKey: space.key,
+        identityKey: signingContext.identityKey,
+        deviceKey: signingContext.deviceKey,
+        designation: AdmittedFeed.Designation.DATA,
+      },
+    }),
+
+    await signingContext.credentialSigner.createCredential({
+      subject: space.key ?? failUndefined(),
+      assertion: {
+        '@type': 'dxos.halo.credentials.Epoch',
+        number: 0,
+        previousId: undefined,
+        timeframe: new Timeframe(),
+        snapshotCid: undefined,
+      },
+    }),
+  ];
+
+  for (const credential of credentials) {
+    await space.controlPipeline.writer.write({
+      credential: { credential },
+    });
+  }
+
+  return credentials;
+};

package/src/packlets/spaces/notarization-plugin.test.ts

@@ -67,11 +67,11 @@ describe('NotarizationPlugin', () => {
     const credential = await generator.createFeedAdmission(
       await keyring.createKey(),
       await keyring.createKey(),
-      AdmittedFeed.Designation.CONTROL
+      AdmittedFeed.Designation.CONTROL,
     );
 
     const notarized = peer2.notarizationPlugin.notarize({
-      credentials: [credential]
+      credentials: [credential],
     });
 
     await testBuilder.connect(peer1, peer2);

package/src/packlets/spaces/notarization-plugin.ts

@@ -80,12 +80,12 @@ export class NotarizationPlugin implements CredentialProcessor {
     credentials,
     timeout = DEFAULT_NOTARIZE_TIMEOUT,
     retryTimeout = DEFAULT_RETRY_TIMEOUT,
-    successDelay = DEFAULT_SUCCESS_DELAY
+    successDelay = DEFAULT_SUCCESS_DELAY,
   }: NotarizeParams) {
     log('notarize', { credentials });
     assert(
       credentials.every((credential) => credential.id),
-      'Credentials must have an id'
+      'Credentials must have an id',
     );
 
     const errors = new Trigger();
@@ -94,7 +94,7 @@ export class NotarizationPlugin implements CredentialProcessor {
         log.warn('Notarization error', { err });
         void ctx.dispose();
         errors.throw(err);
-      }
+      },
     });
     opCtx?.onDispose(() => ctx.dispose());
 
@@ -105,12 +105,12 @@ export class NotarizationPlugin implements CredentialProcessor {
         () => {
           log.warn('Notarization timeout', {
             timeout,
-            peers: Array.from(this._extensions).map((extension) => extension.remotePeerId)
+            peers: Array.from(this._extensions).map((extension) => extension.remotePeerId),
           });
           void ctx.dispose();
           errors.throw(new TimeoutError(timeout, 'Notarization timed out'));
         },
-        timeout
+        timeout,
       );
     }
 
@@ -137,7 +137,7 @@ export class NotarizationPlugin implements CredentialProcessor {
         peersTried.add(peer);
         log('try notarizing', { peer: peer.localPeerId, credentialId: credentials.map((credential) => credential.id) });
         await peer.rpc.NotarizationService.notarize({
-          credentials: credentials.filter((credential) => !this._processedCredentials.has(credential.id!))
+          credentials: credentials.filter((credential) => !this._processedCredentials.has(credential.id!)),
         });
         log('success');
         await sleep(successDelay); // wait before trying with a new peer
@@ -209,7 +209,7 @@ export class NotarizationPlugin implements CredentialProcessor {
         log('extension closed', { peer: extension.localPeerId });
         this._extensions.delete(extension);
       },
-      onNotarize: this._onNotarize.bind(this)
+      onNotarize: this._onNotarize.bind(this),
     });
     return extension;
   }
@@ -225,11 +225,11 @@ export class NotarizationTeleportExtension extends RpcExtension<Services, Servic
   constructor(private readonly _params: NotarizationTeleportExtensionParams) {
     super({
       requested: {
-        NotarizationService: schema.getService('dxos.mesh.teleport.notarization.NotarizationService')
+        NotarizationService: schema.getService('dxos.mesh.teleport.notarization.NotarizationService'),
       },
       exposed: {
-        NotarizationService: schema.getService('dxos.mesh.teleport.notarization.NotarizationService')
-      }
+        NotarizationService: schema.getService('dxos.mesh.teleport.notarization.NotarizationService'),
+      },
     });
   }
 
@@ -238,8 +238,8 @@ export class NotarizationTeleportExtension extends RpcExtension<Services, Servic
       NotarizationService: {
         notarize: async (request) => {
           await this._params.onNotarize(request);
-        }
-      }
+        },
+      },
     };
   }
 

package/src/packlets/spaces/spaces-service.test.ts

@@ -30,7 +30,7 @@ describe('SpacesService', () => {
       async () => {
         await serviceContext.initialized.wait();
         return serviceContext.dataSpaceManager!;
-      }
+      },
     );
   });
 
@@ -69,7 +69,7 @@ describe('SpacesService', () => {
       const existingSpaces = [
         await spacesService.createSpace(),
         await spacesService.createSpace(),
-        await spacesService.createSpace()
+        await spacesService.createSpace(),
       ];
 
       const query = spacesService.querySpaces();

package/src/packlets/spaces/spaces-service.ts

@@ -7,7 +7,9 @@ import { Stream } from '@dxos/codec-protobuf';
 import { raise, todo } from '@dxos/debug';
 import { DataServiceSubscriptions, SpaceManager, SpaceNotFoundError } from '@dxos/echo-pipeline';
 import { log } from '@dxos/log';
+import { encodeError } from '@dxos/protocols';
 import {
+  CreateEpochRequest,
   PostMessageRequest,
   QueryCredentialsRequest,
   QuerySpacesResponse,
@@ -16,11 +18,11 @@ import {
   SpacesService,
   SubscribeMessagesRequest,
   UpdateSpaceRequest,
-  WriteCredentialsRequest
+  WriteCredentialsRequest,
 } from '@dxos/protocols/proto/dxos/client/services';
 import { Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { GossipMessage } from '@dxos/protocols/proto/dxos/mesh/teleport/gossip';
-import { humanize, Provider } from '@dxos/util';
+import { Provider, humanize } from '@dxos/util';
 
 import { IdentityManager } from '../identity';
 import { DataSpace } from './data-space';
@@ -36,7 +38,7 @@ export class SpacesServiceImpl implements SpacesService {
     private readonly _identityManager: IdentityManager,
     private readonly _spaceManager: SpaceManager,
     private readonly _dataServiceSubscriptions: DataServiceSubscriptions,
-    private readonly _getDataSpaceManager: Provider<Promise<DataSpaceManager>>
+    private readonly _getDataSpaceManager: Provider<Promise<DataSpaceManager>>,
   ) {}
 
   async createSpace(): Promise<Space> {
@@ -83,7 +85,7 @@ export class SpacesServiceImpl implements SpacesService {
               subscriptions.add(
                 space.dataPipeline.pipelineState.timeframeUpdate
                   .debounce(TIMEFRAME_UPDATE_DEBOUNCE_TIME)
-                  .on(ctx, onUpdate)
+                  .on(ctx, onUpdate),
               );
             }
           }
@@ -130,7 +132,7 @@ export class SpacesServiceImpl implements SpacesService {
       const processor = space.spaceState.registerProcessor({
         process: async (credential) => {
           next(credential);
-        }
+        },
       });
       ctx.onDispose(() => processor.close());
       scheduleTask(ctx, () => processor.open());
@@ -144,10 +146,17 @@ export class SpacesServiceImpl implements SpacesService {
     }
   }
 
+  async createEpoch({ spaceKey }: CreateEpochRequest) {
+    const dataSpaceManager = await this._getDataSpaceManager();
+    const space = dataSpaceManager.spaces.get(spaceKey) ?? raise(new SpaceNotFoundError(spaceKey));
+    await space.createEpoch();
+  }
+
   private _transformSpace(space: DataSpace): Space {
     return {
       spaceKey: space.key,
       state: space.state,
+      error: space.error ? encodeError(space.error) : undefined,
       pipeline: {
         controlFeeds: space.inner.controlPipeline.state.feeds.map((feed) => feed.key),
         currentControlTimeframe: space.inner.controlPipeline.state.timeframe,
@@ -157,21 +166,23 @@ export class SpacesServiceImpl implements SpacesService {
         dataFeeds: space.dataPipeline.pipelineState?.feeds.map((feed) => feed.key) ?? [],
         currentDataTimeframe: space.dataPipeline.pipelineState?.timeframe,
         targetDataTimeframe: space.dataPipeline.pipelineState?.targetTimeframe,
-        totalDataTimeframe: space.dataPipeline.pipelineState?.endTimeframe
+        totalDataTimeframe: space.dataPipeline.pipelineState?.endTimeframe,
       },
       members: Array.from(space.inner.spaceState.members.values()).map((member) => ({
         identity: {
           identityKey: member.key,
           profile: {
-            displayName: member.assertion.profile?.displayName ?? humanize(member.key)
-          }
+            displayName: member.assertion.profile?.displayName ?? humanize(member.key),
+          },
         },
         presence:
           this._identityManager.identity?.identityKey.equals(member.key) ||
           space.presence.getPeersOnline().filter(({ identityKey }) => identityKey.equals(member.key)).length > 0
             ? SpaceMember.PresenceState.ONLINE
-            : SpaceMember.PresenceState.OFFLINE
-      }))
+            : SpaceMember.PresenceState.OFFLINE,
+      })),
+      cache: space.cache,
+      metrics: space.metrics,
     };
   }
 }

package/src/packlets/storage/storage.ts

@@ -16,7 +16,7 @@ export const createStorageObjects = (config: Runtime.Client.Storage) => {
     path = 'dxos/storage', // TODO(burdon): Factor out const.
     storageType,
     keyStorage,
-    persistent = false
+    persistent = false,
   } = config ?? {};
 
   if (persistent && storageType === StorageDriver.RAM) {
@@ -35,8 +35,8 @@ export const createStorageObjects = (config: Runtime.Client.Storage) => {
   return {
     storage: createStorage({
       type: persistent ? toStorageType(storageType) : StorageType.RAM,
-      root: `${path}/`
-    })
+      root: `${path}/`,
+    }),
   };
 };
 

package/src/packlets/system/system-service.test.ts

@@ -40,7 +40,7 @@ describe('SystemService', () => {
       },
       onReset: () => {
         reset.wake(true);
-      }
+      },
     });
   });
 

package/src/packlets/system/system-service.ts

@@ -9,7 +9,7 @@ import {
   SystemStatus,
   SystemStatusResponse,
   SystemService,
-  UpdateSystemStatusRequest
+  UpdateSystemStatusRequest,
 } from '@dxos/protocols/proto/dxos/client/services';
 import { MaybePromise } from '@dxos/util';
 

package/src/packlets/testing/credential-utils.ts

@@ -9,7 +9,7 @@ import { Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 
 export const createMockCredential = async ({
   signer,
-  issuer
+  issuer,
 }: {
   signer: Signer;
   issuer: PublicKey;
@@ -22,7 +22,7 @@ export const createMockCredential = async ({
       '@type': 'example.testing.rpc.MessageWithAny',
       payload: {
         '@type': 'google.protobuf.Any',
-        value: Buffer.from('test')
-      }
-    }
+        value: Buffer.from('test'),
+      },
+    },
   });

package/src/packlets/testing/index.ts

@@ -3,6 +3,5 @@
 //
 
 export * from './credential-utils';
-export * from './host-test-builder';
 export * from './invitation-utils';
 export * from './test-builder';