@dxos/client-services 0.10.0 → 0.11.0

package/src/packlets/space/control-pipeline.test.ts

@@ -0,0 +1,165 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { describe, expect, onTestFinished, test } from 'vitest';
+
+import { Context } from '@dxos/context';
+import { CredentialGenerator, createCredential } from '@dxos/credentials';
+import { FeedFactory, FeedStore } from '@dxos/feed-store';
+import { Keyring } from '@dxos/keyring';
+import { type PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { StorageType, createStorage } from '@dxos/random-access-storage';
+import { Timeframe } from '@dxos/timeframe';
+
+import { MetadataStore } from '../metadata';
+import { valueEncoding } from '../pipeline';
+import { ControlPipeline } from './control-pipeline';
+
+describe('space/control-pipeline', () => {
+  test('admits feeds', async () => {
+    const keyring = new Keyring();
+    const spaceKey = await keyring.createKey();
+    const identityKey = await keyring.createKey();
+    const deviceKey = await keyring.createKey();
+
+    const feedStore = new FeedStore<FeedMessage>({
+      factory: new FeedFactory<FeedMessage>({
+        root: createStorage({ type: StorageType.RAM }).createDirectory(),
+        signer: keyring,
+        hypercore: {
+          valueEncoding,
+        },
+      }),
+    });
+
+    const createFeed = async () => {
+      const feedKey = await keyring.createKey();
+      return feedStore.openFeed(feedKey, { writable: true });
+    };
+
+    // TODO(dmaretskyi): Separate test for cold start after genesis.
+    const genesisFeed = await createFeed();
+    const metadata = new MetadataStore(createStorage({ type: StorageType.RAM }).createDirectory());
+    await metadata.addSpace({ key: spaceKey, genesisFeedKey: genesisFeed.key, controlFeedKey: genesisFeed.key });
+    const controlPipeline = new ControlPipeline({
+      spaceKey,
+      genesisFeed,
+      feedProvider: (key) => feedStore.openFeed(key),
+      metadataStore: metadata,
+    });
+
+    const admittedFeeds: PublicKey[] = [];
+    controlPipeline.onFeedAdmitted.set(async (info) => {
+      log.debug('feed admitted');
+      admittedFeeds.push(info.key);
+    });
+    expect(admittedFeeds).toEqual([]);
+
+    await controlPipeline.setWriteFeed(genesisFeed);
+    await controlPipeline.start(Context.default());
+
+    onTestFinished(() => controlPipeline.stop());
+
+    //
+    // Genesis
+    //
+    {
+      const generator = new CredentialGenerator(keyring, identityKey, deviceKey);
+      const credentials = await generator.createSpaceGenesis(spaceKey, genesisFeed.key);
+      expect(credentials).toHaveLength(3);
+
+      for (const credential of credentials) {
+        await controlPipeline.pipeline.writer?.write({
+          credential: {
+            credential,
+          },
+        });
+      }
+
+      await controlPipeline.pipeline.state.waitUntilTimeframe(controlPipeline.pipeline.state.endTimeframe);
+      expect(admittedFeeds).toEqual([genesisFeed.key]);
+    }
+
+    // New control feed.
+    const controlFeed2 = await createFeed();
+    {
+      await controlPipeline.pipeline.writer!.write({
+        credential: {
+          credential: await createCredential({
+            signer: keyring,
+            issuer: identityKey,
+            subject: controlFeed2.key,
+            assertion: {
+              '@type': 'dxos.halo.credentials.AdmittedFeed',
+              spaceKey,
+              identityKey,
+              deviceKey,
+              'designation': AdmittedFeed.Designation.CONTROL,
+            },
+          }),
+        },
+      });
+
+      await controlPipeline.pipeline.state.waitUntilTimeframe(controlPipeline.pipeline.state.endTimeframe);
+      expect(admittedFeeds).toEqual([genesisFeed.key, controlFeed2.key]);
+    }
+
+    // New data feed.
+    const dataFeed1 = await createFeed();
+    {
+      await controlPipeline.pipeline.writer!.write({
+        credential: {
+          credential: await createCredential({
+            signer: keyring,
+            issuer: identityKey,
+            subject: dataFeed1.key,
+            assertion: {
+              '@type': 'dxos.halo.credentials.AdmittedFeed',
+              spaceKey,
+              identityKey,
+              deviceKey,
+              'designation': AdmittedFeed.Designation.DATA,
+            },
+          }),
+        },
+      });
+
+      const end = controlPipeline.pipeline.state.endTimeframe;
+      await controlPipeline.pipeline.state.waitUntilTimeframe(end);
+      expect(admittedFeeds).toEqual([genesisFeed.key, controlFeed2.key, dataFeed1.key]);
+    }
+
+    // TODO(dmaretskyi): Move to other test (data feed cannot admit feeds).
+    const dataFeed2 = await createFeed();
+    {
+      await dataFeed1.append({
+        payload: {
+          '@type': 'dxos.echo.feed.FeedMessage',
+          'timeframe': controlPipeline.pipeline.state.timeframe,
+          'credential': {
+            credential: await createCredential({
+              signer: keyring,
+              issuer: identityKey,
+              subject: dataFeed2.key,
+              assertion: {
+                '@type': 'dxos.halo.credentials.AdmittedFeed',
+                spaceKey,
+                identityKey,
+                deviceKey,
+                'designation': AdmittedFeed.Designation.DATA,
+              },
+            }),
+          },
+        },
+        timeframe: new Timeframe(),
+      });
+
+      await controlPipeline.pipeline.state.waitUntilTimeframe(controlPipeline.pipeline.state.endTimeframe);
+      expect(admittedFeeds).toEqual([genesisFeed.key, controlFeed2.key, dataFeed1.key]);
+    }
+  });
+});

package/src/packlets/space/control-pipeline.ts

@@ -0,0 +1,229 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { DeferredTask, scheduleMicroTask, sleepWithContext, trackLeaks } from '@dxos/async';
+import { Context } from '@dxos/context';
+import {
+  type DelegateInvitationCredential,
+  type FeedInfo,
+  type MemberInfo,
+  type SpaceState,
+  SpaceStateMachine,
+} from '@dxos/credentials';
+import { type FeedWrapper } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { type FeedMessageBlock } from '@dxos/protocols';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { type ControlPipelineSnapshot } from '@dxos/protocols/proto/dxos/echo/metadata';
+import { AdmittedFeed, type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { Timeframe } from '@dxos/timeframe';
+import { trace } from '@dxos/tracing';
+import { type AsyncCallback, Callback, tracer } from '@dxos/util';
+
+import { type IMetadataStore } from '../metadata';
+import { Pipeline, type PipelineAccessor } from '../pipeline';
+
+export type ControlPipelineProps = {
+  spaceKey: PublicKey;
+  genesisFeed: FeedWrapper<FeedMessage>;
+  feedProvider: (feedKey: PublicKey) => Promise<FeedWrapper<FeedMessage>>;
+  metadataStore: IMetadataStore;
+};
+
+const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
+
+const CONTROL_PIPELINE_SNAPSHOT_DELAY = 10_000;
+
+const USE_SNAPSHOTS = true;
+
+/**
+ * Processes HALO credentials, which include genesis and invitations.
+ */
+@trackLeaks('start', 'stop')
+export class ControlPipeline {
+  private readonly _ctx = new Context();
+  private readonly _pipeline: Pipeline;
+  private readonly _spaceStateMachine: SpaceStateMachine;
+
+  private readonly _spaceKey: PublicKey;
+  private readonly _metadata: IMetadataStore;
+  private _targetTimeframe?: Timeframe;
+  private _lastTimeframeSaveTime: number = Date.now();
+
+  public readonly onFeedAdmitted = new Callback<AsyncCallback<FeedInfo>>();
+  public readonly onMemberRoleChanged: Callback<AsyncCallback<MemberInfo[]>>;
+  public readonly onCredentialProcessed: Callback<AsyncCallback<Credential>>;
+  public readonly onDelegatedInvitation: Callback<AsyncCallback<DelegateInvitationCredential>>;
+  public readonly onDelegatedInvitationRemoved: Callback<AsyncCallback<DelegateInvitationCredential>>;
+
+  private _snapshotTask = new DeferredTask(this._ctx, async () => {
+    await sleepWithContext(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
+    await this._saveSnapshot();
+  });
+
+  constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }: ControlPipelineProps) {
+    this._spaceKey = spaceKey;
+    this._metadata = metadataStore;
+    this._pipeline = new Pipeline();
+    void this._pipeline.addFeed(genesisFeed); // TODO(burdon): Require async open/close?
+
+    this._spaceStateMachine = new SpaceStateMachine(spaceKey);
+    this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
+      // log('feed admitted', { info });
+      log('feed admitted', { key: info.key });
+
+      // TODO(burdon): Check not stopping.
+      if (info.assertion.designation === AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
+        scheduleMicroTask(this._ctx, async () => {
+          try {
+            const feed = await feedProvider(info.key);
+            if (this._ctx.disposed) {
+              return;
+            }
+            if (!this._pipeline.hasFeed(feed.key)) {
+              await this._pipeline.addFeed(feed);
+            }
+          } catch (err: any) {
+            log.catch(err);
+          }
+        });
+      }
+
+      await this.onFeedAdmitted.callIfSet(info);
+    });
+
+    this.onMemberRoleChanged = this._spaceStateMachine.onMemberRoleChanged;
+    this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
+    this.onDelegatedInvitation = this._spaceStateMachine.onDelegatedInvitation;
+    this.onDelegatedInvitationRemoved = this._spaceStateMachine.onDelegatedInvitationRemoved;
+  }
+
+  get spaceState(): SpaceState {
+    return this._spaceStateMachine;
+  }
+
+  get pipeline(): PipelineAccessor {
+    return this._pipeline;
+  }
+
+  async setWriteFeed(feed: FeedWrapper<FeedMessage>): Promise<void> {
+    await this._pipeline.addFeed(feed);
+    this._pipeline.setWriteFeed(feed);
+  }
+
+  @trace.span({ showInBrowserTimeline: true, op: 'lifecycle' })
+  async start(ctx: Context): Promise<void> {
+    const snapshot = this._metadata.getSpaceControlPipelineSnapshot(this._spaceKey);
+    log('load snapshot', { key: this._spaceKey, present: !!snapshot, tf: snapshot?.timeframe });
+    if (USE_SNAPSHOTS && snapshot) {
+      await this._processSnapshot(snapshot);
+    }
+
+    log('starting...');
+    setTimeout(async () => {
+      void this._consumePipeline(ctx);
+    });
+
+    await this._pipeline.start();
+    log('started');
+  }
+
+  private async _processSnapshot(snapshot: ControlPipelineSnapshot): Promise<void> {
+    await this._pipeline.setCursor(snapshot.timeframe);
+
+    for (const message of snapshot.messages ?? []) {
+      const result = await this._spaceStateMachine.process(message.credential, {
+        sourceFeed: message.feedKey,
+        skipVerification: true,
+      });
+
+      if (!result) {
+        log.warn('credential processing failed from snapshot', { message });
+      }
+    }
+  }
+
+  private async _saveSnapshot(): Promise<void> {
+    await this._pipeline.pause();
+    const snapshot: ControlPipelineSnapshot = {
+      timeframe: this._pipeline.state.timeframe,
+      messages: this._spaceStateMachine.credentialEntries.map((entry) => ({
+        feedKey: entry.sourceFeed,
+        credential: entry.credential,
+      })),
+    };
+    await this._pipeline.unpause();
+
+    log('save snapshot', { key: this._spaceKey, snapshot: getSnapshotLoggerContext(snapshot) });
+    await this._metadata.setSpaceControlPipelineSnapshot(this._spaceKey, snapshot);
+  }
+
+  private async _consumePipeline(ctx: Context): Promise<void> {
+    for await (const msg of this._pipeline.consume()) {
+      try {
+        await this._processMessage(ctx, msg);
+      } catch (err: any) {
+        log.catch(err);
+      }
+    }
+  }
+
+  @trace.span({ showInBrowserTimeline: true, showInRemoteTracing: false })
+  private async _processMessage(ctx: Context, msg: FeedMessageBlock): Promise<void> {
+    log('processing', { key: msg.feedKey, seq: msg.seq });
+    if (msg.data.payload.credential) {
+      const timer = tracer.mark('dxos.echo.pipeline.control');
+      const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, {
+        sourceFeed: PublicKey.from(msg.feedKey),
+      });
+
+      timer.end();
+      if (!result) {
+        log.warn('processing failed', { msg });
+      } else {
+        await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
+      }
+
+      this._snapshotTask.schedule();
+    }
+  }
+
+  private async _noteTargetStateIfNeeded(timeframe: Timeframe): Promise<void> {
+    // TODO(dmaretskyi): Replace this with a proper debounce/throttle.
+
+    if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
+      this._lastTimeframeSaveTime = Date.now();
+
+      await this._saveTargetTimeframe(timeframe);
+    }
+  }
+
+  async stop(): Promise<void> {
+    log('stopping...');
+    await this._ctx.dispose();
+    await this._pipeline.stop();
+    await this._saveTargetTimeframe(this._pipeline.state.timeframe);
+    log('stopped');
+  }
+
+  private async _saveTargetTimeframe(timeframe: Timeframe): Promise<void> {
+    try {
+      const newTimeframe = Timeframe.merge(this._targetTimeframe ?? new Timeframe(), timeframe);
+      await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
+      this._targetTimeframe = newTimeframe;
+    } catch (err: any) {
+      log(err);
+    }
+  }
+}
+
+const getSnapshotLoggerContext = (snapshot: ControlPipelineSnapshot) => {
+  return snapshot.messages?.map((msg) => {
+    const issuer = msg.credential.issuer;
+    const subject = msg.credential.subject.id;
+    const type = msg.credential.subject.assertion['@type'];
+    return { issuer, subject, type };
+  });
+};

package/src/packlets/space/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export * from './auth';
+export * from './space';
+export * from './space-manager';
+export * from './space-protocol';
+export * from './admission-discovery-extension';

package/src/packlets/space/replication.browser.test.ts

@@ -0,0 +1,58 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+import { describe, expect, test } from 'vitest';
+
+import { FeedFactory, FeedStore } from '@dxos/feed-store';
+import { Keyring } from '@dxos/keyring';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { createStorage } from '@dxos/random-access-storage';
+import { Timeframe } from '@dxos/timeframe';
+
+import { valueEncoding } from '../pipeline';
+
+describe('replication', () => {
+  test('replicates a feed through a direct stream', async () => {
+    // Some storage drivers may break when there are multiple storage instances.
+    const storage = createStorage();
+
+    // Creates an appropriate persistent storage for the browser: IDB in Chrome or File storage in Firefox.
+    const keyring1 = new Keyring();
+    const feedStore1 = new FeedStore<FeedMessage>({
+      factory: new FeedFactory<FeedMessage>({
+        root: storage.createDirectory('feeds1'),
+        signer: keyring1,
+        hypercore: {
+          valueEncoding,
+        },
+      }),
+    });
+
+    const keyring2 = new Keyring();
+    const feedStore2 = new FeedStore<FeedMessage>({
+      factory: new FeedFactory<FeedMessage>({
+        root: storage.createDirectory('feeds2'),
+        signer: keyring2,
+        hypercore: {
+          valueEncoding,
+        },
+      }),
+    });
+
+    const feed1 = await feedStore1.openFeed(await keyring1.createKey(), {
+      writable: true,
+    });
+    const feed2 = await feedStore2.openFeed(feed1.key);
+
+    const stream1 = feed1.replicate(true, { live: true, noise: false, encrypted: false });
+    const stream2 = feed2.replicate(false, { live: true, noise: false, encrypted: false });
+    stream1.pipe(stream2).pipe(stream1);
+
+    await feed1.append({
+      timeframe: new Timeframe([[feed1.key, 123]]),
+    });
+
+    await expect.poll(() => feed2.properties.length).toEqual(1);
+  });
+});

package/src/packlets/space/space-manager.browser.test.ts

@@ -0,0 +1,40 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { describe, onTestFinished, test } from 'vitest';
+
+import { createStorage } from '@dxos/random-access-storage';
+
+import { TestAgentBuilder } from './testing';
+
+describe('space-manager', () => {
+  test.skip('invitations', async () => {
+    const builder = new TestAgentBuilder({
+      storage: createStorage(),
+    });
+    onTestFinished(async () => {
+      await builder.close();
+    });
+
+    const peer1 = await builder.createPeer();
+    const spaceManager1 = peer1.spaceManager;
+    await spaceManager1.open();
+
+    const peer2 = await builder.createPeer();
+    const spaceManager2 = peer2.spaceManager;
+    await spaceManager2.open();
+
+    onTestFinished(() => spaceManager1.close());
+    onTestFinished(() => spaceManager2.close());
+
+    // const space1 = await spaceManager1.createSpace();
+    // expect(space1.isOpen).to.be.true;
+
+    // TODO(burdon): Create invitation and join.
+    // TODO(burdon): Need to factor out DataInvitations from services.
+
+    // const space2 = await spaceManager2.acceptSpace();
+    // expect(space2.key).not.to.be.undefined;
+  });
+});