@dxos/echo-pipeline 0.1.41 → 0.1.45-next.45667d4

package/src/pipeline/pipeline.ts

@@ -4,14 +4,16 @@
 
 import assert from 'node:assert';
 
-import { sleep, synchronized, Trigger } from '@dxos/async';
+import { Event, sleep, synchronized, Trigger } from '@dxos/async';
 import { Context, rejectOnDispose } from '@dxos/context';
+import { failUndefined } from '@dxos/debug';
 import { FeedSetIterator, FeedWrapper, FeedWriter } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { FeedMessageBlock } from '@dxos/protocols';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
+import { ComplexMap } from '@dxos/util';
 
 import { createMappedFeedWriter } from '../common';
 import { createMessageSelector } from './message-selector';
@@ -37,6 +39,8 @@ export class PipelineState {
   // TODO(dmaretskyi): Remove?.
   public readonly timeframeUpdate = this._timeframeClock.update;
 
+  public readonly stalled = new Event();
+
   /**
    * Target timeframe we are waiting to reach.
    */
@@ -46,7 +50,7 @@ export class PipelineState {
 
   // prettier-ignore
   constructor(
-    private _iterator: FeedSetIterator<any>,
+    private _feeds: ComplexMap<PublicKey, FeedWrapper<FeedMessage>>,
     private _timeframeClock: TimeframeClock
   ) { }
 
@@ -57,12 +61,12 @@ export class PipelineState {
   // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
   get endTimeframe() {
     return mapFeedIndexesToTimeframe(
-      this._iterator.feeds
+      Array.from(this._feeds.values())
         .filter((feed) => feed.length > 0)
         .map((feed) => ({
           feedKey: feed.key,
-          index: feed.length - 1
-        }))
+          index: feed.length - 1,
+        })),
     );
   }
 
@@ -79,7 +83,7 @@ export class PipelineState {
   }
 
   get feeds() {
-    return this._iterator.feeds;
+    return Array.from(this._feeds.values());
   }
 
   async waitUntilTimeframe(target: Timeframe) {
@@ -100,19 +104,19 @@ export class PipelineState {
   async waitUntilReachedTargetTimeframe({
     ctx = new Context(),
     timeout,
-    breakOnStall = true
+    breakOnStall = true,
   }: WaitUntilReachedTargetParams = {}) {
     log('waitUntilReachedTargetTimeframe', {
       timeout,
       current: this.timeframe,
-      target: this.targetTimeframe
+      target: this.targetTimeframe,
     });
 
     this._reachedTargetPromise ??= Promise.race([
       this._timeframeClock.update.waitForCondition(() => {
         return Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
       }),
-      ...(breakOnStall ? [this._iterator.stalled.discardParameter().waitForCount(1)] : [])
+      ...(breakOnStall ? [this.stalled.discardParameter().waitForCount(1)] : []),
     ]);
 
     let done = false;
@@ -132,9 +136,9 @@ export class PipelineState {
             timeout,
             current: this.timeframe,
             target: this.targetTimeframe,
-            dependencies: Timeframe.dependencies(this.targetTimeframe, this.timeframe)
+            dependencies: Timeframe.dependencies(this.targetTimeframe, this.timeframe),
           });
-        })
+        }),
       ]);
     } else {
       return this._reachedTargetPromise;
@@ -180,33 +184,31 @@ export interface PipelineAccessor {
  * 5. Wait for the writable feed to be added.
  */
 export class Pipeline implements PipelineAccessor {
-  private readonly _timeframeClock = new TimeframeClock(this._initialTimeframe);
+  private readonly _timeframeClock = new TimeframeClock(new Timeframe());
+
+  private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<FeedMessage>>(PublicKey.hash);
 
   // Inbound feed stream.
-  private readonly _feedSetIterator = new FeedSetIterator<FeedMessage>(createMessageSelector(this._timeframeClock), {
-    start: startAfter(this._initialTimeframe),
-    stallTimeout: 1000
-  });
+  private _feedSetIterator?: FeedSetIterator<FeedMessage>;
 
   // External state accessor.
-  private readonly _state: PipelineState = new PipelineState(this._feedSetIterator, this._timeframeClock);
+  private readonly _state: PipelineState = new PipelineState(this._feeds, this._timeframeClock);
 
   // Waits for the message consumer to process the message and yield control back to the pipeline.
   private readonly _processingTrigger = new Trigger().wake();
 
-  // Outbound feed writer.
-  private _writer: FeedWriter<FeedMessage.Payload> | undefined;
+  private readonly _pauseTrigger = new Trigger().wake();
+
+  private _isStopping = false;
+
+  private _isStarted = false;
 
   private _isOpen = false;
 
-  // prettier-ignore
-  constructor(
-    private readonly _initialTimeframe: Timeframe = new Timeframe()
-  ) {
-    this._feedSetIterator.stalled.on((iterator) => {
-      log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`);
-    });
-  }
+  private _isPaused = false;
+
+  // Outbound feed writer.
+  private _writer: FeedWriter<FeedMessage.Payload> | undefined;
 
   get state() {
     return this._state;
@@ -218,17 +220,20 @@ export class Pipeline implements PipelineAccessor {
   }
 
   getFeeds(): FeedWrapper<FeedMessage>[] {
-    return this._feedSetIterator.feeds;
+    return this._feedSetIterator!.feeds;
   }
 
   // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
   // which might be opening feeds during the mutation processing, which w
   async addFeed(feed: FeedWrapper<FeedMessage>) {
-    await this._feedSetIterator.addFeed(feed);
+    this._feeds.set(feed.key, feed);
+    if (this._feedSetIterator) {
+      await this._feedSetIterator.addFeed(feed);
+    }
   }
 
   hasFeed(feedKey: PublicKey) {
-    return this._feedSetIterator.hasFeed(feedKey);
+    return this._feeds.has(feedKey);
   }
 
   setWriteFeed(feed: FeedWrapper<FeedMessage>) {
@@ -238,25 +243,69 @@ export class Pipeline implements PipelineAccessor {
     this._writer = createMappedFeedWriter<FeedMessage.Payload, FeedMessage>(
       (payload: FeedMessage.Payload) => ({
         timeframe: this._timeframeClock.timeframe,
-        payload
+        payload,
       }),
-      feed.createFeedWriter()
+      feed.createFeedWriter(),
     );
   }
 
   @synchronized
   async start() {
     log('starting...', {});
-    await this._feedSetIterator.open();
+    await this._initIterator();
+    await this._feedSetIterator!.open();
     log('started');
+    this._isStarted = true;
   }
 
   @synchronized
   async stop() {
     log('stopping...', {});
-    await this._feedSetIterator.close();
+    this._isStopping = true;
+    await this._feedSetIterator?.close();
     await this._processingTrigger.wait(); // Wait for the in-flight message to be processed.
     log('stopped');
+    this._isStarted = false;
+  }
+
+  /**
+   * @param timeframe Timeframe of already processed messages. The pipeline will start processing messages AFTER this timeframe.
+   */
+  @synchronized
+  async setCursor(timeframe: Timeframe) {
+    assert(!this._isStarted || this._isPaused, 'Invalid state.');
+
+    this._timeframeClock.setTimeframe(timeframe);
+
+    if (this._feedSetIterator) {
+      await this._feedSetIterator.close();
+      await this._initIterator();
+      await this._feedSetIterator.open();
+    }
+  }
+
+  /**
+   * Calling pause while processing will cause a deadlock.
+   */
+  @synchronized
+  async pause() {
+    assert(this._isStarted, 'Pipeline is not open.');
+    if (this._isPaused) {
+      return;
+    }
+
+    this._pauseTrigger.reset();
+    await this._processingTrigger.wait();
+    this._isPaused = true;
+  }
+
+  @synchronized
+  async unpause() {
+    assert(this._isStarted, 'Pipeline is not open.');
+    assert(this._isPaused, 'Pipeline is not paused.');
+
+    this._pauseTrigger.wake();
+    this._isPaused = false;
   }
 
   /**
@@ -267,7 +316,26 @@ export class Pipeline implements PipelineAccessor {
     assert(!this._isOpen, 'Pipeline is already being consumed.');
     this._isOpen = true;
 
-    for await (const block of this._feedSetIterator) {
+    assert(this._feedSetIterator, 'Iterator not initialized.');
+    let lastFeedSetIterator = this._feedSetIterator;
+    let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
+
+    while (!this._isStopping) {
+      await this._pauseTrigger.wait();
+
+      // Iterator might have been changed while we were waiting for the processing to complete.
+      if (lastFeedSetIterator !== this._feedSetIterator) {
+        assert(this._feedSetIterator, 'Iterator not initialized.');
+        lastFeedSetIterator = this._feedSetIterator;
+        iterable = lastFeedSetIterator[Symbol.asyncIterator]();
+      }
+
+      const { done, value } = await iterable.next();
+      if (done) {
+        continue;
+      }
+
+      const block = value ?? failUndefined();
       this._processingTrigger.reset();
       this._timeframeClock.updatePendingTimeframe(PublicKey.from(block.feedKey), block.seq);
       yield block;
@@ -279,4 +347,19 @@ export class Pipeline implements PipelineAccessor {
     // TODO(burdon): Test re-entrant?
     this._isOpen = false;
   }
+
+  private async _initIterator() {
+    this._feedSetIterator = new FeedSetIterator<FeedMessage>(createMessageSelector(this._timeframeClock), {
+      start: startAfter(this._timeframeClock.timeframe),
+      stallTimeout: 1000,
+    });
+    this._feedSetIterator.stalled.on((iterator) => {
+      log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`);
+      this._state.stalled.emit();
+    });
+
+    for (const feed of this._feeds.values()) {
+      await this._feedSetIterator.addFeed(feed);
+    }
+  }
 }

package/src/pipeline/timeframe-clock.ts

@@ -48,6 +48,12 @@ export class TimeframeClock {
     return this._pendingTimeframe;
   }
 
+  setTimeframe(timeframe: Timeframe) {
+    this._timeframe = timeframe;
+    this._pendingTimeframe = timeframe;
+    this.update.emit(this._timeframe);
+  }
+
   updatePendingTimeframe(key: PublicKey, seq: number) {
     this._pendingTimeframe = Timeframe.merge(this._pendingTimeframe, new Timeframe([[key, seq]]));
   }
@@ -64,12 +70,12 @@ export class TimeframeClock {
 
   @timed(5_000)
   async waitUntilReached(target: Timeframe) {
-    log.info('waitUntilReached', { target, current: this._timeframe });
+    log('waitUntilReached', { target, current: this._timeframe });
     await this.update.waitForCondition(() => {
       log('check if reached', {
         target,
         current: this._timeframe,
-        deps: Timeframe.dependencies(target, this._timeframe)
+        deps: Timeframe.dependencies(target, this._timeframe),
       });
 
       return Timeframe.dependencies(target, this._timeframe).isEmpty();

package/src/space/auth.ts

@@ -27,18 +27,18 @@ export class AuthExtension extends RpcExtension<Services, Services> {
   private readonly _ctx = new Context({
     onError: (err) => {
       log.catch(err);
-    }
+    },
   });
 
   constructor(private readonly _authParams: AuthExtensionParams) {
     super({
       requested: {
-        AuthService: schema.getService('dxos.mesh.teleport.auth.AuthService')
+        AuthService: schema.getService('dxos.mesh.teleport.auth.AuthService'),
       },
       exposed: {
-        AuthService: schema.getService('dxos.mesh.teleport.auth.AuthService')
+        AuthService: schema.getService('dxos.mesh.teleport.auth.AuthService'),
       },
-      timeout: 60 * 1000 // Long timeout because auth can wait for sync in certain cases.
+      timeout: 60 * 1000, // Long timeout because auth can wait for sync in certain cases.
     });
   }
 
@@ -56,8 +56,8 @@ export class AuthExtension extends RpcExtension<Services, Services> {
             log.error('failed to generate auth credentials', err);
             throw new Error('auth rejected');
           }
-        }
-      }
+        },
+      },
     };
   }
 

package/src/space/control-pipeline.test.ts

@@ -30,9 +30,9 @@ describe('space/control-pipeline', () => {
         root: createStorage({ type: StorageType.RAM }).createDirectory(),
         signer: keyring,
         hypercore: {
-          valueEncoding
-        }
-      })
+          valueEncoding,
+        },
+      }),
     });
 
     const createFeed = async () => {
@@ -45,8 +45,7 @@ describe('space/control-pipeline', () => {
     const controlPipeline = new ControlPipeline({
       spaceKey,
       genesisFeed,
-      initialTimeframe: new Timeframe(),
-      feedProvider: (key) => feedStore.openFeed(key)
+      feedProvider: (key) => feedStore.openFeed(key),
     });
 
     const admittedFeeds: PublicKey[] = [];
@@ -72,8 +71,8 @@ describe('space/control-pipeline', () => {
       for (const credential of credentials) {
         await controlPipeline.pipeline.writer?.write({
           credential: {
-            credential
-          }
+            credential,
+          },
         });
       }
 
@@ -95,10 +94,10 @@ describe('space/control-pipeline', () => {
               spaceKey,
               identityKey,
               deviceKey,
-              designation: AdmittedFeed.Designation.CONTROL
-            }
-          })
-        }
+              designation: AdmittedFeed.Designation.CONTROL,
+            },
+          }),
+        },
       });
 
       await controlPipeline.pipeline.state.waitUntilTimeframe(controlPipeline.pipeline.state.endTimeframe);
@@ -119,10 +118,10 @@ describe('space/control-pipeline', () => {
               spaceKey,
               identityKey,
               deviceKey,
-              designation: AdmittedFeed.Designation.DATA
-            }
-          })
-        }
+              designation: AdmittedFeed.Designation.DATA,
+            },
+          }),
+        },
       });
 
       const end = controlPipeline.pipeline.state.endTimeframe;
@@ -147,12 +146,12 @@ describe('space/control-pipeline', () => {
                 spaceKey,
                 identityKey,
                 deviceKey,
-                designation: AdmittedFeed.Designation.DATA
-              }
-            })
-          }
+                designation: AdmittedFeed.Designation.DATA,
+              },
+            }),
+          },
         },
-        timeframe: new Timeframe()
+        timeframe: new Timeframe(),
       });
 
       await controlPipeline.pipeline.state.waitUntilTimeframe(controlPipeline.pipeline.state.endTimeframe);

package/src/space/control-pipeline.ts

@@ -8,7 +8,6 @@ import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed, Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
-import { Timeframe } from '@dxos/timeframe';
 import { AsyncCallback, Callback } from '@dxos/util';
 
 import { Pipeline, PipelineAccessor } from '../pipeline';
@@ -17,7 +16,6 @@ export type ControlPipelineParams = {
   spaceKey: PublicKey;
   genesisFeed: FeedWrapper<FeedMessage>;
   feedProvider: (feedKey: PublicKey) => Promise<FeedWrapper<FeedMessage>>;
-  initialTimeframe?: Timeframe;
 };
 
 /**
@@ -31,8 +29,8 @@ export class ControlPipeline {
   public readonly onMemberAdmitted: Callback<AsyncCallback<MemberInfo>>;
   public readonly onCredentialProcessed: Callback<AsyncCallback<Credential>>;
 
-  constructor({ spaceKey, genesisFeed, feedProvider, initialTimeframe }: ControlPipelineParams) {
-    this._pipeline = new Pipeline(initialTimeframe);
+  constructor({ spaceKey, genesisFeed, feedProvider }: ControlPipelineParams) {
+    this._pipeline = new Pipeline();
     void this._pipeline.addFeed(genesisFeed); // TODO(burdon): Require async open/close?
 
     this._spaceStateMachine = new SpaceStateMachine(spaceKey);
@@ -79,7 +77,7 @@ export class ControlPipeline {
           if (msg.data.payload.credential) {
             const result = await this._spaceStateMachine.process(
               msg.data.payload.credential.credential,
-              PublicKey.from(msg.feedKey)
+              PublicKey.from(msg.feedKey),
             );
             if (!result) {
               log.warn('processing failed', { msg });