@dxos/feed-store 0.7.4 → 0.7.5-feature-compute.4d9d99a

package/dist/types/src/feed-set-iterator.d.ts

@@ -29,6 +29,11 @@ export declare class FeedSetIterator<T extends {}> extends AbstractFeedIterator<
     private readonly _trigger;
     private readonly _subscriptions;
     readonly stalled: Event<FeedSetIterator<T>>;
+    /**
+     * Blocks to be retried once all feeds are processed or stalled.
+     * It is used to retry blocks that failed to be processed.
+     */
+    private readonly _reiterateBlocks;
     constructor(_selector: FeedBlockSelector<T>, options?: FeedSetIteratorOptions);
     [inspect.custom](): string;
     toJSON(): {
@@ -39,10 +44,12 @@ export declare class FeedSetIterator<T extends {}> extends AbstractFeedIterator<
     get size(): number;
     get feeds(): FeedWrapper<T>[];
     get indexes(): FeedIndex[];
+    reiterateBlock(block: FeedBlock<T>): void;
     addFeed(feed: FeedWrapper<T>): Promise<void>;
     hasFeed(feedKey: PublicKey): boolean;
     _onOpen(): Promise<void>;
     _onClose(): Promise<void>;
+    private _getNextBlockForReiteration;
     /**
      * Gets the next block from the selected queue.
      */

package/dist/types/src/feed-set-iterator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"feed-set-iterator.d.ts","sourceRoot":"","sources":["../../../src/feed-set-iterator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,KAAK,EAA+B,MAAM,aAAa,CAAC;AAGjE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAIvC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,SAAS,CAAC;AAEzC;;GAEG;AACH,MAAM,MAAM,iBAAiB,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,GAAG,SAAS,CAAC;AAElF,MAAM,MAAM,SAAS,GAAG;IACtB,OAAO,EAAE,SAAS,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IAEnC,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,eAAO,MAAM,6BAA6B;;CAEzC,CAAC;AAEF;;GAEG;AACH,qBAAa,eAAe,CAAC,CAAC,SAAS,EAAE,CAAE,SAAQ,oBAAoB,CAAC,CAAC,CAAC;IAStE,OAAO,CAAC,QAAQ,CAAC,SAAS;aACV,OAAO,EAAE,sBAAsB;IATjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2D;IAEvF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoC;IAC7D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA4B;IAE3D,SAAgB,OAAO,4BAAmC;gBAGvC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAChC,OAAO,GAAE,sBAAsD;IAOjF,CAAC,OAAO,CAAC,MAAM,CAAC;IAIP,MAAM;;;;;IAQf,IAAI,IAAI,WAEP;IAED,IAAI,KAAK,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,CAE5B;IAED,IAAI,OAAO,IAAI,SAAS,EAAE,CAKzB;IAEK,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;IAsBlC,OAAO,CAAC,OAAO,EAAE,SAAS;IAIX,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAMxB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAQxC;;OAEG;IACY,UAAU,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;CA8C/D"}
+{"version":3,"file":"feed-set-iterator.d.ts","sourceRoot":"","sources":["../../../src/feed-set-iterator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,KAAK,EAA+B,MAAM,aAAa,CAAC;AAGjE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAIvC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,SAAS,CAAC;AAEzC;;GAEG;AACH,MAAM,MAAM,iBAAiB,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,GAAG,SAAS,CAAC;AAElF,MAAM,MAAM,SAAS,GAAG;IACtB,OAAO,EAAE,SAAS,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IAEnC,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,eAAO,MAAM,6BAA6B;;CAEzC,CAAC;AAEF;;GAEG;AACH,qBAAa,eAAe,CAAC,CAAC,SAAS,EAAE,CAAE,SAAQ,oBAAoB,CAAC,CAAC,CAAC;IAetE,OAAO,CAAC,QAAQ,CAAC,SAAS;aACV,OAAO,EAAE,sBAAsB;IAfjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2D;IAEvF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoC;IAC7D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA4B;IAE3D,SAAgB,OAAO,4BAAmC;IAE1D;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAsB;gBAGpC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAChC,OAAO,GAAE,sBAAsD;IAOjF,CAAC,OAAO,CAAC,MAAM,CAAC;IAIP,MAAM;;;;;IAQf,IAAI,IAAI,WAEP;IAED,IAAI,KAAK,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,CAE5B;IAED,IAAI,OAAO,IAAI,SAAS,EAAE,CAKzB;IAED,cAAc,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;IAK5B,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;IAsBlC,OAAO,CAAC,OAAO,EAAE,SAAS;IAIX,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAMxB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAQxC,OAAO,CAAC,2BAA2B;IAOnC;;OAEG;IACY,UAAU,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;CAsD/D"}

package/dist/types/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"version":"5.7.2"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/feed-store",
-  "version": "0.7.4",
+  "version": "0.7.5-feature-compute.4d9d99a",
   "description": "A consistent store for hypercore feeds.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -37,27 +37,27 @@
     "lodash.defaultsdeep": "^4.6.1",
     "race-as-promised": "^0.0.2",
     "streamx": "^2.12.5",
-    "@dxos/async": "0.7.4",
-    "@dxos/context": "0.7.4",
-    "@dxos/codec-protobuf": "0.7.4",
-    "@dxos/crypto": "0.7.4",
-    "@dxos/debug": "0.7.4",
-    "@dxos/hypercore": "0.7.4",
-    "@dxos/invariant": "0.7.4",
-    "@dxos/keyring": "0.7.4",
-    "@dxos/keys": "0.7.4",
-    "@dxos/node-std": "0.7.4",
-    "@dxos/log": "0.7.4",
-    "@dxos/random-access-storage": "0.7.4",
-    "@dxos/util": "0.7.4"
+    "@dxos/async": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/codec-protobuf": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/context": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/debug": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/hypercore": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/invariant": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/keyring": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/keys": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/crypto": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/node-std": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/log": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/random-access-storage": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/util": "0.7.5-feature-compute.4d9d99a"
   },
   "devDependencies": {
     "@types/lodash.defaultsdeep": "^4.6.6",
-    "@dxos/random": "0.7.4",
-    "@dxos/keys": "0.7.4"
+    "@dxos/keys": "0.7.5-feature-compute.4d9d99a",
+    "@dxos/random": "0.7.5-feature-compute.4d9d99a"
   },
   "optionalDependencies": {
-    "@dxos/random": "0.7.4"
+    "@dxos/random": "0.7.5-feature-compute.4d9d99a"
   },
   "publishConfig": {
     "access": "public"

package/src/feed-set-iterator.ts

@@ -47,6 +47,12 @@ export class FeedSetIterator<T extends {}> extends AbstractFeedIterator<T> {
 
   public readonly stalled = new Event<FeedSetIterator<T>>();
 
+  /**
+   * Blocks to be retried once all feeds are processed or stalled.
+   * It is used to retry blocks that failed to be processed.
+   */
+  private readonly _reiterateBlocks: FeedBlock<T>[] = [];
+
   constructor(
     private readonly _selector: FeedBlockSelector<T>,
     public readonly options: FeedSetIteratorOptions = defaultFeedSetIteratorOptions,
@@ -83,6 +89,11 @@ export class FeedSetIterator<T extends {}> extends AbstractFeedIterator<T> {
     }));
   }
 
+  reiterateBlock(block: FeedBlock<T>) {
+    this._reiterateBlocks.push(block);
+    this._trigger.wake();
+  }
+
   async addFeed(feed: FeedWrapper<T>) {
     invariant(!this._feedQueues.has(feed.key), `Feed already added: ${feed.key}`);
     invariant(feed.properties.opened);
@@ -123,6 +134,13 @@ export class FeedSetIterator<T extends {}> extends AbstractFeedIterator<T> {
     this._trigger.wake();
   }
 
+  private _getNextBlockForReiteration() {
+    const block = this._reiterateBlocks.shift();
+    invariant(block);
+    log('reiterate', { key: block?.feedKey.toHex(), seq: block?.seq });
+    return block;
+  }
+
   /**
    * Gets the next block from the selected queue.
    */
@@ -130,6 +148,7 @@ export class FeedSetIterator<T extends {}> extends AbstractFeedIterator<T> {
     let t: NodeJS.Timeout | undefined;
 
     while (this._running) {
+      let stalled = false;
       // Get blocks from the head of each queue.
       const queues = Array.from(this._feedQueues.values());
       const blocks = queues.map((queue) => queue.peek()).filter(isNotNullOrUndefined);
@@ -142,6 +161,8 @@ export class FeedSetIterator<T extends {}> extends AbstractFeedIterator<T> {
           if (t === undefined) {
             t = setTimeout(() => {
               this.stalled.emit(this);
+              this._trigger.wake();
+              stalled = true;
             }, this.options.stallTimeout);
           }
         } else {
@@ -166,10 +187,15 @@ export class FeedSetIterator<T extends {}> extends AbstractFeedIterator<T> {
             // console.log(Array.from(this._feedQueues.values()));
           }
         }
+      } else if (this._reiterateBlocks.length) {
+        return this._getNextBlockForReiteration();
       }
 
       // Wait until feed added, new block, or closing.
       await this._trigger.wait();
+      if (stalled && this._reiterateBlocks.length) {
+        return this._getNextBlockForReiteration();
+      }
     }
   }
 }